From eb7b22d44ae2da899df4d4df2d1d69afb6e0994b Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sun, 27 Sep 2026 04:27:15 +0200 Subject: [PATCH 01/19] Test what queries hand to a LINQ provider Every query test so far runs over AsQueryable() and compares rows, so a tree that only LINQ to Objects can run passes. These scenarios also check the tree: the query must stay on its provider, and must not contain what a C# expression lambda never does - a block, a parameter nothing binds, a call into FSharp.Core, a join nested in a collection selector, a member read from a tuple built without naming its members. A scenario names what is wrong with its tree today. The name has to be removed by the change that makes it untrue, because a scenario fails when its tree is cleaner than it says. Co-Authored-By: Claude Fable 5.1 --- .../FSharp.Core.UnitTests.fsproj | 9 + .../QueryKeyScenarios.fs | 170 +++++++++ .../QueryMethodSyntaxScenarios.fs | 116 ++++++ .../QueryOperatorScenarios.fs | 197 ++++++++++ .../QueryProjectionScenarios.fs | 130 +++++++ .../QueryProviderShape.fs | 355 ++++++++++++++++++ .../Microsoft.FSharp.Linq/QueryScenario.fs | 189 ++++++++++ .../QueryScenarioDomain.fs | 109 ++++++ .../Microsoft.FSharp.Linq/QueryShapeTests.fs | 31 ++ .../QueryStructureScenarios.fs | 268 +++++++++++++ 10 files changed, 1574 insertions(+) create mode 100644 tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs create mode 100644 tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs create mode 100644 tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryOperatorScenarios.fs create mode 100644 tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs create mode 100644 tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs create mode 100644 tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs create mode 100644 tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenarioDomain.fs create mode 100644 tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs create mode 100644 tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj b/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj index 494509d0906..3a20b60f7e4 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj +++ b/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj @@ -92,6 +92,15 @@ + + + + + + + + + diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs new file mode 100644 index 00000000000..03011a4d99f --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module FSharp.Core.UnitTests.Linq.QueryKeyScenarios + +open System.Linq +open System.Linq.Expressions +open Microsoft.FSharp.Linq.RuntimeHelpers + +let private block = [ Violation.Statement ExpressionType.Block ] + +let private customers (data: Data) = Seq.ofList data.Customers + +/// Orders paired with the lines that share both the order number and, as quantity, the customer number. +let private matchingLines (data: Data) = + seq { + for o in data.Orders do + for l in data.Lines do + if l.OrderId = o.Id && l.Quantity = o.CustomerId then + yield o.Id, l.Id + } + +let private matchingLinesOrNone (data: Data) = + seq { + for o in data.Orders do + match matchingLines data |> Seq.filter (fun (order, _) -> order = o.Id) |> List.ofSeq with + | [] -> yield o.Id, -1 + | matches -> yield! matches + } + +let private joins () = + [ Scenario.rows + "join on a scalar" + (fun db -> query { for c in db.Customers do join o in db.Orders on (c.Id = o.CustomerId); select (c.Name, o.Id) }) + (fun data -> seq { for c in data.Customers do for o in c.Orders -> c.Name, o.Id }) + + Scenario.rows + "join on a tuple" + (fun db -> + query { for o in db.Orders do join l in db.Lines on ((o.Id, o.CustomerId) = (l.OrderId, l.Quantity)); select (o.Id, l.Id) }) + matchingLines + + Scenario.rows + "join on a struct tuple" + (fun db -> + query { + for o in db.Orders do + join l in db.Lines on (struct (o.Id, o.CustomerId) = struct (l.OrderId, l.Quantity)) + select (o.Id, l.Id) + }) + matchingLines + + Scenario.rows + "join on a record, fields in declaration order" + (fun db -> + query { + for o in db.Orders do + join l in db.Lines on ({ OrderNumber = o.Id; Units = o.CustomerId } = { OrderNumber = l.OrderId; Units = l.Quantity }) + select (o.Id, l.Id) + }) + matchingLines + + Scenario.rows + "join on a record, fields out of declaration order" + (fun db -> + query { + for o in db.Orders do + join l in db.Lines on ({ Units = o.CustomerId; OrderNumber = o.Id } = { Units = l.Quantity; OrderNumber = l.OrderId }) + select (o.Id, l.Id) + }) + matchingLines + |> Scenario.violating block + + Scenario.rows + "join on an anonymous record, fields in alphabetical order" + (fun db -> + query { + for o in db.Orders do + join l in db.Lines on ({| Order = o.Id; Units = o.CustomerId |} = {| Order = l.OrderId; Units = l.Quantity |}) + select (o.Id, l.Id) + }) + matchingLines + + Scenario.rows + "join on an anonymous record, fields out of alphabetical order" + (fun db -> + query { + for o in db.Orders do + join l in db.Lines on ({| Units = o.CustomerId; Order = o.Id |} = {| Units = l.Quantity; Order = l.OrderId |}) + select (o.Id, l.Id) + }) + matchingLines + |> Scenario.violating block + + Scenario.rows + "groupJoin on a tuple" + (fun db -> + query { + for o in db.Orders do + groupJoin l in db.Lines on ((o.Id, o.CustomerId) = (l.OrderId, l.Quantity)) into lines + select (o.Id, lines.Count()) + }) + (fun data -> seq { for o in data.Orders -> o.Id, matchingLines data |> Seq.filter (fun (order, _) -> order = o.Id) |> Seq.length }) + + Scenario.rows + "leftOuterJoin on a tuple" + (fun db -> + query { + for o in db.Orders do + leftOuterJoin l in db.Lines on ((o.Id, o.CustomerId) = (l.OrderId, l.Quantity)) into lines + for l in lines do + select (o.Id, (if obj.ReferenceEquals(l, null) then -1 else l.Id)) + }) + matchingLinesOrNone ] + +let private groupings () = + [ Scenario.rows + "groupBy a scalar" + (fun db -> query { for c in db.Customers do groupBy c.City into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy _.City) + + Scenario.rows + "groupBy a scalar, groups themselves" + (fun db -> query { for c in db.Customers do groupBy c.City }) + (customers >> Seq.groupBy _.City >> Seq.map (fun (city, group) -> Grouping(city, group) :> IGrouping<_, _>)) + + Scenario.rows + "groupBy a tuple" + (fun db -> query { for c in db.Customers do groupBy (c.City, c.Age > 28) into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> c.City, c.Age > 28)) + + Scenario.rows + "groupBy a tuple, element of the key" + (fun db -> query { for c in db.Customers do groupBy (c.City, c.Age > 28) into g; select (fst g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> c.City, c.Age > 28) >> Seq.map (fun ((city, _), count) -> city, count)) + |> Scenario.violating [ Violation.RuntimeHelperCall "Operators.Fst" ] + + Scenario.rows + "groupBy a struct tuple" + (fun db -> query { for c in db.Customers do groupBy (struct (c.City, c.Age > 28)) into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> struct (c.City, c.Age > 28))) + + Scenario.rows + "groupBy a record, fields in declaration order" + (fun db -> query { for c in db.Customers do groupBy { KeyCity = c.City; KeyAge = c.Age / 10 } into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> { KeyCity = c.City; KeyAge = c.Age / 10 })) + + Scenario.rows + "groupBy a record, fields out of declaration order" + (fun db -> query { for c in db.Customers do groupBy { KeyAge = c.Age / 10; KeyCity = c.City } into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> { KeyCity = c.City; KeyAge = c.Age / 10 })) + |> Scenario.violating block + + Scenario.rows + "groupBy an anonymous record, fields in alphabetical order" + (fun db -> query { for c in db.Customers do groupBy {| City = c.City; Decade = c.Age / 10 |} into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> {| City = c.City; Decade = c.Age / 10 |})) + + Scenario.rows + "groupBy an anonymous record, fields out of alphabetical order" + (fun db -> query { for c in db.Customers do groupBy {| Decade = c.Age / 10; City = c.City |} into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> {| City = c.City; Decade = c.Age / 10 |})) + |> Scenario.violating block + + Scenario.rows + "groupValBy a tuple of values" + (fun db -> query { for c in db.Customers do groupValBy (c.Name, c.Age) c.City into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy _.City) + |> Scenario.violating [ Violation.RuntimeHelperCall "new Grouping`2" ] ] + +let cases () = joins () @ groupings () diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs new file mode 100644 index 00000000000..b8a704a846e --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module FSharp.Core.UnitTests.Linq.QueryMethodSyntaxScenarios + +open System.Linq +open System.Linq.Expressions + +let private block = [ Violation.Statement ExpressionType.Block ] + +let private reading typeName = + [ Violation.ReadsUnnamedMember typeName ] + +let private customers (data: Data) = Seq.ofList data.Customers + +let cases () = + [ Scenario.rows "Select a tuple" (fun db -> db.Customers.Select(fun c -> (c.Id, c.Name))) (customers >> Seq.map (fun c -> c.Id, c.Name)) + + Scenario.rows + "Select a struct tuple" + (fun db -> db.Customers.Select(fun c -> struct (c.Id, c.Name))) + (customers >> Seq.map (fun c -> struct (c.Id, c.Name))) + + Scenario.rows + "Where over a query built earlier" + (fun db -> + let withLargeOrders = db.Orders.Where(fun o -> o.Total > 4.0).Select(fun o -> o.CustomerId) + db.Customers.Where(fun c -> withLargeOrders.Contains c.Id).Select(fun c -> c.Name)) + (customers >> Seq.filter (fun c -> c.Orders |> Seq.exists (fun o -> o.Total > 4.0)) >> Seq.map _.Name) + + Scenario.rows + "Where over a query built earlier with a let" + (fun db -> + let withLargeOrders = db.Orders.Where(fun o -> let total = o.Total in total > 4.0).Select(fun o -> o.CustomerId) + db.Customers.Where(fun c -> withLargeOrders.Contains c.Id).Select(fun c -> c.Name)) + (customers >> Seq.filter (fun c -> c.Orders |> Seq.exists (fun o -> o.Total > 4.0)) >> Seq.map _.Name) + |> Scenario.violating block + + Scenario.rows + "Select a record, fields in declaration order" + (fun db -> db.Customers.Select(fun c -> { PersonName = c.Name; PersonAge = c.Age })) + (customers >> Seq.map (fun c -> { PersonName = c.Name; PersonAge = c.Age })) + + Scenario.rows + "Select a record, fields out of declaration order" + (fun db -> db.Customers.Select(fun c -> { PersonAge = c.Age; PersonName = c.Name })) + (customers >> Seq.map (fun c -> { PersonName = c.Name; PersonAge = c.Age })) + |> Scenario.violating block + + Scenario.rows + "Select an anonymous record, fields in alphabetical order" + (fun db -> db.Customers.Select(fun c -> {| Age = c.Age; Name = c.Name |})) + (customers >> Seq.map (fun c -> {| Age = c.Age; Name = c.Name |})) + + Scenario.rows + "Select an anonymous record, fields out of alphabetical order" + (fun db -> db.Customers.Select(fun c -> {| Name = c.Name; Age = c.Age |})) + (customers >> Seq.map (fun c -> {| Age = c.Age; Name = c.Name |})) + |> Scenario.violating block + + Scenario.rows + "Select with a let used twice" + (fun db -> db.Customers.Select(fun c -> let upper = c.Name.ToUpper() in upper + upper)) + (customers >> Seq.map (fun c -> c.Name.ToUpper() + c.Name.ToUpper())) + |> Scenario.violating block + + Scenario.rows + "Where taking a tuple apart" + (fun db -> db.Customers.Select(fun c -> (c.Id, c.Name)).Where(fun (id, _) -> id > 1)) + (customers >> Seq.map (fun c -> c.Id, c.Name) >> Seq.filter (fun (id, _) -> id > 1)) + |> Scenario.violating (reading "Tuple`2" @ block) + + Scenario.rows + "Where taking the last of nine apart" + (fun db -> + db.Customers + .Select(fun c -> (c.Id + 100, c.Name, c.City, c.Age, c.Id * 7, c.Name.Length, c.City.Length, c.Age * 2, c.Id)) + .Where(fun (_, _, _, _, _, _, _, _, last) -> last > 1) + .Select(fun (_, name, _, _, _, _, _, _, _) -> name)) + (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) + |> Scenario.violating (reading "Tuple`8" @ block) + + Scenario.rows + "Where over a query that selects a tuple" + (fun db -> (query { for c in db.Customers do select (c.Id, c.Name) }).Where(fun (id, _) -> id > 1)) + (customers >> Seq.map (fun c -> c.Id, c.Name) >> Seq.filter (fun (id, _) -> id > 1)) + |> Scenario.violating (reading "Tuple`2" @ block) + + Scenario.rows + "Where on a nullable property" + (fun db -> db.Customers.Where(fun c -> c.Registered.HasValue && c.Registered.Value.Year > 2020)) + (customers >> Seq.filter (fun c -> c.Registered.HasValue && c.Registered.Value.Year > 2020)) + |> Scenario.violating block + + Scenario.rows + "Where with a lambda over another source" + (fun db -> db.Customers.Where(fun c -> db.Orders.Any(fun o -> o.CustomerId = c.Id))) + (customers >> Seq.filter (fun c -> c.Orders.Count > 0)) + |> Scenario.violating + [ Violation.RuntimeHelperCall "LeafExpressionConverter.QuotationToLambdaExpression" + Violation.RuntimeHelperCall "LeafExpressionConverter.SubstHelper" ] + + Scenario.rows + "OrderBy, ThenByDescending" + (fun db -> db.Customers.OrderBy(fun c -> c.City).ThenByDescending(fun c -> c.Age).Select(fun c -> c.Name)) + (customers >> Seq.sortBy (fun c -> c.City, -c.Age) >> Seq.map _.Name) + |> Scenario.ordered + + Scenario.rows + "GroupBy a tuple" + (fun db -> db.Customers.GroupBy(fun c -> (c.City, c.Age > 28)).Select(fun g -> (g.Key, g.Count()))) + (customers >> Seq.countBy (fun c -> c.City, c.Age > 28)) + + Scenario.rows + "SelectMany over a navigation collection" + (fun db -> db.Customers.SelectMany(fun c -> c.Orders :> seq).Select(fun o -> o.Id)) + (fun data -> data.Orders |> Seq.map _.Id) ] diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryOperatorScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryOperatorScenarios.fs new file mode 100644 index 00000000000..6b2339a996b --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryOperatorScenarios.fs @@ -0,0 +1,197 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module FSharp.Core.UnitTests.Linq.QueryOperatorScenarios + +open System +open System.Linq +open Microsoft.FSharp.Linq +open Microsoft.FSharp.Linq.NullableOperators + +let private customers (data: Data) = Seq.ofList data.Customers + +let private lines (data: Data) = Seq.ofList data.Lines + +let private discountOrLowest (line: OrderLine) = + if line.Discount.HasValue then line.Discount.Value else Int32.MinValue + +let private discounts (data: Data) = + data.Lines |> List.filter _.Discount.HasValue |> List.map _.Discount.Value + +let private sequences () = + [ Scenario.rows + "where" + (fun db -> query { for c in db.Customers do where (c.Age > 26); select c.Name }) + (customers >> Seq.filter (fun c -> c.Age > 26) >> Seq.map _.Name) + + Scenario.rows + "where with string methods" + (fun db -> query { for c in db.Customers do where (c.Name.StartsWith "A" || c.City = "Paris"); select c.Name }) + (customers >> Seq.filter (fun c -> c.Name.StartsWith "A" || c.City = "Paris") >> Seq.map _.Name) + + Scenario.rows + "where on a nullable property" + (fun db -> query { for c in db.Customers do where (c.Registered.HasValue && c.Registered.Value.Year > 2020); select c.Name }) + (customers >> Seq.filter (fun c -> c.Registered.HasValue && c.Registered.Value.Year > 2020) >> Seq.map _.Name) + + Scenario.rows + "where with a nullable operator" + (fun db -> query { for l in db.Lines do where (l.Discount ?> 1); select l.Id }) + (lines >> Seq.filter (fun l -> l.Discount ?> 1) >> Seq.map _.Id) + + Scenario.rows + "if then" + (fun db -> query { for c in db.Customers do if c.Age > 26 then yield c.Name }) + (customers >> Seq.filter (fun c -> c.Age > 26) >> Seq.map _.Name) + + Scenario.rows + "sortBy, thenByDescending" + (fun db -> query { for c in db.Customers do sortBy c.City; thenByDescending c.Age; select c.Name }) + (customers >> Seq.sortBy (fun c -> c.City, -c.Age) >> Seq.map _.Name) + |> Scenario.ordered + |> Scenario.flat [ "OrderBy"; "ThenByDescending"; "Select" ] + + Scenario.rows + "sortByDescending, thenBy" + (fun db -> query { for c in db.Customers do sortByDescending c.City; thenBy c.Age; select c.Name }) + (customers >> Seq.sortBy _.Age >> Seq.sortByDescending _.City >> Seq.map _.Name) + |> Scenario.ordered + |> Scenario.flat [ "OrderByDescending"; "ThenBy"; "Select" ] + + Scenario.rows + "sortByNullable, thenByNullableDescending" + (fun db -> query { for l in db.Lines do sortByNullable l.Discount; thenByNullableDescending (Nullable l.Id); select l.Id }) + (lines >> Seq.sortBy (fun l -> discountOrLowest l, -l.Id) >> Seq.map _.Id) + |> Scenario.ordered + |> Scenario.flat [ "OrderBy"; "ThenByDescending"; "Select" ] + + Scenario.rows + "sortByNullableDescending, thenByNullable" + (fun db -> query { for l in db.Lines do sortByNullableDescending l.Discount; thenByNullable (Nullable l.Id); select l.Id }) + (lines >> Seq.sortBy (fun l -> -(int64 (discountOrLowest l)), l.Id) >> Seq.map _.Id) + |> Scenario.ordered + |> Scenario.flat [ "OrderByDescending"; "ThenBy"; "Select" ] + + Scenario.rows "distinct" (fun db -> query { for c in db.Customers do select c.City; distinct }) (customers >> Seq.map _.City >> Seq.distinct) + + Scenario.rows + "skip, take" + (fun db -> query { for c in db.Customers do sortBy c.Id; skip 1; take 2; select c.Name }) + (customers >> Seq.skip 1 >> Seq.take 2 >> Seq.map _.Name) + |> Scenario.ordered + + Scenario.rows + "skipWhile, takeWhile" + (fun db -> query { for c in db.Customers do sortBy c.Id; skipWhile (c.Id < 2); takeWhile (c.Id < 4); select c.Name }) + (customers >> Seq.skipWhile (fun c -> c.Id < 2) >> Seq.takeWhile (fun c -> c.Id < 4) >> Seq.map _.Name) + |> Scenario.ordered + + Scenario.rows + "join" + (fun db -> query { for o in db.Orders do join l in db.Lines on (o.Id = l.OrderId); select (o.Id, l.Id) }) + (fun data -> seq { for o in data.Orders do for l in o.Lines -> o.Id, l.Id }) + + Scenario.rows + "groupJoin" + (fun db -> query { for o in db.Orders do groupJoin l in db.Lines on (o.Id = l.OrderId) into lines; select (o.Id, lines.Count()) }) + (fun data -> seq { for o in data.Orders -> o.Id, o.Lines.Count }) + + Scenario.rows + "leftOuterJoin" + (fun db -> + query { + for o in db.Orders do + leftOuterJoin l in db.Lines on (o.Id = l.OrderId) into lines + for l in lines do + select (o.Id, (if obj.ReferenceEquals(l, null) then -1 else l.Id)) + }) + (fun data -> + seq { + for o in data.Orders do + if o.Lines.Count = 0 then yield o.Id, -1 else for l in o.Lines -> o.Id, l.Id + }) + + Scenario.rows + "leftOuterJoin tested with isNull" + (fun db -> + query { + for o in db.Orders do + leftOuterJoin l in db.Lines on (o.Id = l.OrderId) into lines + for l in lines do + select (o.Id, isNull l) + }) + (fun data -> + seq { + for o in data.Orders do + if o.Lines.Count = 0 then yield o.Id, true else for _ in o.Lines -> o.Id, false + }) + |> Scenario.violating [ Violation.RuntimeHelperCall "Operators.IsNull" ] ] + +let private values () = + [ Scenario.value "count" (fun db -> query { for c in db.Customers do where (c.Age > 26); count }) (fun _ -> 3) + + Scenario.value "exists" (fun db -> query { for c in db.Customers do exists (c.City = "Paris") }) (fun _ -> true) + + Scenario.value "all" (fun db -> query { for c in db.Customers do all (c.Age > 26) }) (fun _ -> false) + + Scenario.value "contains" (fun db -> query { for c in db.Customers do select c.Id; contains 2 }) (fun _ -> true) + + Scenario.value "find" (fun db -> query { for c in db.Customers do find (c.Age < 30) }) (fun data -> data.Customers.[1]) + + Scenario.value "head" (fun db -> query { for c in db.Customers do sortBy c.Id; head }) (fun data -> data.Customers.[0]) + + Scenario.value + "headOrDefault" + (fun db -> query { for c in db.Customers do where (c.Age > 100); headOrDefault }) + (fun _ -> null) + + Scenario.value "last" (fun db -> query { for c in db.Customers do sortBy c.Id; last }) (fun data -> data.Customers.[3]) + + Scenario.value + "lastOrDefault" + (fun db -> query { for c in db.Customers do where (c.Age > 100); lastOrDefault }) + (fun _ -> null) + + Scenario.value + "exactlyOne" + (fun db -> query { for c in db.Customers do where (c.City = "Paris"); exactlyOne }) + (fun data -> data.Customers.[1]) + + Scenario.value + "exactlyOneOrDefault" + (fun db -> query { for c in db.Customers do where (c.City = "Rome"); exactlyOneOrDefault }) + (fun _ -> null) + + Scenario.value "nth" (fun db -> query { for c in db.Customers do sortBy c.Id; nth 2 }) (fun data -> data.Customers.[2]) + + Scenario.value "minBy" (fun db -> query { for c in db.Customers do minBy c.Age }) (customers >> Seq.map _.Age >> Seq.min) + + Scenario.value "maxBy" (fun db -> query { for c in db.Customers do maxBy c.Age }) (customers >> Seq.map _.Age >> Seq.max) + + Scenario.value "sumBy" (fun db -> query { for c in db.Customers do sumBy c.Age }) (customers >> Seq.sumBy _.Age) + + Scenario.value + "averageBy" + (fun db -> query { for c in db.Customers do averageBy (float c.Age) }) + (customers >> Seq.averageBy (fun c -> float c.Age)) + + Scenario.value + "minByNullable" + (fun db -> query { for l in db.Lines do minByNullable l.Discount }) + (discounts >> List.min >> Nullable) + + Scenario.value + "maxByNullable" + (fun db -> query { for l in db.Lines do maxByNullable l.Discount }) + (discounts >> List.max >> Nullable) + + Scenario.value + "sumByNullable" + (fun db -> query { for l in db.Lines do sumByNullable l.Discount }) + (discounts >> List.sum >> Nullable) + + Scenario.value + "averageByNullable" + (fun db -> query { for l in db.Lines do averageByNullable (Nullable.float l.Discount) }) + (discounts >> List.averageBy float >> Nullable) ] + +let cases () = sequences () @ values () diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs new file mode 100644 index 00000000000..44d598d3d13 --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module FSharp.Core.UnitTests.Linq.QueryProjectionScenarios + +open System.Linq.Expressions + +let private block = [ Violation.Statement ExpressionType.Block ] + +let private customers (data: Data) = Seq.ofList data.Customers + +let cases () = + [ Scenario.rows "scalar" (fun db -> query { for c in db.Customers do select c.Name }) (customers >> Seq.map _.Name) + + Scenario.rows "entity" (fun db -> query { for c in db.Customers do select c }) customers + + Scenario.rows + "computed scalars" + (fun db -> query { for c in db.Customers do select (c.Name + "!", c.Age + 1, (if c.Age > 28 then "senior" else "junior")) }) + (customers >> Seq.map (fun c -> c.Name + "!", c.Age + 1, (if c.Age > 28 then "senior" else "junior"))) + + Scenario.rows + "tuple of two" + (fun db -> query { for c in db.Customers do select (c.Id, c.Name) }) + (customers >> Seq.map (fun c -> c.Id, c.Name)) + + Scenario.rows + "tuple of seven" + (fun db -> query { for c in db.Customers do select (c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2) }) + (customers >> Seq.map (fun c -> c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2)) + + Scenario.rows + "tuple of eight" + (fun db -> query { for c in db.Customers do select (c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2, c.Id * 7) }) + (customers >> Seq.map (fun c -> c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2, c.Id * 7)) + + Scenario.rows + "tuple of nine" + (fun db -> query { for c in db.Customers do select (c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2, c.Id * 7, c.Age - c.Id) }) + (customers >> Seq.map (fun c -> c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2, c.Id * 7, c.Age - c.Id)) + + Scenario.rows + "nested tuple" + (fun db -> query { for c in db.Customers do select ((c.Id, c.Name), c.Age) }) + (customers >> Seq.map (fun c -> (c.Id, c.Name), c.Age)) + + Scenario.rows + "tuple taken apart by the next for" + (fun db -> + query { + for (id, name) in (query { for c in db.Customers do select (c.Id, c.Name) }) do + where (id > 1) + select name + }) + (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) + |> Scenario.violating [ Violation.ReadsUnnamedMember "Tuple`2" ] + + Scenario.rows + "record, fields in declaration order" + (fun db -> query { for c in db.Customers do select { PersonName = c.Name; PersonAge = c.Age } }) + (customers >> Seq.map (fun c -> { PersonName = c.Name; PersonAge = c.Age })) + + Scenario.rows + "record, fields out of declaration order" + (fun db -> query { for c in db.Customers do select { PersonAge = c.Age; PersonName = c.Name } }) + (customers >> Seq.map (fun c -> { PersonName = c.Name; PersonAge = c.Age })) + |> Scenario.violating block + + Scenario.rows + "record of three, fields rotated" + (fun db -> query { for c in db.Customers do select { Town = c.City; FullName = c.Name; Years = c.Age } }) + (customers >> Seq.map (fun c -> { FullName = c.Name; Years = c.Age; Town = c.City })) + |> Scenario.violating block + + Scenario.rows + "record holding a tuple" + (fun db -> query { for c in db.Customers do select {| Age = c.Age; Pair = (c.Id, c.Name) |} }) + (customers >> Seq.map (fun c -> {| Age = c.Age; Pair = (c.Id, c.Name) |})) + + Scenario.rows + "anonymous record, fields in alphabetical order" + (fun db -> query { for c in db.Customers do select {| Age = c.Age; Name = c.Name |} }) + (customers >> Seq.map (fun c -> {| Age = c.Age; Name = c.Name |})) + + Scenario.rows + "anonymous record, fields out of alphabetical order" + (fun db -> query { for c in db.Customers do select {| Name = c.Name; Age = c.Age |} }) + (customers >> Seq.map (fun c -> {| Age = c.Age; Name = c.Name |})) + |> Scenario.violating block + + Scenario.rows + "anonymous record, method calls in fields out of order" + (fun db -> query { for c in db.Customers do select {| Upper = c.Name.ToUpper(); City = c.City.Trim() |} }) + (customers >> Seq.map (fun c -> {| City = c.City.Trim(); Upper = c.Name.ToUpper() |})) + |> Scenario.violating block + + Scenario.rows + "anonymous record inside an anonymous record" + (fun db -> query { for c in db.Customers do select {| Person = {| Name = c.Name; Age = c.Age |} |} }) + (customers >> Seq.map (fun c -> {| Person = {| Age = c.Age; Name = c.Name |} |})) + |> Scenario.violating block + + Scenario.rows + "struct anonymous record, fields in alphabetical order" + (fun db -> query { for c in db.Customers do select (struct {| Age = c.Age; Name = c.Name |}) }) + (customers >> Seq.map (fun c -> struct {| Age = c.Age; Name = c.Name |})) + + Scenario.rows + "struct anonymous record, fields out of alphabetical order" + (fun db -> query { for c in db.Customers do select (struct {| Name = c.Name; Age = c.Age |}) }) + (customers >> Seq.map (fun c -> struct {| Age = c.Age; Name = c.Name |})) + |> Scenario.violating block + + Scenario.rows + "CLIMutable record, fields in declaration order" + (fun db -> query { for c in db.Customers do select { DtoName = c.Name; DtoAge = c.Age } }) + (customers >> Seq.map (fun c -> { DtoName = c.Name; DtoAge = c.Age })) + + Scenario.rows + "CLIMutable record, fields out of declaration order" + (fun db -> query { for c in db.Customers do select { DtoAge = c.Age; DtoName = c.Name } }) + (customers >> Seq.map (fun c -> { DtoName = c.Name; DtoAge = c.Age })) + |> Scenario.violating block + + Scenario.rows + "class with property initializers" + (fun db -> query { for c in db.Customers do select (CustomerView(ViewName = c.Name, ViewAge = c.Age)) }) + (customers >> Seq.map (fun c -> CustomerView(ViewName = c.Name, ViewAge = c.Age))) + + Scenario.rows "option" (fun db -> query { for c in db.Customers do select (Some c.Age) }) (customers >> Seq.map (fun c -> Some c.Age)) + |> Scenario.violating [ Violation.RuntimeHelperCall "FSharpOption`1.Some" ] ] diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs new file mode 100644 index 00000000000..bad748d75ab --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs @@ -0,0 +1,355 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Core.UnitTests.Linq + +open System +open System.Collections +open System.Collections.Generic +open System.Linq +open System.Linq.Expressions +open System.Reflection +open Microsoft.FSharp.Reflection + +type ICapturedQuery = + abstract InMemory: IQueryable + +// EnumerableQuery turns Queryable calls into Enumerable ones only over its own constants; +// over any other IQueryable it calls Queryable again, which here would never end. +type private InMemoryRewriter() = + inherit ExpressionVisitor() + + override _.VisitConstant node = + match node.Value with + | :? ICapturedQuery as captured -> Expression.Constant captured.InMemory + | _ -> node + +/// An in-memory query that stays on the provider that created it and prints its root by name. +[] +type CapturedQuery<'T>(provider: IQueryProvider, name: string, items: seq<'T>, derived: Expression voption) = + + let enumerate () = + match derived with + | ValueSome expression -> provider.Execute>(expression).GetEnumerator() + | ValueNone -> items.GetEnumerator() + + override _.ToString() = + match derived with + | ValueSome expression -> expression.ToString() + | ValueNone -> name + + interface ICapturedQuery with + member _.InMemory = + match derived with + | ValueSome expression -> EnumerableQuery<'T>(InMemoryRewriter().Visit expression) + | ValueNone -> EnumerableQuery<'T> items + + interface IOrderedQueryable<'T> with + member this.Expression = + match derived with + | ValueSome expression -> expression + | ValueNone -> Expression.Constant this + + member _.ElementType = typeof<'T> + member _.Provider = provider + member _.GetEnumerator() : IEnumerator<'T> = enumerate () + member _.GetEnumerator() : IEnumerator = enumerate () + +/// Executes in memory, and records what it was asked to execute. +[] +type CapturingQueryProvider() = + let executed = ResizeArray() + let mutable depth = 0 + + let inMemory () = + EnumerableQuery Seq.empty :> IQueryProvider + + let track (expression: Expression) (run: Expression -> 'TResult) = + if depth = 0 then + executed.Add expression + + depth <- depth + 1 + + try + run (InMemoryRewriter().Visit expression) + finally + depth <- depth - 1 + + /// The expressions executed directly, not from inside another execution. + member _.Executed = executed :> IReadOnlyList + + member this.Source(name: string, items: seq<'T>) : IQueryable<'T> = + CapturedQuery<'T>(this, name, items, ValueNone) + + member this.Derived<'T>(expression: Expression) : IQueryable<'T> = + CapturedQuery<'T>(this, "", Seq.empty, ValueSome expression) + + interface IQueryProvider with + member this.CreateQuery<'T>(expression: Expression) : IQueryable<'T> = this.Derived<'T> expression + + member this.CreateQuery(expression: Expression) : IQueryable = + let elementType = + expression.Type.GetInterfaces() + |> Array.append [| expression.Type |] + |> Array.find (fun ty -> ty.IsGenericType && ty.GetGenericTypeDefinition() = typedefof>) + |> fun ty -> ty.GetGenericArguments().[0] + + typeof + .GetMethod(nameof Unchecked.defaultof.Derived) + .MakeGenericMethod(elementType) + .Invoke(this, [| expression |]) + :?> IQueryable + + member _.Execute<'TResult>(expression: Expression) : 'TResult = + track expression (inMemory ()).Execute<'TResult> + + member _.Execute(expression: Expression) : obj = + track expression (inMemory ()).Execute + +/// Something in an expression tree that a C# expression lambda never contains, so no LINQ provider translates it. +[] +type Violation = + | Statement of ExpressionType + | LambdaInvocation + | UnboundParameter of name: string + | RuntimeHelperCall of methodName: string + | NestedInCollectionSelector of methodName: string + | ReadsUnnamedMember of typeName: string + | LeftProvider + | Throws of exceptionType: string + +module ProviderShape = + + let private fsharpCore = typeof.Assembly + + let private allMembers = BindingFlags.Public ||| BindingFlags.NonPublic + + let private isLinqOperator (call: MethodCallExpression) = + call.Arguments.Count > 0 + && (call.Method.DeclaringType = typeof + || call.Method.DeclaringType = typeof) + + let private isStatement nodeType = + match nodeType with + | ExpressionType.Block + | ExpressionType.Assign + | ExpressionType.Loop + | ExpressionType.Goto + | ExpressionType.Label + | ExpressionType.Try + | ExpressionType.Switch + | ExpressionType.RuntimeVariables + | ExpressionType.DebugInfo -> true + | _ -> false + + let private flattensOrJoins (call: MethodCallExpression) = + isLinqOperator call + && (match call.Method.Name with + | "SelectMany" + | "Join" + | "GroupJoin" -> true + | _ -> false) + + type private ShapeVisitor() = + inherit ExpressionVisitor() + + let found = ResizeArray() + let bound = HashSet(HashIdentity.Reference) + let mutable collectionSelectorDepth = 0 + + let bind (parameters: seq) = + parameters |> Seq.filter bound.Add |> Seq.toArray + + let unbind (parameters: ParameterExpression array) = + for parameter in parameters do + bound.Remove parameter |> ignore + + [] + let (|AssignsBound|_|) (expression: Expression) = + match expression with + | :? BinaryExpression as assign when assign.NodeType = ExpressionType.Assign -> + match assign.Left with + | :? ParameterExpression as variable when bound.Contains variable -> ValueSome assign.Right + | _ -> ValueNone + | _ -> ValueNone + + member _.Violations = found |> Seq.distinct |> List.ofSeq + + override _.VisitLambda<'T>(node: Expression<'T>) = + let added = bind node.Parameters + + try + base.VisitLambda node + finally + unbind added + + // A block is reported once: its variables and their initial assignments are part of it. + override this.VisitBlock node = + found.Add(Violation.Statement ExpressionType.Block) + let added = bind node.Variables + + try + for expression in node.Expressions do + match expression with + | AssignsBound value -> this.Visit value |> ignore + | expression -> this.Visit expression |> ignore + + node + finally + unbind added + + override _.VisitParameter node = + if not (bound.Contains node) then + found.Add(Violation.UnboundParameter node.Name) + + node + + override _.VisitInvocation node = + found.Add Violation.LambdaInvocation + base.VisitInvocation node + + // A query built earlier and used in this one is part of what the provider translates. + override this.VisitConstant node = + match node.Value with + | :? IQueryable as query -> + match query.Expression with + | :? ConstantExpression as root when obj.ReferenceEquals(root.Value, query) -> () + | built -> this.Visit built |> ignore + | _ -> () + + node + + override _.VisitNew node = + if node.Type.Assembly = fsharpCore && not (node.Type.Name.StartsWith("AnonymousObject", StringComparison.Ordinal)) then + found.Add(Violation.RuntimeHelperCall $"new {node.Type.Name}") + + base.VisitNew node + + override this.VisitMethodCall node = + if node.Method.DeclaringType.Assembly = fsharpCore then + found.Add(Violation.RuntimeHelperCall $"{node.Method.DeclaringType.Name}.{node.Method.Name}") + + if collectionSelectorDepth > 0 && flattensOrJoins node then + found.Add(Violation.NestedInCollectionSelector node.Method.Name) + + if isLinqOperator node && node.Method.Name = "SelectMany" then + this.Visit node.Object |> ignore + this.Visit node.Arguments.[0] |> ignore + collectionSelectorDepth <- collectionSelectorDepth + 1 + + try + this.Visit node.Arguments.[1] |> ignore + finally + collectionSelectorDepth <- collectionSelectorDepth - 1 + + for argument in Seq.skip 2 node.Arguments do + this.Visit argument |> ignore + + node + else + base.VisitMethodCall node + + override this.Visit node = + match node with + | null -> null + | node -> + if node.NodeType <> ExpressionType.Block && isStatement node.NodeType then + found.Add(Violation.Statement node.NodeType) + + base.Visit node + + [] + let private (|Lambda|_|) (argument: Expression) = + match argument with + | :? UnaryExpression as quote when quote.NodeType = ExpressionType.Quote -> ValueSome(quote.Operand :?> LambdaExpression) + | :? LambdaExpression as lambda -> ValueSome lambda + | _ -> ValueNone + + /// A tuple or record built without saying which member each argument becomes. + let private buildsUnnamed (body: Expression) = + match body with + | :? NewExpression as construction when construction.Arguments.Count > 0 -> + match construction.Members with + | null -> + FSharpType.IsTuple construction.Type + || FSharpType.IsRecord(construction.Type, allMembers) + | _ -> false + | _ -> false + + type private MemberReadFinder(owner: ParameterExpression) = + inherit ExpressionVisitor() + + member val Found = false with get, set + + override this.VisitMember node = + if obj.ReferenceEquals(node.Expression, owner) then + this.Found <- true + + base.VisitMember node + + let private readsMemberOf (owner: ParameterExpression) (body: Expression) = + let finder = MemberReadFinder owner + finder.Visit body |> ignore + finder.Found + + let private yields (elementType: Type) (source: Expression) = + typedefof>.MakeGenericType(elementType).IsAssignableFrom source.Type + + /// Members read from elements that an earlier operator built without naming them: + /// the provider has to take apart an object it was not told how to. + let private unnamedMemberReads (expression: Expression) = + let found = ResizeArray() + + let rec producesUnnamed (expression: Expression) = + match expression with + | :? MethodCallExpression as call when isLinqOperator call -> + let source = call.Arguments.[0] + let sourceUnnamed = producesUnnamed source + + let lambdas = + [ for argument in Seq.skip 1 call.Arguments do + match argument with + | Lambda lambda -> lambda + | _ -> () ] + + if sourceUnnamed then + for lambda in lambdas do + let element = lambda.Parameters.[0] + + if yields element.Type source && readsMemberOf element lambda.Body then + found.Add(Violation.ReadsUnnamedMember element.Type.Name) + + match call.Method.Name with + | "Select" + | "SelectMany" + | "Join" + | "GroupJoin" -> lambdas |> List.last |> _.Body |> buildsUnnamed + | "Where" + | "OrderBy" + | "OrderByDescending" + | "ThenBy" + | "ThenByDescending" + | "Skip" + | "SkipWhile" + | "Take" + | "TakeWhile" + | "Distinct" + | "DefaultIfEmpty" -> sourceUnnamed + | _ -> false + | _ -> false + + producesUnnamed expression |> ignore + List.ofSeq found + + let violations (expression: Expression) = + let visitor = ShapeVisitor() + visitor.Visit expression |> ignore + visitor.Violations @ unnamedMemberReads expression |> List.distinct + + /// The LINQ operators along the source chain, outermost last. + let spine (expression: Expression) = + let rec walk (expression: Expression) operators = + match expression with + | :? MethodCallExpression as call when isLinqOperator call -> walk call.Arguments.[0] (call.Method.Name :: operators) + | _ -> operators + + walk expression [] diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs new file mode 100644 index 00000000000..b2a270042e9 --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Core.UnitTests.Linq + +open System +open System.Collections +open System.Globalization +open System.Linq +open System.Linq.Expressions +open System.Reflection +open Microsoft.FSharp.Reflection + +[] +type Outcome = + | Rows of IQueryable + | Value of obj + +/// One query, what it must return, and what is known to be wrong with its expression tree today. +[] +type Scenario = + { Name: string + Run: Db -> Outcome + Expected: Data -> obj list + Ordered: bool + Spine: string list voption + Violations: Violation list } + + override this.ToString() = this.Name + +module Scenario = + + let private allMembers = BindingFlags.Public ||| BindingFlags.NonPublic + + [] + let private (|Grouping|_|) (value: obj) = + match value.GetType().GetInterface(typedefof>.Name) with + | null -> ValueNone + | grouping -> ValueSome(grouping.GetProperty("Key").GetValue value) + + [] + let private (|Tuple|_|) (value: obj) = + if FSharpType.IsTuple(value.GetType()) then + ValueSome(FSharpValue.GetTupleFields value) + else + ValueNone + + [] + let private (|Record|_|) (value: obj) = + if FSharpType.IsRecord(value.GetType(), allMembers) then + let names = FSharpType.GetRecordFields(value.GetType(), allMembers) |> Seq.map _.Name + ValueSome(Seq.zip names (FSharpValue.GetRecordFields(value, allMembers))) + else + ValueNone + + [] + let private (|Union|_|) (value: obj) = + if FSharpType.IsUnion(value.GetType(), allMembers) then + let case, fields = FSharpValue.GetUnionFields(value, value.GetType(), allMembers) + ValueSome struct (case.Name, fields) + else + ValueNone + + /// Text that is equal for equal results, whichever provider materialized them. + let rec render (value: obj) = + let renderAll (values: seq) = + values |> Seq.map render |> String.concat "; " + + match value with + | null -> "null" + | :? string as text -> $"\"{text}\"" + | :? IFormattable as number -> number.ToString(null, CultureInfo.InvariantCulture) + | Grouping key -> $"{render key} => [{renderAll (Seq.cast (value :?> IEnumerable))}]" + | :? IEnumerable as items -> $"[{renderAll (Seq.cast items)}]" + | Tuple elements -> $"({renderAll elements})" + | Record fields -> + let rendered = fields |> Seq.map (fun (name, field) -> $"{name} = {render field}") + $"""{{ {String.concat "; " rendered} }}""" + | Union(case, fields) -> $"{case}({renderAll fields})" + | value -> string value + + let rows name (run: Db -> IQueryable<'T>) (expected: Data -> seq<'T>) = + { Name = name + Run = fun db -> Outcome.Rows(run db) + Expected = fun data -> [ for row in expected data -> box row ] + Ordered = false + Spine = ValueNone + Violations = [] } + + let value name (run: Db -> 'T) (expected: Data -> 'T) = + { Name = name + Run = fun db -> Outcome.Value(box (run db)) + Expected = fun data -> [ box (expected data) ] + Ordered = true + Spine = ValueNone + Violations = [] } + + let ordered scenario = { scenario with Ordered = true } + + /// The operators the source chain must consist of, outermost last. + let flat spine scenario = + { scenario with + Spine = ValueSome spine } + + /// What is wrong with the expression tree today; the scenario fails once that stops being true. + let violating violations scenario = + { scenario with + Violations = violations } + + [] + type private Observed = + { Expression: Expression + Violations: Violation list + Rows: obj list } + + let rec private rootCause (error: exn) = + match error, error.InnerException with + | :? TargetInvocationException, null -> error + | :? TargetInvocationException, inner -> rootCause inner + | _ -> error + + let private observe (scenario: Scenario) = + let provider = CapturingQueryProvider() + let data = Data() + + let observed = + try + match scenario.Run(Db.inMemory provider data) with + | Outcome.Rows rows -> + let stayed = obj.ReferenceEquals(rows.Provider, provider) + + { Expression = rows.Expression + Violations = (if stayed then [] else [ Violation.LeftProvider ]) + Rows = [ for row in rows -> row ] } + // The value comes from the last execution; whatever ran before it went to the provider too. + | Outcome.Value value when provider.Executed.Count > 0 -> + let earlier = provider.Executed |> Seq.take (provider.Executed.Count - 1) + + { Expression = provider.Executed.[provider.Executed.Count - 1] + Violations = [ for expression in earlier do yield! ProviderShape.violations expression ] + Rows = [ value ] } + | Outcome.Value value -> + { Expression = Expression.Empty() + Violations = [ Violation.LeftProvider ] + Rows = [ value ] } + with error -> + { Expression = Expression.Empty() + Violations = [ Violation.Throws((rootCause error).GetType().Name) ] + Rows = [] } + + data, observed + + let private inOrder ordered (rows: obj list) = + let rendered = List.map render rows + if ordered then rendered else List.sort rendered + + /// Everything the scenario gets wrong, empty when it behaves as described. + let failures (scenario: Scenario) = + let data, observed = observe scenario + + let violations = + observed.Violations @ ProviderShape.violations observed.Expression + |> List.distinct + + let threw = + violations + |> List.exists (function + | Violation.Throws _ -> true + | _ -> false) + + [ if set violations <> set scenario.Violations then + $"violations: expected %A{scenario.Violations}, got %A{violations}" + + if not threw then + let expected = inOrder scenario.Ordered (scenario.Expected data) + let actual = inOrder scenario.Ordered observed.Rows + + if expected <> actual then + $"rows: expected %A{expected}, got %A{actual}" + + match scenario.Spine with + | ValueSome spine when spine <> ProviderShape.spine observed.Expression -> + $"operators: expected %A{spine}, got %A{ProviderShape.spine observed.Expression}" + | _ -> () ] + |> List.map (fun failure -> $"{scenario.Name}: {failure}\n in {observed.Expression}") + + let check scenario = + match failures scenario with + | [] -> () + | failures -> failwith (String.concat "\n" failures) diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenarioDomain.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenarioDomain.fs new file mode 100644 index 00000000000..ada7f2ba70f --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenarioDomain.fs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Core.UnitTests.Linq + +open System +open System.Collections.Generic +open System.Linq + +[] +type Product() = + member val Id = 0 with get, set + member val Name = "" with get, set + member val Category = "" with get, set + override this.ToString() = $"Product#{this.Id}" + +[] +type OrderLine() = + member val Id = 0 with get, set + member val OrderId = 0 with get, set + member val ProductId = 0 with get, set + member val Quantity = 0 with get, set + member val Price = 0.0 with get, set + member val Discount = Nullable() with get, set + override this.ToString() = $"OrderLine#{this.Id}" + +[] +type Order() = + member val Id = 0 with get, set + member val CustomerId = 0 with get, set + member val Total = 0.0 with get, set + member val Lines: ICollection = List() with get, set + override this.ToString() = $"Order#{this.Id}" + +[] +type Customer() = + member val Id = 0 with get, set + member val Name = "" with get, set + member val City = "" with get, set + member val Age = 0 with get, set + member val Registered = Nullable() with get, set + member val Orders: ICollection = List() with get, set + override this.ToString() = $"Customer#{this.Id}" + +type CustomerView() = + member val ViewName = "" with get, set + member val ViewAge = 0 with get, set + override this.ToString() = $"CustomerView({this.ViewName}, {this.ViewAge})" + +type NameAndAge = { PersonName: string; PersonAge: int } + +type NameAgeCity = + { FullName: string + Years: int + Town: string } + +[] +type NameAndAgeDto = { DtoName: string; DtoAge: int } + +type CityAndAge = { KeyCity: string; KeyAge: int } + +type OrderAndQuantity = { OrderNumber: int; Units: int } + +type Data() = + let line id orderId productId quantity price discount = + OrderLine(Id = id, OrderId = orderId, ProductId = productId, Quantity = quantity, Price = price, Discount = discount) + + let order id customerId total (lines: OrderLine list) = + Order(Id = id, CustomerId = customerId, Total = total, Lines = List lines) + + let customer id name city age registered (orders: Order list) = + Customer(Id = id, Name = name, City = city, Age = age, Registered = registered, Orders = List orders) + + let orders = + [ order 10 1 12.5 [ line 100 10 1 2 2.5 (Nullable 1); line 101 10 2 1 7.5 (Nullable()) ] + order 11 1 3.0 [ line 102 11 3 1 3.0 (Nullable 2) ] + order 12 2 5.0 [ line 103 12 1 2 2.5 (Nullable()) ] + order 13 3 0.0 [] ] + + let ordersOf customerId = + orders |> List.filter (fun order -> order.CustomerId = customerId) + + member val Products = + [ Product(Id = 1, Name = "Pen", Category = "Office") + Product(Id = 2, Name = "Ink", Category = "Office") + Product(Id = 3, Name = "Tea", Category = "Food") ] + + member val Customers = + [ customer 1 "Alice" "London" 30 (Nullable(DateTime(2020, 1, 1))) (ordersOf 1) + customer 2 "Bob" "Paris" 25 (Nullable()) (ordersOf 2) + customer 3 "Carol" "London" 35 (Nullable(DateTime(2021, 6, 15))) (ordersOf 3) + customer 4 "Dave" "Berlin" 28 (Nullable()) [] ] + + member _.Orders = orders + member _.Lines = orders |> List.collect (fun order -> List.ofSeq order.Lines) + +[] +type Db = + { Customers: IQueryable + Orders: IQueryable + Lines: IQueryable + Products: IQueryable } + +module Db = + + let inMemory (provider: CapturingQueryProvider) (data: Data) = + { Customers = provider.Source("Customers", data.Customers) + Orders = provider.Source("Orders", data.Orders) + Lines = provider.Source("Lines", data.Lines) + Products = provider.Source("Products", data.Products) } diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs new file mode 100644 index 00000000000..6e284ed7d4d --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Core.UnitTests.Linq + +open Xunit + +type QueryShapeTests() = + + static let rows (cases: Scenario list) = + cases |> Seq.map (fun scenario -> [| box scenario |]) + + static member Projections = rows (QueryProjectionScenarios.cases ()) + static member Operators = rows (QueryOperatorScenarios.cases ()) + static member Keys = rows (QueryKeyScenarios.cases ()) + static member Structures = rows (QueryStructureScenarios.cases ()) + static member MethodSyntax = rows (QueryMethodSyntaxScenarios.cases ()) + + [] + member _.Projection(scenario: Scenario) = Scenario.check scenario + + [] + member _.Operator(scenario: Scenario) = Scenario.check scenario + + [] + member _.Key(scenario: Scenario) = Scenario.check scenario + + [] + member _.Structure(scenario: Scenario) = Scenario.check scenario + + [] + member _.``Method syntax``(scenario: Scenario) = Scenario.check scenario diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs new file mode 100644 index 00000000000..6f806f4bb43 --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs @@ -0,0 +1,268 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module FSharp.Core.UnitTests.Linq.QueryStructureScenarios + +open System.Linq + +let private quotedLambda = + [ Violation.RuntimeHelperCall "LeafExpressionConverter.QuotationToLambdaExpression" + Violation.RuntimeHelperCall "LeafExpressionConverter.SubstHelper" ] + +let private nested operators = + operators |> List.map Violation.NestedInCollectionSelector + +let private orDefault (items: seq<'T>) = + if Seq.isEmpty items then Seq.singleton Unchecked.defaultof<'T> else items + +let private idOrNone (order: Order) = + match order with + | null -> -1 + | order -> order.Id + +let private nameOrNone (product: Product) = + match product with + | null -> "" + | product -> product.Name + +let private ordersWithLines (data: Data) = + seq { for c in data.Customers do for o in c.Orders do for l in o.Lines -> c.Name, o.Id, l.Id } + +let private customersWithOrders (data: Data) = + seq { for c in data.Customers do for o in c.Orders -> c.Name, o.Id } + +let private leftJoinedOrders (data: Data) = + seq { for c in data.Customers do for o in orDefault c.Orders -> c.Name, idOrNone o } + +let private leftJoinedOrdersAndProducts (data: Data) = + seq { + for c in data.Customers do + for o in orDefault c.Orders do + for p in orDefault (data.Products |> List.filter (fun p -> p.Id = c.Id)) -> c.Name, idOrNone o, nameOrNone p + } + +let private loops () = + [ Scenario.rows + "two for" + (fun db -> query { for c in db.Customers do for p in db.Products do select (c.Name, p.Name) }) + (fun data -> seq { for c in data.Customers do for p in data.Products -> c.Name, p.Name }) + |> Scenario.flat [ "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "two for, where over both" + (fun db -> query { for c in db.Customers do for o in db.Orders do where (o.CustomerId = c.Id); select (c.Name, o.Id) }) + customersWithOrders + |> Scenario.flat [ "SelectMany"; "Where"; "Select"; "Select" ] + + Scenario.rows + "for, where, for, where" + (fun db -> + query { + for c in db.Customers do + where (c.Age > 26) + for o in db.Orders do + where (o.CustomerId = c.Id) + select (c.Name, o.Id) + }) + (fun data -> seq { for c in data.Customers do if c.Age > 26 then for o in c.Orders -> c.Name, o.Id }) + |> Scenario.flat [ "Where"; "SelectMany"; "Where"; "Select"; "Select" ] + + Scenario.rows + "for over a navigation collection" + (fun db -> query { for c in db.Customers do for o in c.Orders do select (c.Name, o.Id) }) + customersWithOrders + |> Scenario.flat [ "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "for over two levels of navigation collections" + (fun db -> query { for c in db.Customers do for o in c.Orders do for l in o.Lines do select (c.Name, o.Id, l.Id) }) + ordersWithLines + |> Scenario.violating (nested [ "SelectMany" ]) + + Scenario.rows + "three for" + (fun db -> + query { + for c in db.Customers do + for o in db.Orders do + for l in db.Lines do + where (o.CustomerId = c.Id && l.OrderId = o.Id) + select (c.Name, o.Id, l.Id) + }) + ordersWithLines + |> Scenario.violating (nested [ "SelectMany" ]) + + Scenario.rows + "for after sortBy" + (fun db -> query { for c in db.Customers do sortBy c.Age; for p in db.Products do select (c.Name, p.Name) }) + (fun data -> seq { for c in data.Customers |> List.sortBy _.Age do for p in data.Products -> c.Name, p.Name }) + |> Scenario.ordered + |> Scenario.flat [ "OrderBy"; "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "for after join" + (fun db -> + query { + for c in db.Customers do + join o in db.Orders on (c.Id = o.CustomerId) + for l in db.Lines do + where (l.OrderId = o.Id) + select (c.Name, o.Id, l.Id) + }) + ordersWithLines + |> Scenario.flat [ "Join"; "SelectMany"; "Where"; "Select"; "Select" ] + + Scenario.rows + "let between for" + (fun db -> + query { for c in db.Customers do let next = c.Id + 9 in for o in db.Orders do where (o.Id = next); select (c.Name, o.Id) }) + (fun data -> seq { for c in data.Customers do for o in data.Orders do if o.Id = c.Id + 9 then yield c.Name, o.Id }) + |> Scenario.flat [ "SelectMany"; "Where"; "Select"; "Select" ] + + Scenario.rows + "if then around for" + (fun db -> query { for c in db.Customers do if c.Age > 26 then for o in c.Orders do yield (c.Name, o.Id) }) + (fun data -> seq { for c in data.Customers do if c.Age > 26 then for o in c.Orders -> c.Name, o.Id }) + |> Scenario.flat [ "Where"; "SelectMany"; "Select" ] + + Scenario.rows + "for around if then" + (fun db -> query { for c in db.Customers do for o in db.Orders do if o.CustomerId = c.Id then yield (c.Name, o.Id) }) + customersWithOrders + |> Scenario.flat [ "SelectMany"; "Select" ] + + Scenario.rows + "yield! of a navigation collection" + (fun db -> query { for c in db.Customers do yield! c.Orders }) + (fun data -> Seq.ofList data.Orders) + |> Scenario.flat [ "SelectMany" ] + + Scenario.rows + "distinct and take after two for" + (fun db -> query { for c in db.Customers do for o in c.Orders do select c.City; distinct; take 1 }) + (fun _ -> seq { "London" }) + |> Scenario.ordered + |> Scenario.flat [ "SelectMany"; "Select"; "Distinct"; "Take" ] + + Scenario.rows + "groupBy after two for" + (fun db -> query { for c in db.Customers do for o in c.Orders do groupBy c.City into g; select (g.Key, g.Count()) }) + (fun data -> seq { for c in data.Customers do for _ in c.Orders -> c.City } |> Seq.countBy id) + |> Scenario.violating [ Violation.RuntimeHelperCall "new Grouping`2" ] + + Scenario.value "count over two for" (fun db -> query { for c in db.Customers do for o in c.Orders do count }) (fun _ -> 4) + + Scenario.value + "exists over two for" + (fun db -> query { for c in db.Customers do for o in c.Orders do exists (o.Total > 10.0 && c.City = "London") }) + (fun _ -> true) + |> Scenario.violating [ Violation.ReadsUnnamedMember "Tuple`2" ] ] + +let private subqueries () = + [ Scenario.rows + "nested query as the source" + (fun db -> query { for o in (query { for o in db.Orders do where (o.Total > 4.0); select o }) do select o.Id }) + (fun data -> seq { for o in data.Orders do if o.Total > 4.0 then yield o.Id }) + |> Scenario.flat [ "Where"; "Select" ] + + Scenario.rows + "nested query as the source of the second for" + (fun db -> + query { + for c in db.Customers do + for o in (query { for o in db.Orders do where (o.CustomerId = c.Id); select o }) do + select (c.Name, o.Id) + }) + customersWithOrders + |> Scenario.flat [ "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "nested query in select" + (fun db -> + query { for c in db.Customers do select (c.Name, query { for o in db.Orders do where (o.CustomerId = c.Id); select o.Id }) }) + (fun data -> seq { for c in data.Customers -> c.Name, (seq { for o in c.Orders -> o.Id }).AsQueryable() }) + + Scenario.rows + "second for over a source filtered by a lambda" + (fun db -> query { for c in db.Customers do for o in db.Orders.Where(fun o -> o.CustomerId = c.Id) do select (c.Name, o.Id) }) + customersWithOrders + |> Scenario.violating quotedLambda + + Scenario.rows + "where with a lambda over another source" + (fun db -> query { for c in db.Customers do where (db.Orders.Any(fun o -> o.CustomerId = c.Id)); select c.Name }) + (fun data -> seq { for c in data.Customers do if c.Orders.Count > 0 then yield c.Name }) + |> Scenario.violating quotedLambda ] + +let private leftJoins () = + [ Scenario.rows + "left join spelled leftOuterJoin" + (fun db -> + query { + for c in db.Customers do + leftOuterJoin o in db.Orders on (c.Id = o.CustomerId) into orders + for o in orders do + select (c.Name, (if obj.ReferenceEquals(o, null) then -1 else o.Id)) + }) + leftJoinedOrders + |> Scenario.flat [ "GroupJoin"; "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "left join spelled groupJoin and DefaultIfEmpty" + (fun db -> + query { + for c in db.Customers do + groupJoin o in db.Orders on (c.Id = o.CustomerId) into orders + for o in orders.DefaultIfEmpty() do + select (c.Name, (if obj.ReferenceEquals(o, null) then -1 else o.Id)) + }) + leftJoinedOrders + |> Scenario.flat [ "GroupJoin"; "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "two left joins spelled leftOuterJoin" + (fun db -> + query { + for c in db.Customers do + leftOuterJoin o in db.Orders on (c.Id = o.CustomerId) into orders + for o in orders do + leftOuterJoin p in db.Products on (c.Id = p.Id) into products + for p in products do + select + (c.Name, + (if obj.ReferenceEquals(o, null) then -1 else o.Id), + (if obj.ReferenceEquals(p, null) then "" else p.Name)) + }) + leftJoinedOrdersAndProducts + |> Scenario.violating (nested [ "GroupJoin"; "SelectMany" ]) + + Scenario.rows + "two left joins spelled groupJoin and DefaultIfEmpty" + (fun db -> + query { + for c in db.Customers do + groupJoin o in db.Orders on (c.Id = o.CustomerId) into orders + for o in orders.DefaultIfEmpty() do + groupJoin p in db.Products on (c.Id = p.Id) into products + for p in products.DefaultIfEmpty() do + select + (c.Name, + (if obj.ReferenceEquals(o, null) then -1 else o.Id), + (if obj.ReferenceEquals(p, null) then "" else p.Name)) + }) + leftJoinedOrdersAndProducts + |> Scenario.violating (nested [ "GroupJoin"; "SelectMany" ]) + + Scenario.rows + "join after a left join" + (fun db -> + query { + for c in db.Customers do + leftOuterJoin o in db.Orders on (c.Id = o.CustomerId) into orders + for o in orders do + join p in db.Products on (c.Id = p.Id) + select (c.Name, (if obj.ReferenceEquals(o, null) then -1 else o.Id), p.Name) + }) + (leftJoinedOrdersAndProducts >> Seq.filter (fun (_, _, product) -> product <> "")) + |> Scenario.violating (nested [ "Join" ]) ] + +let cases () = loops () @ subqueries () @ leftJoins () From 8bd4cd28dd4c73ce7c97ea77f2248b2ceae615df Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sun, 27 Sep 2026 04:29:30 +0200 Subject: [PATCH 02/19] Translate struct tuples in queries Selecting a struct tuple failed with "unreachable": the tuple was replaced by an anonymous object, but only reference tuples had an anonymous object type to be replaced by, and the way back always built a reference tuple. Both kinds now share the anonymous object of their arity, and the way back builds the kind the query asked for. Taking a struct tuple apart in a lambda failed with a tuple index out of range, because a struct tuple has fields where a reference tuple has properties. Fixes #15522 Co-Authored-By: Claude Fable 5.1 --- docs/release-notes/.FSharp.Core/11.0.200.md | 1 + src/FSharp.Core/Linq.fs | 9 ++++- src/FSharp.Core/Query.fs | 5 +-- src/FSharp.Core/QueryExtensions.fs | 35 ++++++------------- .../QueryMethodSyntaxScenarios.fs | 16 +++++++++ .../QueryProjectionScenarios.fs | 32 +++++++++++++++++ 6 files changed, 71 insertions(+), 27 deletions(-) diff --git a/docs/release-notes/.FSharp.Core/11.0.200.md b/docs/release-notes/.FSharp.Core/11.0.200.md index 49e81a0efdf..fd72f4da417 100644 --- a/docs/release-notes/.FSharp.Core/11.0.200.md +++ b/docs/release-notes/.FSharp.Core/11.0.200.md @@ -6,5 +6,6 @@ * Correct `FSharpValue.PreComputeTuplePropertyInfo` to report that struct tuples store elements in fields, not properties, and recommend supported tuple readers instead of reporting a valid index as out of range. ([Issue #20801](https://github.com/dotnet/fsharp/issues/20801), [PR #20815](https://github.com/dotnet/fsharp/pull/20815)) * Quotation debug text from `Expr.ToString()`, `Expr.ToString(full)`, and `%A` now prints struct tuple construction as `NewStructTuple`; reference tuples remain `NewTuple`. ([Issue #20800](https://github.com/dotnet/fsharp/issues/20800), [PR #20814](https://github.com/dotnet/fsharp/pull/20814)) +* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522)) ### Changed diff --git a/src/FSharp.Core/Linq.fs b/src/FSharp.Core/Linq.fs index b8025b39bb4..431f6fd9514 100644 --- a/src/FSharp.Core/Linq.fs +++ b/src/FSharp.Core/Linq.fs @@ -475,7 +475,14 @@ module LeafExpressionConverter = | TupleGet(arg, n) -> let argP = ConvExprToLinqInContext env arg - let rec build ty argP n = + let rec build (ty: Type) argP n = + // A struct tuple holds its elements in fields, which PreComputeTuplePropertyInfo cannot return + if ty.IsValueType then + if n < 7 then + Expression.Field(argP, ty.GetField("Item" + string (n + 1))) |> asExpr + else + build (ty.GetGenericArguments().[7]) (Expression.Field(argP, ty.GetField "Rest") |> asExpr) (n - 7) + else match FSharpValue.PreComputeTuplePropertyInfo(ty, n) with | propInfo, None -> Expression.Property(argP, propInfo) |> asExpr diff --git a/src/FSharp.Core/Query.fs b/src/FSharp.Core/Query.fs index 2f84efd0eee..56171c629f9 100644 --- a/src/FSharp.Core/Query.fs +++ b/src/FSharp.Core/Query.fs @@ -1156,8 +1156,9 @@ module Query = /// immutable-to-mutable conversion information, convert it back to an expression involving immutable tuples or records. let rec ConvMutableToImmutable conv mutExpr = match conv with - | TupleConv convs -> - Expr.NewTuple (convs |> List.mapi (fun i conv -> ConvMutableToImmutable conv (AnonymousObjectGet (mutExpr, i)))) + | TupleConv (isStruct, convs) -> + let elements = convs |> List.mapi (fun i conv -> ConvMutableToImmutable conv (AnonymousObjectGet (mutExpr, i))) + if isStruct then Expr.NewStructTuple elements else Expr.NewTuple elements | RecordConv (typ, convs) -> Expr.NewRecord (typ, convs |> List.mapi (fun i conv -> ConvMutableToImmutable conv (AnonymousObjectGet (mutExpr, i)))) diff --git a/src/FSharp.Core/QueryExtensions.fs b/src/FSharp.Core/QueryExtensions.fs index eafdae2d187..18edd4b9b5b 100644 --- a/src/FSharp.Core/QueryExtensions.fs +++ b/src/FSharp.Core/QueryExtensions.fs @@ -108,14 +108,6 @@ module internal Adapters = let anonObjectTypes = tupleTypes |> Array.map snd - let tupleToAnonTypeMap = - let t = Dictionary() - - for k, v in tupleTypes do - t.[k] <- v - - t - let anonToTupleTypeMap = let t = Dictionary() @@ -173,18 +165,13 @@ module internal Adapters = walk i e e.Type + /// Reference and struct tuples of the same arity share one anonymous object type. let RewriteTupleType (ty: Type) conv = - // Tuples are generic, so lookup only for generic types - assert ty.IsGenericType - let generic = ty.GetGenericTypeDefinition() - - match tupleToAnonTypeMap.TryGetValue generic with - | true, mutableTupleType -> - // Recursively transform type arguments - mutableTupleType.MakeGenericType(ty.GetGenericArguments() |> Array.toList |> conv |> Array.ofList) - | _ -> - assert false - failwith "unreachable" + assert (FSharpType.IsTuple ty) + let elementTypes = ty.GetGenericArguments() + let mutableTupleType = anonObjectTypes.[elementTypes.Length - 1] + // Recursively transform type arguments + mutableTupleType.MakeGenericType(elementTypes |> Array.toList |> conv |> Array.ofList) let (|RecordFieldGetSimplification|_|) (expr: Expr) = match expr with @@ -206,7 +193,7 @@ module internal Adapters = /// that we can invert the process in final query results. [] type ConversionDescription = - | TupleConv of ConversionDescription list + | TupleConv of isStruct: bool * ConversionDescription list | RecordConv of Type * ConversionDescription list | GroupingConv (* origKeyType: *) of Type (* origElemType: *) * Type * ConversionDescription | SeqConv of ConversionDescription @@ -216,18 +203,18 @@ module internal Adapters = /// "yield" or "select", convert it to a type involving anonymous objects according to the conversion data. let rec ConvImmutableTypeToMutableType conv ty = match conv with - | TupleConv convs -> + | TupleConv(isStruct, convs) -> assert (FSharpType.IsTuple ty) match convs with | x1 :: x2 :: x3 :: x4 :: x5 :: x6 :: x7 :: x8 :: tail -> - let els = [ x1; x2; x3; x4; x5; x6; x7; TupleConv(x8 :: tail) ] + let els = [ x1; x2; x3; x4; x5; x6; x7; TupleConv(isStruct, x8 :: tail) ] RewriteTupleType ty (List.map2 ConvImmutableTypeToMutableType els) | _ -> RewriteTupleType ty (List.map2 ConvImmutableTypeToMutableType convs) | RecordConv(_, convs) -> assert (isPartiallyImmutableRecord ty) let types = [| for f in FSharpType.GetRecordFields ty -> f.PropertyType |] - ConvImmutableTypeToMutableType (TupleConv convs) (FSharpType.MakeTupleType types) + ConvImmutableTypeToMutableType (TupleConv(false, convs)) (FSharpType.MakeTupleType types) | GroupingConv(_keyTy, _elemTy, conv) -> assert ty.IsGenericType assert (ty.GetGenericTypeDefinition() = typedefof>) @@ -324,7 +311,7 @@ module internal Adapters = let argExprsNow, argScripts = exprs |> List.map (ProduceMoreMutables tipf) |> List.unzip - NewAnonymousObject argExprsNow, TupleConv argScripts + NewAnonymousObject argExprsNow, TupleConv(expr.Type.IsValueType, argScripts) // Replace immutable records by anonymous objects | Patterns.NewRecord(typ, args) when isPartiallyImmutableRecord typ -> diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs index b8a704a846e..5325268456b 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs @@ -69,6 +69,12 @@ let cases () = (customers >> Seq.map (fun c -> c.Id, c.Name) >> Seq.filter (fun (id, _) -> id > 1)) |> Scenario.violating (reading "Tuple`2" @ block) + Scenario.rows + "Where taking a struct tuple apart" + (fun db -> db.Customers.Select(fun c -> struct (c.Id, c.Name)).Where(fun (struct (id, _)) -> id > 1)) + (customers >> Seq.map (fun c -> struct (c.Id, c.Name)) >> Seq.filter (fun (struct (id, _)) -> id > 1)) + |> Scenario.violating (reading "ValueTuple`2" @ block) + Scenario.rows "Where taking the last of nine apart" (fun db -> @@ -79,6 +85,16 @@ let cases () = (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) |> Scenario.violating (reading "Tuple`8" @ block) + Scenario.rows + "Where taking the last of a struct nine apart" + (fun db -> + db.Customers + .Select(fun c -> struct (c.Id + 100, c.Name, c.City, c.Age, c.Id * 7, c.Name.Length, c.City.Length, c.Age * 2, c.Id)) + .Where(fun (struct (_, _, _, _, _, _, _, _, last)) -> last > 1) + .Select(fun (struct (_, name, _, _, _, _, _, _, _)) -> name)) + (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) + |> Scenario.violating (reading "ValueTuple`8" @ block) + Scenario.rows "Where over a query that selects a tuple" (fun db -> (query { for c in db.Customers do select (c.Id, c.Name) }).Where(fun (id, _) -> id > 1)) diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs index 44d598d3d13..8c1e684614e 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs @@ -43,6 +43,27 @@ let cases () = (fun db -> query { for c in db.Customers do select ((c.Id, c.Name), c.Age) }) (customers >> Seq.map (fun c -> (c.Id, c.Name), c.Age)) + Scenario.rows + "struct tuple of two" + (fun db -> query { for c in db.Customers do select (struct (c.Id, c.Name)) }) + (customers >> Seq.map (fun c -> struct (c.Id, c.Name))) + + Scenario.rows + "struct tuple of nine" + (fun db -> + query { for c in db.Customers do select (struct (c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2, c.Id * 7, c.Age - c.Id)) }) + (customers >> Seq.map (fun c -> struct (c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2, c.Id * 7, c.Age - c.Id))) + + Scenario.rows + "struct tuple inside a tuple" + (fun db -> query { for c in db.Customers do select (struct (c.Id, c.Name), c.Age) }) + (customers >> Seq.map (fun c -> struct (c.Id, c.Name), c.Age)) + + Scenario.rows + "tuple inside a struct tuple" + (fun db -> query { for c in db.Customers do select (struct ((c.Id, c.Name), c.Age)) }) + (customers >> Seq.map (fun c -> struct ((c.Id, c.Name), c.Age))) + Scenario.rows "tuple taken apart by the next for" (fun db -> @@ -54,6 +75,17 @@ let cases () = (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) |> Scenario.violating [ Violation.ReadsUnnamedMember "Tuple`2" ] + Scenario.rows + "struct tuple taken apart by the next for" + (fun db -> + query { + for struct (id, name) in (query { for c in db.Customers do select (struct (c.Id, c.Name)) }) do + where (id > 1) + select name + }) + (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) + |> Scenario.violating [ Violation.ReadsUnnamedMember "ValueTuple`2" ] + Scenario.rows "record, fields in declaration order" (fun db -> query { for c in db.Customers do select { PersonName = c.Name; PersonAge = c.Age } }) From c3ec9ebb8d5301eb5130e53ed435ada4de190adc Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sun, 1 Nov 2026 22:12:20 +0100 Subject: [PATCH 03/19] Link the release note to the pull request Co-Authored-By: Claude Sonnet 5.5 --- docs/release-notes/.FSharp.Core/11.0.200.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes/.FSharp.Core/11.0.200.md b/docs/release-notes/.FSharp.Core/11.0.200.md index fd72f4da417..0aa52c42d5a 100644 --- a/docs/release-notes/.FSharp.Core/11.0.200.md +++ b/docs/release-notes/.FSharp.Core/11.0.200.md @@ -6,6 +6,6 @@ * Correct `FSharpValue.PreComputeTuplePropertyInfo` to report that struct tuples store elements in fields, not properties, and recommend supported tuple readers instead of reporting a valid index as out of range. ([Issue #20801](https://github.com/dotnet/fsharp/issues/20801), [PR #20815](https://github.com/dotnet/fsharp/pull/20815)) * Quotation debug text from `Expr.ToString()`, `Expr.ToString(full)`, and `%A` now prints struct tuple construction as `NewStructTuple`; reference tuples remain `NewTuple`. ([Issue #20800](https://github.com/dotnet/fsharp/issues/20800), [PR #20814](https://github.com/dotnet/fsharp/pull/20814)) -* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522)) +* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) ### Changed From 526db2687cc2969dad617756d27d52727c487530 Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sun, 1 Nov 2026 22:55:55 +0100 Subject: [PATCH 04/19] Follow every query chain when checking for unnamed member reads The shape tests followed only the first source of each operator. Each lambda parameter is now tied to the sequence it stands for, so the check also covers the inner input of a join, the second parameter of a join result selector, a collection selector of SelectMany, an identity Select, a chain inside the lambda of another operator and a query captured from earlier. A value scenario now has to end in a Queryable operator that gives the value: a count made over the rows of the provider leaves it. The validator itself is tested on trees built by hand. Co-Authored-By: Claude Sonnet 5.5 --- .../QueryProviderShape.fs | 171 +++++++++++-- .../Microsoft.FSharp.Linq/QueryScenario.fs | 12 +- .../Microsoft.FSharp.Linq/QueryShapeTests.fs | 231 ++++++++++++++++++ 3 files changed, 389 insertions(+), 25 deletions(-) diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs index bad748d75ab..ae25aa7b1d6 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs @@ -294,35 +294,70 @@ module ProviderShape = let private yields (elementType: Type) (source: Expression) = typedefof>.MakeGenericType(elementType).IsAssignableFrom source.Type - /// Members read from elements that an earlier operator built without naming them: - /// the provider has to take apart an object it was not told how to. - let private unnamedMemberReads (expression: Expression) = + /// Whether the lambda gives its first parameter back as it is. + let private returnsParameter (lambda: LambdaExpression) = + lambda.Parameters.Count > 0 && obj.ReferenceEquals(lambda.Body, lambda.Parameters.[0]) + + /// A query that is its own root, as opposed to one built from another by operators. + let private isRoot (query: IQueryable) = + match query.Expression with + | :? ConstantExpression as root -> obj.ReferenceEquals(root.Value, query) + | _ -> false + + /// Finds members read from elements that an earlier operator built without naming them: the provider has to + /// take apart an object it was not told how to. Every chain of operators is followed, the ones inside lambdas + /// and the ones of a query captured from earlier too, and each parameter of a lambda is tied to the sequence + /// it stands for, which is not always the first source of the operator. + type private UnnamedMemberFinder() = + inherit ExpressionVisitor() + let found = ResizeArray() - let rec producesUnnamed (expression: Expression) = - match expression with + // Whether the elements are unnamed, for the parameters that stand for a sequence: the group of a GroupJoin + let sequences = Dictionary(HashIdentity.Reference) + + let lambdaAt (call: MethodCallExpression) index = + if index < call.Arguments.Count then + match call.Arguments.[index] with + | Lambda lambda -> ValueSome lambda + | _ -> ValueNone + else + ValueNone + + // The parameter of the lambda stands for an element of the sequence, which was built without names + let report (lambda: LambdaExpression) index (sequence: Expression) (unnamed: bool) = + if unnamed && index < lambda.Parameters.Count then + let parameter = lambda.Parameters.[index] + + if yields parameter.Type sequence && readsMemberOf parameter lambda.Body then + found.Add(Violation.ReadsUnnamedMember parameter.Type.Name) + + member _.Found = found |> Seq.distinct |> List.ofSeq + + /// Whether the elements of the sequence were built without naming their members. + member this.Unnamed(sequence: Expression) : bool = + match sequence with | :? MethodCallExpression as call when isLinqOperator call -> let source = call.Arguments.[0] - let sourceUnnamed = producesUnnamed source let lambdas = - [ for argument in Seq.skip 1 call.Arguments do - match argument with - | Lambda lambda -> lambda - | _ -> () ] - - if sourceUnnamed then - for lambda in lambdas do - let element = lambda.Parameters.[0] - - if yields element.Type source && readsMemberOf element lambda.Body then - found.Add(Violation.ReadsUnnamedMember element.Type.Name) + [ for index in 1 .. call.Arguments.Count - 1 do + match lambdaAt call index with + | ValueSome lambda -> lambda + | ValueNone -> () ] match call.Method.Name with - | "Select" - | "SelectMany" + | "Select" -> + lambdas + |> List.tryLast + |> Option.exists (fun selector -> buildsUnnamed selector.Body || (returnsParameter selector && this.Unnamed source)) + | "SelectMany" -> + match lambdas with + | [ collection ] -> this.Unnamed collection.Body + | [ _; result ] -> buildsUnnamed result.Body + | _ -> false | "Join" - | "GroupJoin" -> lambdas |> List.last |> _.Body |> buildsUnnamed + | "GroupJoin" -> lambdas |> List.tryLast |> Option.exists (fun result -> buildsUnnamed result.Body) | "Where" | "OrderBy" | "OrderByDescending" @@ -333,18 +368,108 @@ module ProviderShape = | "Take" | "TakeWhile" | "Distinct" - | "DefaultIfEmpty" -> sourceUnnamed + | "DefaultIfEmpty" + | "Reverse" + | "AsQueryable" + | "AsEnumerable" -> this.Unnamed source + | "Concat" + | "Union" + | "Intersect" + | "Except" -> this.Unnamed source || (call.Arguments.Count > 1 && this.Unnamed call.Arguments.[1]) + | _ -> false + | :? ConstantExpression as constant -> + match constant.Value with + | :? IQueryable as query when not (isRoot query) -> this.Unnamed query.Expression + | _ -> false + | :? ParameterExpression as parameter -> + match sequences.TryGetValue parameter with + | true, unnamed -> unnamed | _ -> false + | :? UnaryExpression as conversion when conversion.NodeType = ExpressionType.Convert -> this.Unnamed conversion.Operand | _ -> false - producesUnnamed expression |> ignore - List.ofSeq found + /// Reports the members that the lambdas of the operator read from elements built without names. + member this.Check(call: MethodCallExpression) = + let source = call.Arguments.[0] + + match call.Method.Name with + | "Join" + | "GroupJoin" when call.Arguments.Count >= 5 -> + let outer, inner = call.Arguments.[0], call.Arguments.[1] + let outerUnnamed, innerUnnamed = this.Unnamed outer, this.Unnamed inner + + lambdaAt call 2 |> ValueOption.iter (fun key -> report key 0 outer outerUnnamed) + lambdaAt call 3 |> ValueOption.iter (fun key -> report key 0 inner innerUnnamed) + + lambdaAt call 4 + |> ValueOption.iter (fun result -> + report result 0 outer outerUnnamed + + if call.Method.Name = "Join" then + report result 1 inner innerUnnamed + elif result.Parameters.Count > 1 then + // The second parameter is the group of the matching inner elements, not one of them + sequences.[result.Parameters.[1]] <- innerUnnamed) + | "SelectMany" when call.Arguments.Count >= 3 -> + // A collection selector, then a result selector over the element and an element of the collection + let sourceUnnamed = this.Unnamed source + let collection = lambdaAt call 1 + collection |> ValueOption.iter (fun selector -> report selector 0 source sourceUnnamed) + + lambdaAt call 2 + |> ValueOption.iter (fun result -> + report result 0 source sourceUnnamed + + collection + |> ValueOption.iter (fun selector -> report result 1 selector.Body (this.Unnamed selector.Body))) + | "Zip" when call.Arguments.Count = 3 -> + let second = call.Arguments.[1] + + lambdaAt call 2 + |> ValueOption.iter (fun result -> + report result 0 source (this.Unnamed source) + report result 1 second (this.Unnamed second)) + // The first parameter of the function is the accumulated value + | "Aggregate" -> () + | _ -> + let sourceUnnamed = this.Unnamed source + + for index in 1 .. call.Arguments.Count - 1 do + lambdaAt call index |> ValueOption.iter (fun lambda -> report lambda 0 source sourceUnnamed) + + override this.VisitMethodCall node = + if isLinqOperator node then + this.Check node + + base.VisitMethodCall node + + // A query built earlier and used in this one is part of what the provider translates. + override this.VisitConstant node = + match node.Value with + | :? IQueryable as query when not (isRoot query) -> this.Visit query.Expression |> ignore + | _ -> () + + node + + let private unnamedMemberReads (expression: Expression) = + let finder = UnnamedMemberFinder() + finder.Visit expression |> ignore + finder.Found let violations (expression: Expression) = let visitor = ShapeVisitor() visitor.Visit expression |> ignore visitor.Violations @ unnamedMemberReads expression |> List.distinct + /// Whether the expression calls a Queryable operator that gives a value and not a query, as a provider is asked + /// to execute when a query is counted, summed or searched. + let isTerminalOperator (expression: Expression) = + match expression with + | :? MethodCallExpression as call -> + call.Method.DeclaringType = typeof + && not (typeof.IsAssignableFrom call.Type) + | _ -> false + /// The LINQ operators along the source chain, outermost last. let spine (expression: Expression) = let rec walk (expression: Expression) operators = diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs index b2a270042e9..094519b89e0 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs @@ -133,10 +133,18 @@ module Scenario = Rows = [ for row in rows -> row ] } // The value comes from the last execution; whatever ran before it went to the provider too. | Outcome.Value value when provider.Executed.Count > 0 -> + let last = provider.Executed.[provider.Executed.Count - 1] let earlier = provider.Executed |> Seq.take (provider.Executed.Count - 1) - { Expression = provider.Executed.[provider.Executed.Count - 1] - Violations = [ for expression in earlier do yield! ProviderShape.violations expression ] + { Expression = last + Violations = + [ for expression in earlier do + yield! ProviderShape.violations expression + + // Enumerating the rows of a query is an execution too, so the last one has to be the + // operator that gives the value: a count made over those rows is made by the client. + if not (ProviderShape.isTerminalOperator last) then + Violation.LeftProvider ] Rows = [ value ] } | Outcome.Value value -> { Expression = Expression.Empty() diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs index 6e284ed7d4d..0b2ce330374 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs @@ -2,6 +2,11 @@ namespace FSharp.Core.UnitTests.Linq +open System +open System.Collections.Generic +open System.Linq +open System.Linq.Expressions +open System.Reflection open Xunit type QueryShapeTests() = @@ -29,3 +34,229 @@ type QueryShapeTests() = [] member _.``Method syntax``(scenario: Scenario) = Scenario.check scenario + +/// Trees built by hand, the way a C# compiler builds them, to check the validator itself. They do not depend on how +/// FSharp.Core translates a query, so they mean the same in every branch. +module private ValidatorTrees = + + let customers = Queryable.AsQueryable Seq.empty + let orders = Queryable.AsQueryable Seq.empty + + let customerPair = typeof> + let orderPair = typeof> + + let call (name: string) (typeArguments: Type list) (arguments: Expression list) = + Expression.Call(typeof, name, Array.ofList typeArguments, Array.ofList arguments) :> Expression + + let quote (lambda: LambdaExpression) = Expression.Quote lambda :> Expression + + let pair (tuple: Type) (arguments: Expression list) (named: bool) = + let constructor = tuple.GetConstructors().[0] + + if named then + Expression.New(constructor, arguments, [| tuple.GetProperty "Item1" :> MemberInfo; tuple.GetProperty "Item2" :> MemberInfo |]) + else + Expression.New(constructor, arguments) + + /// customers.Select(c => new Tuple(c.Id, c.Name)), with or without naming the members + let customerPairs (named: bool) = + let c = Expression.Parameter(typeof, "c") + let body = pair customerPair [ Expression.Property(c, "Id"); Expression.Property(c, "Name") ] named + + call "Select" [ typeof; customerPair ] [ customers.Expression; quote (Expression.Lambda(body, c)) ] + + /// orders.Select(o => new Tuple(o.Id, o.Total)), with or without naming the members + let orderPairs (named: bool) = + let o = Expression.Parameter(typeof, "o") + let body = pair orderPair [ Expression.Property(o, "Id"); Expression.Property(o, "Total") ] named + + call "Select" [ typeof; orderPair ] [ orders.Expression; quote (Expression.Lambda(body, o)) ] + + /// source.Where(t => t.Item1 > 0) + let whereItem1Positive (source: Expression) (tuple: Type) = + let t = Expression.Parameter(tuple, "t") + let predicate = Expression.Lambda(Expression.GreaterThan(Expression.Property(t, "Item1"), Expression.Constant(0)), t) + + call "Where" [ tuple ] [ source; quote predicate ] + + let item1 (p: ParameterExpression) = Expression.Property(p, "Item1") :> Expression + let item2 (p: ParameterExpression) = Expression.Property(p, "Item2") :> Expression + let zero (_: ParameterExpression) = Expression.Constant(0) :> Expression + + /// outer.Join(inner, o => outerKey(o), i => innerKey(i), (o, i) => result(o, i)), with int keys + let join + (outer: Expression) + (outerElement: Type) + (inner: Expression) + (innerElement: Type) + (outerKey: ParameterExpression -> Expression) + (innerKey: ParameterExpression -> Expression) + (result: ParameterExpression -> ParameterExpression -> Expression) + = + let parameter (ty: Type) name = Expression.Parameter(ty, name) + let o1 = parameter outerElement "o" + let i1 = parameter innerElement "i" + let o2 = parameter outerElement "o" + let i2 = parameter innerElement "i" + let resultBody = result o2 i2 + + call + "Join" + [ outerElement; innerElement; typeof; resultBody.Type ] + [ outer + inner + quote (Expression.Lambda(outerKey o1, o1)) + quote (Expression.Lambda(innerKey i1, i1)) + quote (Expression.Lambda(resultBody, o2, i2)) ] + +type ProviderShapeValidatorTests() = + + static let unnamedRead = [ Violation.ReadsUnnamedMember "Tuple`2" ] + + static let reported tree = ProviderShape.violations tree + + [] + member _.``A member read from an unnamed projection is reported``() = + let unnamed = ValidatorTrees.whereItem1Positive (ValidatorTrees.customerPairs false) ValidatorTrees.customerPair + let named = ValidatorTrees.whereItem1Positive (ValidatorTrees.customerPairs true) ValidatorTrees.customerPair + + Assert.Equal(unnamedRead, reported unnamed) + Assert.Empty(reported named) + + [] + member _.``A chain inside the lambda of another operator is checked``() = + let nested named = + let inner = ValidatorTrees.whereItem1Positive (ValidatorTrees.orderPairs named) ValidatorTrees.orderPair + let any = ValidatorTrees.call "Any" [ ValidatorTrees.orderPair ] [ inner ] + let c = Expression.Parameter(typeof, "c") + + ValidatorTrees.call "Where" [ typeof ] [ ValidatorTrees.customers.Expression; ValidatorTrees.quote (Expression.Lambda(any, c)) ] + + Assert.Equal(unnamedRead, reported (nested false)) + Assert.Empty(reported (nested true)) + + [] + member _.``A query captured from earlier is checked where it is used``() = + let capturing named = + let captured = ValidatorTrees.customers.Provider.CreateQuery>(ValidatorTrees.customerPairs named) + let o = Expression.Parameter(typeof, "o") + let t = Expression.Parameter(ValidatorTrees.customerPair, "t") + let predicate = Expression.Lambda(Expression.GreaterThan(Expression.Property(t, "Item1"), Expression.Property(o, "Id")), t) + let any = ValidatorTrees.call "Any" [ ValidatorTrees.customerPair ] [ Expression.Constant(captured) :> Expression; ValidatorTrees.quote predicate ] + + ValidatorTrees.call "Where" [ typeof ] [ ValidatorTrees.orders.Expression; ValidatorTrees.quote (Expression.Lambda(any, o)) ] + + Assert.Equal(unnamedRead, reported (capturing false)) + Assert.Empty(reported (capturing true)) + + [] + member _.``An identity Select keeps the elements unnamed``() = + let t = Expression.Parameter(ValidatorTrees.customerPair, "t") + + let identity = + ValidatorTrees.call + "Select" + [ ValidatorTrees.customerPair; ValidatorTrees.customerPair ] + [ ValidatorTrees.customerPairs false; ValidatorTrees.quote (Expression.Lambda(t, t)) ] + + Assert.Equal(unnamedRead, reported (ValidatorTrees.whereItem1Positive identity ValidatorTrees.customerPair)) + + [] + member _.``The elements of a collection selector stay unnamed after SelectMany``() = + let flattened named = + let c = Expression.Parameter(typeof, "c") + let collectionType = typedefof>.MakeGenericType(typeof, typeof>>) + let collection = Expression.Lambda(collectionType, ValidatorTrees.orderPairs named, c) + + ValidatorTrees.call + "SelectMany" + [ typeof; ValidatorTrees.orderPair ] + [ ValidatorTrees.customers.Expression; ValidatorTrees.quote collection ] + + Assert.Equal(unnamedRead, reported (ValidatorTrees.whereItem1Positive (flattened false) ValidatorTrees.orderPair)) + Assert.Empty(reported (ValidatorTrees.whereItem1Positive (flattened true) ValidatorTrees.orderPair)) + + [] + member _.``A named input of a join is not blamed for the other one being unnamed``() = + // The two inputs have the same tuple type, and only the outer one is unnamed. Neither key reads a member of it. + let tree = + ValidatorTrees.join + (ValidatorTrees.customerPairs false) + ValidatorTrees.customerPair + (ValidatorTrees.customerPairs true) + ValidatorTrees.customerPair + ValidatorTrees.zero + ValidatorTrees.item1 + (fun _ _ -> ValidatorTrees.zero null) + + Assert.Empty(reported tree) + + [] + member _.``An unnamed inner input of a join is checked against the inner key``() = + let tree = + ValidatorTrees.join + (ValidatorTrees.customerPairs true) + ValidatorTrees.customerPair + (ValidatorTrees.customerPairs false) + ValidatorTrees.customerPair + ValidatorTrees.zero + ValidatorTrees.item1 + (fun _ _ -> ValidatorTrees.zero null) + + Assert.Equal(unnamedRead, reported tree) + + [] + member _.``An unnamed inner input of a join is checked against the second parameter of the result``() = + let tree = + ValidatorTrees.join + (ValidatorTrees.customerPairs true) + ValidatorTrees.customerPair + (ValidatorTrees.customerPairs false) + ValidatorTrees.customerPair + ValidatorTrees.zero + ValidatorTrees.zero + (fun _ i -> ValidatorTrees.item2 i) + + Assert.Equal(unnamedRead, reported tree) + + [] + member _.``An unnamed inner input of a join is checked when the outer input is a plain source``() = + let tree = + ValidatorTrees.join + ValidatorTrees.customers.Expression + typeof + (ValidatorTrees.orderPairs false) + ValidatorTrees.orderPair + (fun c -> Expression.Property(c, "Id") :> Expression) + ValidatorTrees.item1 + (fun _ _ -> ValidatorTrees.zero null) + + Assert.Equal(unnamedRead, reported tree) + + [] + member _.``A selector that is not a lambda is passed over``() = + let selector = + Expression.Constant( + Expression.Lambda>(Expression.Constant(0), Expression.Parameter(typeof, "c")), + typeof>> + ) + + let tree = + ValidatorTrees.call "Select" [ typeof; typeof ] [ ValidatorTrees.customers.Expression; selector :> Expression ] + + Assert.Empty(reported tree) + + [] + member _.``A count made by the provider passes``() = + Scenario.value "count by the provider" (fun db -> db.Customers.Where(fun c -> c.Age > 26).Count()) (fun _ -> 3) + |> Scenario.check + + [] + member _.``A count made over the rows of the provider leaves it``() = + Scenario.value "count on the client" (fun db -> db.Customers.Where(fun c -> c.Age > 26).AsEnumerable().Count()) (fun _ -> 3) + |> Scenario.violating [ Violation.LeftProvider ] + |> Scenario.check + + Scenario.value "count by Enumerable" (fun db -> Enumerable.Count(db.Customers.Where(fun c -> c.Age > 26))) (fun _ -> 3) + |> Scenario.violating [ Violation.LeftProvider ] + |> Scenario.check From 521e8300b0b76d62761543a093005f4c589e62c8 Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Mon, 2 Nov 2026 14:24:09 +0100 Subject: [PATCH 05/19] Count every execution and every violation when checking a scenario A value scenario was accepted when the last execution at the provider was an operator that gives a value. A count made by the client over rows taken from the provider passed whenever a function it ran asked the provider for a count of its own, because that count came last. Now every execution has to be such an operator, and so does the enumeration of a source taken as it is, which is recorded as an execution too. The violations of a tree are compared with every occurrence: a tree with one block was equal to a tree with two. The scenarios that have several of a kind are pinned with their number. The validator tests build a tree with an invoked lambda and one with an unbound parameter, which no scenario has, and the two client counts. The release note names the error a struct tuple gave on main. Co-Authored-By: Claude Sonnet 5.5 --- docs/release-notes/.FSharp.Core/11.0.200.md | 2 +- .../QueryKeyScenarios.fs | 8 ++- .../QueryMethodSyntaxScenarios.fs | 14 +++-- .../QueryProviderShape.fs | 30 ++++++---- .../Microsoft.FSharp.Linq/QueryScenario.fs | 22 ++++--- .../Microsoft.FSharp.Linq/QueryShapeTests.fs | 59 +++++++++++++++++++ 6 files changed, 107 insertions(+), 28 deletions(-) diff --git a/docs/release-notes/.FSharp.Core/11.0.200.md b/docs/release-notes/.FSharp.Core/11.0.200.md index 0aa52c42d5a..74a8bab02a3 100644 --- a/docs/release-notes/.FSharp.Core/11.0.200.md +++ b/docs/release-notes/.FSharp.Core/11.0.200.md @@ -6,6 +6,6 @@ * Correct `FSharpValue.PreComputeTuplePropertyInfo` to report that struct tuples store elements in fields, not properties, and recommend supported tuple readers instead of reporting a valid index as out of range. ([Issue #20801](https://github.com/dotnet/fsharp/issues/20801), [PR #20815](https://github.com/dotnet/fsharp/pull/20815)) * Quotation debug text from `Expr.ToString()`, `Expr.ToString(full)`, and `%A` now prints struct tuple construction as `NewStructTuple`; reference tuples remain `NewTuple`. ([Issue #20800](https://github.com/dotnet/fsharp/issues/20800), [PR #20814](https://github.com/dotnet/fsharp/pull/20814)) -* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) +* Fix struct tuples in query expressions: selecting one no longer fails with "unreachable", and the items of one that a later operator reads are taken from its fields instead of failing with `ArgumentException: Struct tuples store their elements in fields, not properties`. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) ### Changed diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs index 03011a4d99f..0c7e32749cb 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs @@ -8,6 +8,10 @@ open Microsoft.FSharp.Linq.RuntimeHelpers let private block = [ Violation.Statement ExpressionType.Block ] +/// A tree is pinned with every block it has: one that goes from three to two is a change as well +let private blocks count = + List.replicate count (Violation.Statement ExpressionType.Block) + let private customers (data: Data) = Seq.ofList data.Customers /// Orders paired with the lines that share both the order number and, as quantity, the customer number. @@ -68,7 +72,7 @@ let private joins () = select (o.Id, l.Id) }) matchingLines - |> Scenario.violating block + |> Scenario.violating (blocks 2) Scenario.rows "join on an anonymous record, fields in alphabetical order" @@ -89,7 +93,7 @@ let private joins () = select (o.Id, l.Id) }) matchingLines - |> Scenario.violating block + |> Scenario.violating (blocks 2) Scenario.rows "groupJoin on a tuple" diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs index 5325268456b..760c9b78a49 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs @@ -7,6 +7,10 @@ open System.Linq.Expressions let private block = [ Violation.Statement ExpressionType.Block ] +/// A tree is pinned with every block it has: one that goes from three to two is a change as well +let private blocks count = + List.replicate count (Violation.Statement ExpressionType.Block) + let private reading typeName = [ Violation.ReadsUnnamedMember typeName ] @@ -67,7 +71,7 @@ let cases () = "Where taking a tuple apart" (fun db -> db.Customers.Select(fun c -> (c.Id, c.Name)).Where(fun (id, _) -> id > 1)) (customers >> Seq.map (fun c -> c.Id, c.Name) >> Seq.filter (fun (id, _) -> id > 1)) - |> Scenario.violating (reading "Tuple`2" @ block) + |> Scenario.violating (reading "Tuple`2" @ blocks 2) Scenario.rows "Where taking a struct tuple apart" @@ -83,7 +87,7 @@ let cases () = .Where(fun (_, _, _, _, _, _, _, _, last) -> last > 1) .Select(fun (_, name, _, _, _, _, _, _, _) -> name)) (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) - |> Scenario.violating (reading "Tuple`8" @ block) + |> Scenario.violating (reading "Tuple`8" @ reading "Tuple`8" @ blocks 18) Scenario.rows "Where taking the last of a struct nine apart" @@ -93,19 +97,19 @@ let cases () = .Where(fun (struct (_, _, _, _, _, _, _, _, last)) -> last > 1) .Select(fun (struct (_, name, _, _, _, _, _, _, _)) -> name)) (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) - |> Scenario.violating (reading "ValueTuple`8" @ block) + |> Scenario.violating (reading "ValueTuple`8" @ reading "ValueTuple`8" @ blocks 2) Scenario.rows "Where over a query that selects a tuple" (fun db -> (query { for c in db.Customers do select (c.Id, c.Name) }).Where(fun (id, _) -> id > 1)) (customers >> Seq.map (fun c -> c.Id, c.Name) >> Seq.filter (fun (id, _) -> id > 1)) - |> Scenario.violating (reading "Tuple`2" @ block) + |> Scenario.violating (reading "Tuple`2" @ blocks 2) Scenario.rows "Where on a nullable property" (fun db -> db.Customers.Where(fun c -> c.Registered.HasValue && c.Registered.Value.Year > 2020)) (customers >> Seq.filter (fun c -> c.Registered.HasValue && c.Registered.Value.Year > 2020)) - |> Scenario.violating block + |> Scenario.violating (blocks 3) Scenario.rows "Where with a lambda over another source" diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs index ae25aa7b1d6..20665171824 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs @@ -25,12 +25,15 @@ type private InMemoryRewriter() = /// An in-memory query that stays on the provider that created it and prints its root by name. [] -type CapturedQuery<'T>(provider: IQueryProvider, name: string, items: seq<'T>, derived: Expression voption) = +type CapturedQuery<'T>(provider: IQueryProvider, name: string, items: seq<'T>, derived: Expression voption, scanned: Expression -> unit) = - let enumerate () = + let enumerate (self: Expression) = match derived with | ValueSome expression -> provider.Execute>(expression).GetEnumerator() - | ValueNone -> items.GetEnumerator() + | ValueNone -> + // No operator runs at the provider here: whoever enumerates the source takes all of its rows + scanned self + items.GetEnumerator() override _.ToString() = match derived with @@ -51,8 +54,8 @@ type CapturedQuery<'T>(provider: IQueryProvider, name: string, items: seq<'T>, d member _.ElementType = typeof<'T> member _.Provider = provider - member _.GetEnumerator() : IEnumerator<'T> = enumerate () - member _.GetEnumerator() : IEnumerator = enumerate () + member this.GetEnumerator() : IEnumerator<'T> = enumerate (Expression.Constant this) + member this.GetEnumerator() : IEnumerator = enumerate (Expression.Constant this) /// Executes in memory, and records what it was asked to execute. [] @@ -74,14 +77,19 @@ type CapturingQueryProvider() = finally depth <- depth - 1 - /// The expressions executed directly, not from inside another execution. + /// The expressions executed directly, not from inside another execution, and the sources that were enumerated. member _.Executed = executed :> IReadOnlyList + /// A source that is enumerated as it is, without an operator + member _.Scanned(source: Expression) = + if depth = 0 then + executed.Add source + member this.Source(name: string, items: seq<'T>) : IQueryable<'T> = - CapturedQuery<'T>(this, name, items, ValueNone) + CapturedQuery<'T>(this, name, items, ValueNone, this.Scanned) member this.Derived<'T>(expression: Expression) : IQueryable<'T> = - CapturedQuery<'T>(this, "", Seq.empty, ValueSome expression) + CapturedQuery<'T>(this, "", Seq.empty, ValueSome expression, this.Scanned) interface IQueryProvider with member this.CreateQuery<'T>(expression: Expression) : IQueryable<'T> = this.Derived<'T> expression @@ -172,7 +180,7 @@ module ProviderShape = | _ -> ValueNone | _ -> ValueNone - member _.Violations = found |> Seq.distinct |> List.ofSeq + member _.Violations = List.ofSeq found override _.VisitLambda<'T>(node: Expression<'T>) = let added = bind node.Parameters @@ -332,7 +340,7 @@ module ProviderShape = if yields parameter.Type sequence && readsMemberOf parameter lambda.Body then found.Add(Violation.ReadsUnnamedMember parameter.Type.Name) - member _.Found = found |> Seq.distinct |> List.ofSeq + member _.Found = List.ofSeq found /// Whether the elements of the sequence were built without naming their members. member this.Unnamed(sequence: Expression) : bool = @@ -459,7 +467,7 @@ module ProviderShape = let violations (expression: Expression) = let visitor = ShapeVisitor() visitor.Visit expression |> ignore - visitor.Violations @ unnamedMemberReads expression |> List.distinct + visitor.Violations @ unnamedMemberReads expression /// Whether the expression calls a Queryable operator that gives a value and not a query, as a provider is asked /// to execute when a query is counted, summed or searched. diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs index 094519b89e0..042b108c16a 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs @@ -131,19 +131,22 @@ module Scenario = { Expression = rows.Expression Violations = (if stayed then [] else [ Violation.LeftProvider ]) Rows = [ for row in rows -> row ] } - // The value comes from the last execution; whatever ran before it went to the provider too. + // A query that is counted, summed or searched is executed by the operator that gives the value. + // Enumerating the rows of a query is an execution too, and so is every query that a function on the + // client runs while it goes through those rows: the provider made the value only if every + // execution is such an operator. The last one is the expression that is checked as a whole; whatever + // ran before it went to the provider too. | Outcome.Value value when provider.Executed.Count > 0 -> - let last = provider.Executed.[provider.Executed.Count - 1] - let earlier = provider.Executed |> Seq.take (provider.Executed.Count - 1) + let executed = List.ofSeq provider.Executed + let last = List.last executed + let earlier = List.take (executed.Length - 1) executed { Expression = last Violations = [ for expression in earlier do yield! ProviderShape.violations expression - // Enumerating the rows of a query is an execution too, so the last one has to be the - // operator that gives the value: a count made over those rows is made by the client. - if not (ProviderShape.isTerminalOperator last) then + if not (List.forall ProviderShape.isTerminalOperator executed) then Violation.LeftProvider ] Rows = [ value ] } | Outcome.Value value -> @@ -165,9 +168,10 @@ module Scenario = let failures (scenario: Scenario) = let data, observed = observe scenario + // Every occurrence counts: a tree with two blocks is not a tree with one let violations = observed.Violations @ ProviderShape.violations observed.Expression - |> List.distinct + |> List.sort let threw = violations @@ -175,8 +179,8 @@ module Scenario = | Violation.Throws _ -> true | _ -> false) - [ if set violations <> set scenario.Violations then - $"violations: expected %A{scenario.Violations}, got %A{violations}" + [ if violations <> List.sort scenario.Violations then + $"violations: expected %A{List.sort scenario.Violations}, got %A{violations}" if not threw then let expected = inOrder scenario.Ordered (scenario.Expected data) diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs index 0b2ce330374..35c106eedab 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs @@ -246,6 +246,48 @@ type ProviderShapeValidatorTests() = Assert.Empty(reported tree) + [] + member _.``A block is reported once for each block``() = + let c = Expression.Parameter(typeof, "c") + + let block (name: string) = + let x = Expression.Variable(typeof, name) + Expression.Block([| x |], Expression.Assign(x, Expression.Property(c, "Id")), Expression.GreaterThan(x, Expression.Constant 0)) :> Expression + + let where (body: Expression) = + ValidatorTrees.call "Where" [ typeof ] [ ValidatorTrees.customers.Expression; ValidatorTrees.quote (Expression.Lambda(body, c)) ] + + let blockViolation = Violation.Statement ExpressionType.Block + + Assert.Equal([ blockViolation ], reported (where (block "x"))) + Assert.Equal([ blockViolation; blockViolation ], reported (where (Expression.AndAlso(block "x", block "y")))) + + [] + member _.``A lambda that is invoked is reported``() = + let c = Expression.Parameter(typeof, "c") + let n = Expression.Parameter(typeof, "n") + let positive = Expression.Lambda>(Expression.GreaterThan(n, Expression.Constant 0), n) + let tree = Expression.Invoke(positive, Expression.Property(c, "Id")) + + let where = + ValidatorTrees.call "Where" [ typeof ] [ ValidatorTrees.customers.Expression; ValidatorTrees.quote (Expression.Lambda(tree, c)) ] + + Assert.Equal([ Violation.LambdaInvocation ], reported where) + + [] + member _.``A parameter that no lambda binds is reported``() = + let c = Expression.Parameter(typeof, "c") + let n = Expression.Parameter(typeof, "n") + + let where = + ValidatorTrees.call + "Where" + [ typeof ] + [ ValidatorTrees.customers.Expression + ValidatorTrees.quote (Expression.Lambda(Expression.GreaterThan(Expression.Property(c, "Id"), n), c)) ] + + Assert.Equal([ Violation.UnboundParameter "n" ], reported where) + [] member _.``A count made by the provider passes``() = Scenario.value "count by the provider" (fun db -> db.Customers.Where(fun c -> c.Age > 26).Count()) (fun _ -> 3) @@ -260,3 +302,20 @@ type ProviderShapeValidatorTests() = Scenario.value "count by Enumerable" (fun db -> Enumerable.Count(db.Customers.Where(fun c -> c.Age > 26))) (fun _ -> 3) |> Scenario.violating [ Violation.LeftProvider ] |> Scenario.check + + [] + member _.``A count made by the client over counts made by the provider leaves it``() = + // The provider counts the orders once for each customer, and the client counts the customers + Scenario.value + "client count with a provider count in its projection" + (fun db -> db.Customers.Where(fun c -> c.Age > 26).AsEnumerable().Select(fun _ -> db.Orders.Count()).Count()) + (fun _ -> 3) + |> Scenario.violating [ Violation.LeftProvider ] + |> Scenario.check + + Scenario.value + "client count with an inner provider count" + (fun db -> Enumerable.Count(db.Customers.Select(fun _ -> db.Orders.Count()))) + (fun data -> data.Customers.Length) + |> Scenario.violating [ Violation.LeftProvider ] + |> Scenario.check From b418d09d7657b552a834fbfb4d834fa78cf09b1a Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Mon, 2 Nov 2026 17:37:24 +0100 Subject: [PATCH 06/19] Check the elements a result selector hands on, and compare entities by what they hold The shape tests did not see an unnamed element that passes through a result selector giving one of its parameters back, in SelectMany, Join and Zip, and they skipped the element parameter of Aggregate together with the accumulated value. Both are checked now. A function with a seed takes the accumulated value and an element, and one without a seed takes two elements. A scenario compared an entity by the text it makes of itself, which names its key only: a result with the right key and the wrong name or age passed. An object of a class of the scenarios is now compared by every property that gives a plain value. Co-Authored-By: Claude Sonnet 5.5 --- .../QueryProviderShape.fs | 38 ++++- .../Microsoft.FSharp.Linq/QueryScenario.fs | 32 ++++ .../Microsoft.FSharp.Linq/QueryShapeTests.fs | 146 ++++++++++++++++++ 3 files changed, 208 insertions(+), 8 deletions(-) diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs index 20665171824..8709087cd59 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs @@ -302,9 +302,9 @@ module ProviderShape = let private yields (elementType: Type) (source: Expression) = typedefof>.MakeGenericType(elementType).IsAssignableFrom source.Type - /// Whether the lambda gives its first parameter back as it is. - let private returnsParameter (lambda: LambdaExpression) = - lambda.Parameters.Count > 0 && obj.ReferenceEquals(lambda.Body, lambda.Parameters.[0]) + /// Whether the lambda gives the parameter at the index back as it is. + let private returnsParameterAt index (lambda: LambdaExpression) = + index < lambda.Parameters.Count && obj.ReferenceEquals(lambda.Body, lambda.Parameters.[index]) /// A query that is its own root, as opposed to one built from another by operators. let private isRoot (query: IQueryable) = @@ -354,18 +354,30 @@ module ProviderShape = | ValueSome lambda -> lambda | ValueNone -> () ] + // A function of an element of each of two sequences gives unnamed elements when it builds them without + // names, and when it gives back one of its parameters of which the sequence has unnamed elements + let unnamedResult (result: LambdaExpression) (second: Expression) = + buildsUnnamed result.Body + || (returnsParameterAt 0 result && this.Unnamed source) + || (returnsParameterAt 1 result && this.Unnamed second) + match call.Method.Name with | "Select" -> lambdas |> List.tryLast - |> Option.exists (fun selector -> buildsUnnamed selector.Body || (returnsParameter selector && this.Unnamed source)) + |> Option.exists (fun selector -> buildsUnnamed selector.Body || (returnsParameterAt 0 selector && this.Unnamed source)) | "SelectMany" -> match lambdas with | [ collection ] -> this.Unnamed collection.Body - | [ _; result ] -> buildsUnnamed result.Body + | [ collection; result ] -> unnamedResult result collection.Body | _ -> false | "Join" - | "GroupJoin" -> lambdas |> List.tryLast |> Option.exists (fun result -> buildsUnnamed result.Body) + | "Zip" -> lambdas |> List.tryLast |> Option.exists (fun result -> unnamedResult result call.Arguments.[1]) + // The second parameter is the group of the matching inner elements, which no element is + | "GroupJoin" -> + lambdas + |> List.tryLast + |> Option.exists (fun result -> buildsUnnamed result.Body || (returnsParameterAt 0 result && this.Unnamed source)) | "Where" | "OrderBy" | "OrderByDescending" @@ -437,8 +449,18 @@ module ProviderShape = |> ValueOption.iter (fun result -> report result 0 source (this.Unnamed source) report result 1 second (this.Unnamed second)) - // The first parameter of the function is the accumulated value - | "Aggregate" -> () + // The function takes the accumulated value, then an element. With no seed the accumulated value starts as + // an element, and a seed is of a type of its own, which is no element. + | "Aggregate" -> + let sourceUnnamed = this.Unnamed source + let seeded = call.Arguments.Count > 2 + + lambdaAt call (if seeded then 2 else 1) + |> ValueOption.iter (fun func -> + if not seeded then + report func 0 source sourceUnnamed + + report func 1 source sourceUnnamed) | _ -> let sourceUnnamed = this.Unnamed source diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs index 042b108c16a..9ae260a7226 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs @@ -60,6 +60,35 @@ module Scenario = else ValueNone + let rec private isPlain (ty: Type) = + ty.IsPrimitive + || ty.IsEnum + || ty = typeof + || ty = typeof + || ty = typeof + || ty = typeof + || ty = typeof + || (match Nullable.GetUnderlyingType ty with + | null -> false + | underlying -> isPlain underlying) + + /// An object of a class of the scenarios, with what it holds in the properties that give plain values. Its own + /// text says no more than the key of an entity, and a result that is wrong in any other property is not equal to + /// the right one. + [] + let private (|Entity|_|) (value: obj) = + let ty = value.GetType() + + if ty.IsClass && String.Equals(ty.Namespace, typeof.Namespace, StringComparison.Ordinal) then + ValueSome( + ty.GetProperties(BindingFlags.Public ||| BindingFlags.Instance) + |> Seq.filter (fun property -> property.GetIndexParameters().Length = 0 && isPlain property.PropertyType) + |> Seq.map (fun property -> struct (property.Name, property.GetValue value)) + |> Seq.toList + ) + else + ValueNone + /// Text that is equal for equal results, whichever provider materialized them. let rec render (value: obj) = let renderAll (values: seq) = @@ -76,6 +105,9 @@ module Scenario = let rendered = fields |> Seq.map (fun (name, field) -> $"{name} = {render field}") $"""{{ {String.concat "; " rendered} }}""" | Union(case, fields) -> $"{case}({renderAll fields})" + | Entity properties -> + let rendered = properties |> Seq.map (fun (struct (name, property)) -> $"{name} = {render property}") + $"""{value.GetType().Name} {{ {String.concat "; " rendered} }}""" | value -> string value let rows name (run: Db -> IQueryable<'T>) (expected: Data -> seq<'T>) = diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs index 35c106eedab..94b9d25edf6 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs @@ -319,3 +319,149 @@ type ProviderShapeValidatorTests() = (fun data -> data.Customers.Length) |> Scenario.violating [ Violation.LeftProvider ] |> Scenario.check + + [] + member _.``A result selector that gives back an unnamed element keeps the elements unnamed``() = + let item = Expression.Parameter(typeof, "item") + + let flattened named = + let pair = Expression.Parameter(ValidatorTrees.customerPair, "pair") + let collectionType = typedefof>.MakeGenericType(ValidatorTrees.customerPair, typeof>) + let collection = Expression.Lambda(collectionType, Expression.NewArrayInit(typeof, Expression.Constant 0), pair) + let first = Expression.Parameter(ValidatorTrees.customerPair, "pair") + + ValidatorTrees.call + "SelectMany" + [ ValidatorTrees.customerPair; typeof; ValidatorTrees.customerPair ] + [ ValidatorTrees.customerPairs named + ValidatorTrees.quote collection + ValidatorTrees.quote (Expression.Lambda(first, first, item)) ] + + Assert.Equal(unnamedRead, reported (ValidatorTrees.whereItem1Positive (flattened false) ValidatorTrees.customerPair)) + Assert.Empty(reported (ValidatorTrees.whereItem1Positive (flattened true) ValidatorTrees.customerPair)) + + [] + member _.``The second parameter of a SelectMany result selector is an element of the collection``() = + let collected named = + let customer = Expression.Parameter(typeof, "c") + let collectionType = typedefof>.MakeGenericType(typeof, typeof>>) + let collection = Expression.Lambda(collectionType, ValidatorTrees.orderPairs named, customer) + let owner = Expression.Parameter(typeof, "c") + let order = Expression.Parameter(ValidatorTrees.orderPair, "o") + + ValidatorTrees.call + "SelectMany" + [ typeof; ValidatorTrees.orderPair; ValidatorTrees.orderPair ] + [ ValidatorTrees.customers.Expression + ValidatorTrees.quote collection + ValidatorTrees.quote (Expression.Lambda(order, owner, order)) ] + + Assert.Equal(unnamedRead, reported (ValidatorTrees.whereItem1Positive (collected false) ValidatorTrees.orderPair)) + Assert.Empty(reported (ValidatorTrees.whereItem1Positive (collected true) ValidatorTrees.orderPair)) + + [] + member _.``A join or a zip that gives back an unnamed element keeps the elements unnamed``() = + let joined outerNamed innerNamed (result: ParameterExpression -> ParameterExpression -> Expression) = + ValidatorTrees.join + (ValidatorTrees.customerPairs outerNamed) + ValidatorTrees.customerPair + (ValidatorTrees.customerPairs innerNamed) + ValidatorTrees.customerPair + ValidatorTrees.zero + ValidatorTrees.zero + result + + let check tree = reported (ValidatorTrees.whereItem1Positive tree ValidatorTrees.customerPair) + + Assert.Equal(unnamedRead, check (joined false true (fun o _ -> o))) + Assert.Empty(check (joined true false (fun o _ -> o))) + Assert.Equal(unnamedRead, check (joined true false (fun _ i -> i))) + Assert.Empty(check (joined false true (fun _ i -> i))) + + [] + member _.``A zip that gives back an unnamed element keeps the elements unnamed``() = + let zipped firstNamed secondNamed = + let a = Expression.Parameter(ValidatorTrees.customerPair, "a") + let b = Expression.Parameter(ValidatorTrees.customerPair, "b") + + ValidatorTrees.whereItem1Positive + (ValidatorTrees.call + "Zip" + [ ValidatorTrees.customerPair; ValidatorTrees.customerPair; ValidatorTrees.customerPair ] + [ ValidatorTrees.customerPairs firstNamed + ValidatorTrees.customerPairs secondNamed + ValidatorTrees.quote (Expression.Lambda(a, a, b)) ]) + ValidatorTrees.customerPair + + Assert.Equal(unnamedRead, reported (zipped false true)) + Assert.Empty(reported (zipped true false)) + + [] + member _.``The element an Aggregate function takes is checked, and a seed that is accumulated is not``() = + // customers.Select(c => new Tuple(c.Id, c.Name)).Aggregate(0, (acc, pair) => acc + pair.Item1) + let summed named = + let acc = Expression.Parameter(typeof, "acc") + let pair = Expression.Parameter(ValidatorTrees.customerPair, "pair") + let func = Expression.Lambda(Expression.Add(acc, ValidatorTrees.item1 pair), acc, pair) + + ValidatorTrees.call + "Aggregate" + [ ValidatorTrees.customerPair; typeof ] + [ ValidatorTrees.customerPairs named; Expression.Constant 0; ValidatorTrees.quote func ] + + Assert.Equal(unnamedRead, reported (summed false)) + Assert.Empty(reported (summed true)) + + // The accumulated value has the type of the elements here, and a member read from it is no read from an element + let counted = + let acc = Expression.Parameter(ValidatorTrees.customerPair, "acc") + let pair = Expression.Parameter(ValidatorTrees.customerPair, "pair") + let seed = ValidatorTrees.pair ValidatorTrees.customerPair [ Expression.Constant 0; Expression.Constant "" ] true + let count = Expression.Add(ValidatorTrees.item1 acc, Expression.Constant 1) + let next = ValidatorTrees.pair ValidatorTrees.customerPair [ count; Expression.Constant "" ] true + + ValidatorTrees.call + "Aggregate" + [ ValidatorTrees.customerPair; ValidatorTrees.customerPair ] + [ ValidatorTrees.customerPairs false; seed :> Expression; ValidatorTrees.quote (Expression.Lambda(next, acc, pair)) ] + + Assert.Empty(reported counted) + + [] + member _.``Both parameters of an Aggregate function with no seed are elements``() = + // customers.Select(c => new Tuple(c.Id, c.Name)).Aggregate((a, b) => new Tuple(a.Item1 + b.Item1, a.Item2)) + let combined named = + let a = Expression.Parameter(ValidatorTrees.customerPair, "a") + let b = Expression.Parameter(ValidatorTrees.customerPair, "b") + let sum = Expression.Add(ValidatorTrees.item1 a, ValidatorTrees.item1 b) + let body = ValidatorTrees.pair ValidatorTrees.customerPair [ sum; ValidatorTrees.item2 a ] true + + ValidatorTrees.call + "Aggregate" + [ ValidatorTrees.customerPair ] + [ ValidatorTrees.customerPairs named; ValidatorTrees.quote (Expression.Lambda(body, a, b)) ] + + Assert.Equal(unnamedRead @ unnamedRead, reported (combined false)) + Assert.Empty(reported (combined true)) + + [] + member _.``An entity that is wrong in a property other than its key does not pass``() = + let fails (build: Customer -> Customer) = + let scenario = + Scenario.value + "payload" + (fun db -> + query { + for c in db.Customers do + select (build c) into p + head + }) + (fun data -> data.Customers.[0]) + + Scenario.failures scenario |> List.exists (fun failure -> failure.Contains("rows: expected", StringComparison.Ordinal)) + + let copy (c: Customer) = Customer(Id = c.Id, Name = c.Name, City = c.City, Age = c.Age, Registered = c.Registered) + + Assert.True(fails (fun c -> Customer(Id = c.Id, Name = "CORRUPTED", City = c.City, Age = -1, Registered = c.Registered))) + Assert.True(fails (fun c -> Customer(Id = c.Id, Name = c.Name, City = c.City, Age = c.Age))) + Assert.False(fails copy) From 5ba518829ca80048f99dafc8320702d44fd0a321 Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Wed, 4 Nov 2026 10:23:54 +0100 Subject: [PATCH 07/19] Spell the check for a wrong row with IndexOf, which net472 has String.Contains(string, StringComparison) does not exist on .NET Framework, so the test project failed to compile for net472 and all the Windows legs of the CI went red. IndexOf with an ordinal comparison does the same on both. Co-Authored-By: Claude Sonnet 5.5 --- .../FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs index 94b9d25edf6..a9cc9d9535f 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs @@ -458,7 +458,7 @@ type ProviderShapeValidatorTests() = }) (fun data -> data.Customers.[0]) - Scenario.failures scenario |> List.exists (fun failure -> failure.Contains("rows: expected", StringComparison.Ordinal)) + Scenario.failures scenario |> List.exists (fun failure -> failure.IndexOf("rows: expected", StringComparison.Ordinal) >= 0) let copy (c: Customer) = Customer(Id = c.Id, Name = c.Name, City = c.City, Age = c.Age, Registered = c.Registered) From f7f6b4d7a6683b1c63340a10b5d18baa6820f840 Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Wed, 4 Nov 2026 10:39:14 +0100 Subject: [PATCH 08/19] Render the properties of an entity in the order of their names Reflection promises no order of the properties of a class. On .NET Framework one scenario listed the properties of the first order it rendered in another order than those of the three after it, so the expected and the actual rows of the same orders differed as text. The names decide the order now. Co-Authored-By: Claude Sonnet 5.5 --- .../FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs index 9ae260a7226..51d51d47696 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs @@ -83,6 +83,8 @@ module Scenario = ValueSome( ty.GetProperties(BindingFlags.Public ||| BindingFlags.Instance) |> Seq.filter (fun property -> property.GetIndexParameters().Length = 0 && isPlain property.PropertyType) + // Reflection promises no order of the properties, and on .NET Framework two calls for one class gave two orders + |> Seq.sortBy (fun property -> property.Name) |> Seq.map (fun property -> struct (property.Name, property.GetValue value)) |> Seq.toList ) From 4c0ed9ffbc9a90b7e3c4591c0fc87e0ef1ccd770 Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sun, 27 Sep 2026 20:24:48 +0200 Subject: [PATCH 09/19] Replace lets by their values in lambdas handed to a LINQ provider A record or anonymous record with fields written out of order, a tupled lambda parameter, a struct copy such as x.Nullable.HasValue on a class property, or an explicit let: each of these is a let in the quotation, and the converter turned it into an Expression.Block with a local variable. No LINQ provider translates a block, so EF Core and Cosmos failed on all of these. Inside a lambda that goes to a provider - a query operator over IQueryable, or an F# lambda passed as Expression> - the value now takes the place of the variable where that computes the same: - the value only reads: a constant, a parameter of the lambda or a member of one, or what a member or method of a string, number or date makes of those. Such a read can be made at every use; - or the variable is read once, as the first thing the rest evaluates, so that the value is computed at the same moment as before. Any other let keeps its block and is evaluated once where it is written: a call read twice, read by a nested lambda or read after something else has run, an object that is created, a struct of the user's own or a boxed value, a mutable variable, and every let of a lambda that assigns or runs a statement. So does everything EvaluateQuotation and QuotationToExpression convert outside such a lambda. The lets the compiler adds for record fields written out of order are left out as a case of their own, in lambdas and, before the query translator looks at the record, in a select or join over IQueryable. The fields are then evaluated in declaration order, which is the one thing a lambda with side effects in its fields can notice. A join whose key is such a record on one side and comes ready-made from a function on the other keeps the key as written on both sides: the two no longer end up with different types. Fixes #11131 Fixes #15648 Fixes #11332 Co-Authored-By: Claude Opus 5.5 --- docs/release-notes/.FSharp.Core/11.0.200.md | 3 + .../release-notes/PENDING_BREAKING_CHANGES.md | 17 + src/FSharp.Core/Linq.fs | 248 +++++++++++- src/FSharp.Core/Linq.fsi | 4 + src/FSharp.Core/Query.fs | 52 ++- .../FSharp.Core.UnitTests.fsproj | 1 + .../ExpressionLambdaConversionTests.fs | 363 ++++++++++++++++++ .../QueryKeyScenarios.fs | 70 +++- .../QueryMethodSyntaxScenarios.fs | 43 ++- .../QueryOperatorScenarios.fs | 15 + .../QueryProjectionScenarios.fs | 38 +- 11 files changed, 799 insertions(+), 55 deletions(-) create mode 100644 tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs diff --git a/docs/release-notes/.FSharp.Core/11.0.200.md b/docs/release-notes/.FSharp.Core/11.0.200.md index 74a8bab02a3..9f12e38f378 100644 --- a/docs/release-notes/.FSharp.Core/11.0.200.md +++ b/docs/release-notes/.FSharp.Core/11.0.200.md @@ -7,5 +7,8 @@ * Correct `FSharpValue.PreComputeTuplePropertyInfo` to report that struct tuples store elements in fields, not properties, and recommend supported tuple readers instead of reporting a valid index as out of range. ([Issue #20801](https://github.com/dotnet/fsharp/issues/20801), [PR #20815](https://github.com/dotnet/fsharp/pull/20815)) * Quotation debug text from `Expr.ToString()`, `Expr.ToString(full)`, and `%A` now prints struct tuple construction as `NewStructTuple`; reference tuples remain `NewTuple`. ([Issue #20800](https://github.com/dotnet/fsharp/issues/20800), [PR #20814](https://github.com/dotnet/fsharp/pull/20814)) * Fix struct tuples in query expressions: selecting one no longer fails with "unreachable", and the items of one that a later operator reads are taken from its fields instead of failing with `ArgumentException: Struct tuples store their elements in fields, not properties`. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) +* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) +* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332)) +* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522)) ### Changed diff --git a/docs/release-notes/PENDING_BREAKING_CHANGES.md b/docs/release-notes/PENDING_BREAKING_CHANGES.md index df80c994bea..63691df6475 100644 --- a/docs/release-notes/PENDING_BREAKING_CHANGES.md +++ b/docs/release-notes/PENDING_BREAKING_CHANGES.md @@ -38,6 +38,23 @@ a.Equals(b) // Was: false, Now: true // New: if (node.NodeType == ExpressionType.Block) ``` +Inside a lambda handed to a LINQ provider (a query operator over `IQueryable`, or an F# lambda passed as `Expression>`) a let no longer makes a `Block` where leaving it out computes the same: + +- its value only reads: a constant, the lambda's parameter or a member of it (`let age = p.Age`), or what a member or method of a string, number or date makes of those (`let upper = p.Name.ToUpper()`); +- or the variable is read once, as the first thing the rest of the lambda evaluates (`let orders = p.Orders.Count() in orders > 2`). + +This covers the lets the compiler introduces for tupled lambda parameters and struct copies. For record and anonymous record fields written out of order, the lets are left out as well: + +```fsharp +db.People.Select(fun p -> {| Name = p.Name; Age = p.Age |}) +// Was: p => { var Name = p.Name; new <>f__AnonymousType(p.Age, Name) } +// Now: p => new <>f__AnonymousType(p.Age, p.Name) +``` + +**Impact:** When such a lambda is compiled and run, the fields of a record written out of order are evaluated in declaration order instead of the order they are written in, which shows when they have side effects. A let that only reads, such as `let age = p.Age` or `let upper = p.Name.ToUpper()`, is computed where the variable is used: once for each use, at the moment of that use, and not at all on a path that does not use it. + +Every other let keeps its `Block` and is evaluated once, where it is written: a value that is computed by a call and read more than once, by a nested lambda or after something else has run, a struct of your own or a boxed value, a `mutable` variable, and every let of a lambda that assigns, sets a property or field, or runs a statement. `EvaluateQuotation` and `QuotationToExpression` are not affected. + ### Array access uses ArrayIndex ```fsharp diff --git a/src/FSharp.Core/Linq.fs b/src/FSharp.Core/Linq.fs index 431f6fd9514..54a0aafe589 100644 --- a/src/FSharp.Core/Linq.fs +++ b/src/FSharp.Core/Linq.fs @@ -31,7 +31,15 @@ module LeafExpressionConverter = [] type ConvEnv = - { varEnv : Map } + { varEnv : Map + /// Inside a lambda that a LINQ provider receives as an expression tree, which, like a C# expression + /// lambda, must not contain statements. + inExpressionLambda : bool + /// That lambda neither assigns nor runs a statement. Only then can a 'let' in it give way to its + /// value: nothing is written between the place of the 'let' and the place of a read. + substituteLets : bool + /// The variables of the lets around, as opposed to the parameters of the lambdas around + letVariables : ParameterExpression list } let asExpr x = (x :> Expression) let isNamedType(typ:Type) = not (typ.IsArray || typ.IsByRef || typ.IsPointer) @@ -431,6 +439,183 @@ module LeafExpressionConverter = | NewObject(c, [arg]) when equivHeadTypes c.DeclaringType typeof> -> Some arg | _ -> None + /// Whether the expression assigns or runs statements, which no provider translates + let rec hasStatements (expr: Expr) = + match expr with + | VarSet _ + | PropertySet _ + | FieldSet _ + | AddressSet _ + | Sequential _ + | WhileLoop _ + | ForIntegerRangeLoop _ + | TryWith _ + | TryFinally _ -> true + | ExprShape.ShapeVar _ -> false + | ExprShape.ShapeLambda (_, body) -> hasStatements body + | ExprShape.ShapeCombination (_, args) -> List.exists hasStatements args + + let rec countUses (v: Var) (expr: Expr) = + match expr with + | ExprShape.ShapeVar used -> if used = v then 1 else 0 + | ExprShape.ShapeLambda (_, body) -> countUses v body + | ExprShape.ShapeCombination (_, args) -> List.sumBy (countUses v) args + + /// The record that a chain of lets leads to + let rec (|NewRecordAfterLets|_|) (expr: Expr) = + match expr with + | Let (_, _, NewRecordAfterLets args) -> Some args + | NewRecord (_, args) -> Some args + | _ -> None + + /// The compiler evaluates record fields in the order they are written, and binds the ones written before + /// their place to variables: 'let v = e in .. NewRecord (.., v, ..)'. This is such a 'let', and its body with + /// the value in the place of the variable, which evaluates the fields in the order they are declared. + let (|HoistedRecordField|_|) (enabled: bool) (expr: Expr) = + match expr with + | Let (v, e, body) when enabled && not v.IsMutable -> + match body with + | NewRecordAfterLets args + when args |> List.exists (function Var arg -> arg = v | _ -> false) + && countUses v body = 1 + && not (hasStatements expr) -> + Some (body.Substitute (fun used -> if used = v then Some e else None)) + | _ -> None + | _ -> None + + /// A conversion of a reference to a type its object already has, which runs no code, cannot fail and gives + /// the same object. Boxing does not: each box is an object of its own. + let isUpcast (conversion: UnaryExpression) = + match conversion.Method with + | null -> + (conversion.NodeType = ExpressionType.Convert || conversion.NodeType = ExpressionType.TypeAs) + && not conversion.Operand.Type.IsValueType + && conversion.Type.IsAssignableFrom conversion.Operand.Type + | _ -> false + + /// A type whose values cannot change and have no identity, so that its methods give an equal result for + /// equal arguments every time: a number, a string, a date and the like + let isPlainValueType (ty: Type) = + ty.IsPrimitive + || ty.IsEnum + || Type.op_Equality (ty, typeof) + || Type.op_Equality (ty, typeof) + || Type.op_Equality (ty, typeof) + || Type.op_Equality (ty, typeof) + || Type.op_Equality (ty, typeof) + || Type.op_Equality (ty, typeof) + || (ty.IsGenericType + && Type.op_Equality (ty.GetGenericTypeDefinition (), typedefof>) + && ty.GetGenericArguments().[0].IsValueType) + + /// A method of a plain value that gives a plain value, like 'ToUpper' of a string + let isPlainValueMethod (call: MethodCallExpression) = + match call.Object with + | null -> false + | _ -> isPlainValueType call.Method.DeclaringType && isPlainValueType call.Type + + /// Whether the expression reads the data a lambda is given: a parameter of a lambda in the tree, or a member + /// of one. The variable of a 'let' is no such parameter. + let rec isElementRead (letVariables: ParameterExpression list) (expr: Expression) = + match expr with + | :? ParameterExpression as parameter -> + not (letVariables |> List.exists (fun variable -> obj.ReferenceEquals (variable, parameter))) + | :? MemberExpression as read -> + match read.Expression with + | null -> false + // A struct can be changed in place by a call on it: its members are no stable reads + | source -> (not source.Type.IsValueType || isPlainValueType source.Type) && isElementRead letVariables source + | :? UnaryExpression as conversion -> isUpcast conversion && isElementRead letVariables conversion.Operand + | _ -> false + + /// Whether the expression only reads: a constant, the data a lambda is given, or what a member or method of a + /// plain value makes of those. Where a 'let' gives such a read a name, the read can be made wherever the name + /// is used, and as often as it is used. + let rec isRead (letVariables: ParameterExpression list) (expr: Expression) = + match expr with + | :? ConstantExpression -> true + | :? MemberExpression as read when isPlainValueType read.Member.DeclaringType -> + match read.Expression with + | null -> false + | source -> isRead letVariables source + | :? UnaryExpression as conversion when isUpcast conversion -> isRead letVariables conversion.Operand + // The arguments are no calls themselves: a value that mentions another one several times would + // otherwise grow with every 'let' that builds on it. + | :? MethodCallExpression as call -> + isPlainValueMethod call + && isRead letVariables call.Object + && call.Arguments |> Seq.forall (fun argument -> argument :? ConstantExpression || isElementRead letVariables argument) + | _ -> isElementRead letVariables expr + + /// Whether the read of the variable is the first thing that evaluating the expression does, on every path: + /// before it come only constants and reads of other variables. The value of the variable can then be + /// computed at that read instead of before the expression, and nothing can tell the difference. + let readsFirst (variable: ParameterExpression) (expr: Expression) = + // ValueSome true: the variable is read. ValueSome false: nothing ran so far, what comes next may read + // it. ValueNone: something else ran, or may not run at all. + let rec walk (node: Expression) = + match node with + | null -> ValueSome false + | :? ParameterExpression as other -> + if obj.ReferenceEquals (other, variable) then ValueSome true + // What computes the value can change a struct in place, so the struct has to be read after it + elif other.Type.IsValueType && not (isPlainValueType other.Type) then ValueNone + else ValueSome false + | :? ConstantExpression + | :? DefaultExpression + // A lambda only runs later, any number of times + | :? LambdaExpression -> ValueSome false + | :? UnaryExpression as unary -> + if unary.NodeType = ExpressionType.Quote then ValueSome false + elif isUpcast unary then walk unary.Operand + else thenRuns [ unary.Operand ] + | :? BinaryExpression as binary -> + match binary.NodeType with + | ExpressionType.Assign -> walk binary.Right + // The right side runs on some paths only + | ExpressionType.AndAlso + | ExpressionType.OrElse + | ExpressionType.Coalesce -> thenRuns [ binary.Left ] + | _ -> thenRuns [ binary.Left; binary.Right ] + | :? ConditionalExpression as choice -> thenRuns [ choice.Test ] + | :? MemberExpression as read -> thenRuns [ read.Expression ] + | :? TypeBinaryExpression as test -> thenRuns [ test.Expression ] + | :? MethodCallExpression as call -> thenRuns (call.Object :: List.ofSeq call.Arguments) + | :? NewExpression as construction -> thenRuns (List.ofSeq construction.Arguments) + | :? InvocationExpression as invocation -> thenRuns (invocation.Expression :: List.ofSeq invocation.Arguments) + | :? NewArrayExpression as array when array.NodeType = ExpressionType.NewArrayInit -> inOrder (List.ofSeq array.Expressions) + | :? BlockExpression as block -> inOrder (List.ofSeq block.Expressions) + | _ -> ValueNone + + // The parts run one after another + and inOrder (parts: Expression list) = + match parts with + | [] -> ValueSome false + | part :: rest -> + match walk part with + | ValueSome false -> inOrder rest + | reached -> reached + + // The parts run one after another, and then the node itself runs + and thenRuns (parts: Expression list) = + match inOrder parts with + | ValueSome false -> ValueNone + | reached -> reached + + walk expr = ValueSome true + /// Counts the reads of a variable and puts an expression in their place + type VariableReads (variable: ParameterExpression, replacement: Expression) = + inherit ExpressionVisitor () + + member val Count = 0 with get, set + + override this.VisitParameter (node: ParameterExpression) : Expression = + if obj.ReferenceEquals (node, variable) then + this.Count <- this.Count + 1 + replacement + else + node :> Expression + /// Convert F# quotations to LINQ expression trees. /// A more polished LINQ-Quotation translator will be published /// concert with later versions of LINQ. @@ -643,7 +828,9 @@ module LeafExpressionConverter = // Throw away markers inserted to satisfy C#'s design where they pass an argument // or type T to an argument expecting Expression. - | ImplicitExpressionConversionHelperQ (_, GenericArgs [|_|], [x1]) -> ConvExprToLinqInContext env x1 + | ImplicitExpressionConversionHelperQ (_, GenericArgs [|_|], [x1]) -> + let substituteLets = (env.substituteLets || not env.inExpressionLambda) && not (hasStatements x1) + ConvExprToLinqInContext { env with inExpressionLambda = true; substituteLets = substituteLets } x1 // Use witnesses if they are available | CallWithWitnesses (objArgOpt, _, minfo2, witnessArgs, args) -> @@ -743,7 +930,7 @@ module LeafExpressionConverter = | NewDelegate(delegateTy, vs, b) -> let vsP = List.map ConvVarToLinq vs - let env = { varEnv = List.foldBack2 (fun (v:Var) vP -> Map.add v (vP |> asExpr)) vs vsP env.varEnv } + let env = { env with varEnv = List.foldBack2 (fun (v:Var) vP -> Map.add v (vP |> asExpr)) vs vsP env.varEnv } let bodyP = ConvExprToLinqInContext env b Expression.Lambda(delegateTy, bodyP, vsP) |> asExpr @@ -784,6 +971,9 @@ module LeafExpressionConverter = (Expression.NewArrayInit(typeof, [| for fv in fvs -> Expression.Convert(env.varEnv.[fv], typeof) |> asExpr |]) |> asExpr) |]) |> asExpr + | HoistedRecordField env.substituteLets inlined -> + ConvExprToLinqInContext env inlined + | Let (v, e, b) -> // Use Expression.Block to properly scope the variable. This: // 1. Evaluates the expression exactly once (preserving reference semantics) @@ -791,14 +981,50 @@ module LeafExpressionConverter = // 3. Works for both mutable and immutable variables let vP = ConvVarToLinq v let eP = ConvExprToLinqInContext env e - let envinner = { varEnv = Map.add v (vP |> asExpr) env.varEnv } - let bodyP = ConvExprToLinqInContext envinner b - // Create a block with the variable declaration, initial assignment, and body - Expression.Block([| vP |], Expression.Assign(vP, eP), bodyP) |> asExpr + + // No provider translates a block. In a lambda that goes to one, the value takes the place of the + // variable where that computes the same: the value is a mere read, which can be made at every use, + // or the variable is read once, before anything else happens. + let isStructCopy = v.IsMutable && String.Equals (v.Name, "copyOfStruct", StringComparison.Ordinal) + + // A struct of the user's bound by a 'let' is a copy, and what is done to the copy must not reach + // the original, which the value would stand for. + let mayGiveWay = + env.substituteLets + && (not v.IsMutable || isStructCopy) + && (not v.Type.IsValueType || isPlainValueType v.Type || isStructCopy) + && not (Type.op_Equality (eP.Type, typeof)) + + // The Coerce case drops reference upcasts, and uses of the variable may need its own type. + let value () = if Type.op_Equality (eP.Type, v.Type) then eP else Expression.Convert (eP, v.Type) |> asExpr + + if mayGiveWay && isRead env.letVariables eP then + ConvExprToLinqInContext { env with varEnv = Map.add v (value ()) env.varEnv } b + else + let envinner = { env with varEnv = Map.add v (vP |> asExpr) env.varEnv; letVariables = vP :: env.letVariables } + let bodyP = ConvExprToLinqInContext envinner b + + let withoutVariable = + if mayGiveWay then + let reads = VariableReads (vP, value ()) + let replaced = reads.Visit bodyP + + if reads.Count = 1 && readsFirst vP bodyP then + ValueSome replaced + else + ValueNone + else + ValueNone + + match withoutVariable with + | ValueSome replaced -> replaced + | ValueNone -> + // Create a block with the variable declaration, initial assignment, and body + Expression.Block([| vP |], Expression.Assign(vP, eP), bodyP) |> asExpr | Lambda(v, body) -> let vP = ConvVarToLinq v - let env = { varEnv = Map.add v (vP |> asExpr) env.varEnv } + let env = { env with varEnv = Map.add v (vP |> asExpr) env.varEnv } let bodyP = ConvExprToLinqInContext env body let lambdaTy, tyargs = if bodyP.Type = typeof then @@ -946,10 +1172,12 @@ module LeafExpressionConverter = //printf "** Expression .Parameter(%a, %a)\n" output_any ty output_any nm Expression.Parameter(v.Type, v.Name) - let ConvExprToLinq (e: Expr) = ConvExprToLinqInContext { varEnv = Map.empty } e + let ConvExprToLinq (e: Expr) = + ConvExprToLinqInContext { varEnv = Map.empty; inExpressionLambda = false; substituteLets = false; letVariables = [] } e let QuotationToExpression (e: Expr) = ConvExprToLinq e - let QuotationToLambdaExpression (e: Microsoft.FSharp.Quotations.Expr<'T>) = (ConvExprToLinq e) :?> Expression<'T> + let QuotationToLambdaExpression (e: Microsoft.FSharp.Quotations.Expr<'T>) = + ConvExprToLinqInContext { varEnv = Map.empty; inExpressionLambda = true; substituteLets = not (hasStatements e); letVariables = [] } e :?> Expression<'T> // This contorted compilation is used because LINQ's "Compile" is only allowed on lambda expressions, and LINQ // provides no other way to evaluate the expression. diff --git a/src/FSharp.Core/Linq.fsi b/src/FSharp.Core/Linq.fsi index bd72874448c..fcbcdb7221b 100644 --- a/src/FSharp.Core/Linq.fsi +++ b/src/FSharp.Core/Linq.fsi @@ -89,3 +89,7 @@ module LeafExpressionConverter = val internal (|SpecificCallToMethod|_|): RuntimeMethodHandle -> (Expr -> (Expr option * Reflection.MethodInfo * Expr list) option) + + val internal hasStatements: expr: Expr -> bool + + val internal (|HoistedRecordField|_|): enabled: bool -> expr: Expr -> Expr option diff --git a/src/FSharp.Core/Query.fs b/src/FSharp.Core/Query.fs index 56171c629f9..73bf8e3cadf 100644 --- a/src/FSharp.Core/Query.fs +++ b/src/FSharp.Core/Query.fs @@ -443,7 +443,7 @@ module Query = /// Project F# function expressions to Linq LambdaExpression nodes let FuncExprToLinqFunc2Expression (srcTy, targetTy, v, body) = - FuncExprToDelegateExpr(srcTy, targetTy, v, body) |> QuotationToExpression + FuncExprToDelegateExpr(srcTy, targetTy, v, body) |> MakeImplicitExpressionConversion |> QuotationToExpression let FuncExprToLinqFunc2 (srcTy, targetTy, v, body) = FuncExprToDelegateExpr(srcTy, targetTy, v, body) |> EvaluateQuotation @@ -1043,6 +1043,23 @@ module Query = | MacroReduction reduced -> Some (walk reduced) | _ -> None) + /// Without the lets the compiler adds for record fields written out of order, the record reaches + /// ProduceMoreMutables as a record construction. + let rec InlineHoistedFields (p: Expr) = + match p with + | HoistedRecordField true inlined -> InlineHoistedFields inlined + | QuoteTyped _ + | QuoteRaw _ -> p + | ExprShape.ShapeCombination(_, []) -> p + | ExprShape.ShapeCombination(comb, args) -> ExprShape.RebuildShapeCombination(comb, List.map InlineHoistedFields args) + | ExprShape.ShapeLambda(v, body) -> Expr.Lambda(v, InlineHoistedFields body) + | ExprShape.ShapeVar _ -> p + + /// A provider reads the fields of a record in any order, so what goes to one loses the lets that keep the + /// written order. What is evaluated in memory keeps them, and with them the order. + let ProduceMoreMutablesFor isIQ tipf expr = + ProduceMoreMutables tipf (if isIQ && not (hasStatements expr) then InlineHoistedFields expr else expr) + let (|CallQueryBuilderRunQueryable|_|) : Expr -> _ = (|SpecificCallToMethod|_|) (methodhandleof (fun (b :QueryBuilder, v) -> b.Run v)) let (|CallQueryBuilderRunValue|_|) : Expr -> _ = (|SpecificCallToMethod|_|) (methodhandleof (fun (b : QueryBuilder, v: Expr<'a>) -> b.Run v) : 'a) @@ -1306,7 +1323,7 @@ module Query = | CallQueryBuilderYield(_, _, immutSelectorBody) -> let mutSelectorVar, mutSelectorBody = ConvertImmutableConsumerToMutableConsumer sourceConv (immutSelectorVar, MacroExpand immutSelectorBody) - let mutSelectorBody, selectorConv = ProduceMoreMutables TransInnerNoCheck mutSelectorBody + let mutSelectorBody, selectorConv = ProduceMoreMutablesFor (qTyIsIQueryable qTy) TransInnerNoCheck mutSelectorBody let mutSelectorBody = CleanupLeaf mutSelectorBody TransInnerResult.MakeSelect (canElim, qTyIsIQueryable qTy, mutSource, mutSelectorVar, mutSelectorBody), selectorConv @@ -1367,7 +1384,7 @@ module Query = | CallSelect (_, [_; qTy; _], mutSource, Lambda(immutSelectorVar, immutSelectorBody)) -> let mutSource, _sourceConv, mutSelectorVar, mutSelectorBody = TransInnerApplicative check mutSource (immutSelectorVar, immutSelectorBody) - let mutSelectorBody, selectorConv = ProduceMoreMutables TransInnerNoCheck mutSelectorBody + let mutSelectorBody, selectorConv = ProduceMoreMutablesFor (qTyIsIQueryable qTy) TransInnerNoCheck mutSelectorBody let mutSelectorBody = CleanupLeaf mutSelectorBody TransInnerResult.MakeSelect(canElim, qTyIsIQueryable qTy, mutSource, mutSelectorVar, mutSelectorBody), selectorConv @@ -1473,7 +1490,7 @@ module Query = let mutSource, sourceConv = TransInnerAndCommit CanEliminate.Yes check immutSource let mutVar2, mutKeySelector = ConvertImmutableConsumerToMutableConsumer sourceConv (immutVar2, MacroExpand immutKeySelector) let mutVar1, mutElementSelector = ConvertImmutableConsumerToMutableConsumer sourceConv (immutVar1, MacroExpand immutElementSelector) - let mutElementSelector, selectorConv = ProduceMoreMutables TransInnerNoCheck mutElementSelector + let mutElementSelector, selectorConv = ProduceMoreMutablesFor (qTyIsIQueryable qTy) TransInnerNoCheck mutElementSelector let mutElementSelector = CleanupLeaf mutElementSelector let conv = match selectorConv with @@ -1489,11 +1506,11 @@ module Query = LambdasNoDetupling([immutOuterResultGroupVar; immutInnerResultKeyVar], immutElementSelector)]) -> let (mutOuterSource, outerSourceConv, mutInnerSource, innerSourceConv, mutOuterKeyVar: Var, mutOuterKeySelector, mutInnerKeyVar: Var, mutInnerKeySelector: Expr) = - TransJoinInputs check (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) + TransJoinInputs check (qTyIsIQueryable qTy) (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) let mutOuterResultVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer outerSourceConv (immutOuterResultGroupVar, MacroExpand immutElementSelector) let mutInnerResultKeyVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer innerSourceConv (immutInnerResultKeyVar, mutElementSelector) - let mutElementSelector, elementSelectorConv = ProduceMoreMutables TransInnerNoCheck mutElementSelector + let mutElementSelector, elementSelectorConv = ProduceMoreMutablesFor (qTyIsIQueryable qTy) TransInnerNoCheck mutElementSelector let mutElementSelector = CleanupLeaf mutElementSelector let joinExpr = @@ -1513,12 +1530,12 @@ module Query = LambdasNoDetupling([immutOuterResultGroupVar; immutInnerResultGroupVar], immutElementSelector)]) -> let (mutOuterSource, outerSourceConv, mutInnerSource, innerSourceConv, mutOuterKeyVar: Var, mutOuterKeySelector, mutInnerKeyVar: Var, mutInnerKeySelector: Expr) = - TransJoinInputs check (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) + TransJoinInputs check (qTyIsIQueryable qTy) (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) let mutOuterResultGroupVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer outerSourceConv (immutOuterResultGroupVar, MacroExpand immutElementSelector) let innerGroupConv = MakeSeqConv innerSourceConv let mutInnerResultKeyVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer innerGroupConv (immutInnerResultGroupVar, mutElementSelector) - let mutElementSelector, elementSelectorConv = ProduceMoreMutables TransInnerNoCheck mutElementSelector + let mutElementSelector, elementSelectorConv = ProduceMoreMutablesFor (qTyIsIQueryable qTy) TransInnerNoCheck mutElementSelector let mutElementSelector = CleanupLeaf mutElementSelector let joinExpr = @@ -1542,11 +1559,11 @@ module Query = let immutElementSelector = immutElementSelector.Substitute (fun v -> if v = immutInnerResultGroupVar then Some (MakeDefaultIfEmpty ([immutInnerSourceTy], [Expr.Var immutInnerResultGroupVar])) else None) let (mutOuterSource, outerSourceConv, mutInnerSource, innerSourceConv, mutOuterKeyVar: Var, mutOuterKeySelector, mutInnerKeyVar: Var, mutInnerKeySelector: Expr) = - TransJoinInputs check (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) + TransJoinInputs check (qTyIsIQueryable qTy) (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) let mutOuterResultGroupVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer outerSourceConv (immutOuterResultGroupVar, MacroExpand immutElementSelector) let mutInnerResultKeyVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer innerSourceConv (immutInnerResultGroupVar, mutElementSelector) - let mutElementSelector, elementSelectorConv = ProduceMoreMutables TransInnerNoCheck mutElementSelector + let mutElementSelector, elementSelectorConv = ProduceMoreMutablesFor (qTyIsIQueryable qTy) TransInnerNoCheck mutElementSelector let mutElementSelector = CleanupLeaf mutElementSelector let joinExpr = @@ -1638,7 +1655,7 @@ module Query = | _ -> e, NoConv - and TransJoinInputs check (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) = + and TransJoinInputs check isIQ (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) = let mutOuterSource, outerSourceConv = TransInnerAndCommit CanEliminate.Yes check immutOuterSource let mutInnerSource, innerSourceConv = TransInnerAndCommit CanEliminate.Yes check immutInnerSource @@ -1646,9 +1663,18 @@ module Query = let mutInnerKeyVar, mutInnerKeySelector = ConvertImmutableConsumerToMutableConsumer innerSourceConv (immutInnerKeyVar, MacroExpand immutInnerKeySelector) // Keys may be composite tuples - convert them to be mutables. Note, if there is a tuple on one side, there must be a tuple on the other side. - let mutOuterKeySelector, _ = ProduceMoreMutables TransNone mutOuterKeySelector + let mutOuterKey, _ = ProduceMoreMutablesFor isIQ TransNone mutOuterKeySelector + let mutInnerKey, _ = ProduceMoreMutablesFor isIQ TransNone mutInnerKeySelector + + // A key that one side builds and the other gets ready-made changes its type on one side only. Both keep + // the type they were written with then. + let mutOuterKeySelector, mutInnerKeySelector = + if mutOuterKey.Type = mutInnerKey.Type then + mutOuterKey, mutInnerKey + else + mutOuterKeySelector, mutInnerKeySelector + let mutOuterKeySelector = CleanupLeaf mutOuterKeySelector - let mutInnerKeySelector, _ = ProduceMoreMutables TransNone mutInnerKeySelector let mutInnerKeySelector = CleanupLeaf mutInnerKeySelector mutOuterSource, outerSourceConv, mutInnerSource, innerSourceConv, mutOuterKeyVar, mutOuterKeySelector, mutInnerKeyVar, mutInnerKeySelector diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj b/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj index 3a20b60f7e4..8b5a73f6423 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj +++ b/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj @@ -101,6 +101,7 @@ + diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs new file mode 100644 index 00000000000..607b5cb464f --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs @@ -0,0 +1,363 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module FSharp.Core.UnitTests.Linq.ExpressionLambdaConversionTests + +open System +open System.Linq.Expressions +open Xunit +open Microsoft.FSharp.Linq.RuntimeHelpers + +type private Counter() = + let mutable calls = 0 + member _.Calls = calls + + member _.Next() = + calls <- calls + 1 + calls + +let private bodyOf (lambda: Expression<'T>) = lambda.Body + +[] +let ``Anonymous record with fields out of alphabetical order becomes a New in an expression lambda`` () = + let lambda = LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun x -> {| B = x; A = x + 1 |}) @> + Assert.Equal(ExpressionType.New, (bodyOf lambda).NodeType) + +[] +let ``Record with fields out of declaration order becomes a New in an expression lambda`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun c -> { PersonAge = c.Age; PersonName = c.Name }) @> + + Assert.Equal(ExpressionType.New, (bodyOf lambda).NodeType) + +[] +let ``Tupled lambda parameters read the tuple directly in an expression lambda`` () = + let lambda = LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun (id, _) -> id > 1) @> + + match bodyOf lambda with + | :? BinaryExpression as comparison -> + match comparison.Left with + | :? MemberExpression as item -> Assert.Equal("Item1", item.Member.Name) + | left -> Assert.Fail $"Expected a read of Item1, got {left}" + | body -> Assert.Fail $"Expected a comparison, got {body}" + +[] +let ``Assigned mutable variable stays a block in an expression lambda`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func(fun x -> + let mutable y = x + y <- y + 1 + y) + @> + + Assert.Equal(ExpressionType.Block, (bodyOf lambda).NodeType) + Assert.Equal(6, lambda.Compile().Invoke 5) + +[] +let ``EvaluateQuotation still evaluates a let once`` () = + let counter = Counter() + let result = LeafExpressionConverter.EvaluateQuotation <@ let v = counter.Next() in v + v @> + Assert.Equal(2, result :?> int) + Assert.Equal(1, counter.Calls) + +[] +type Tally = + val mutable Count: int + + member this.Add() = + this.Count <- this.Count + 1 + + member this.AddAndCount() = + this.Count <- this.Count + 1 + this.Count + +type TallyHolder() = + [] + val mutable Tally: Tally + +type Reading = { First: int; Second: int } + +let private run (lambda: Expression>) (argument: int) = lambda.Compile().Invoke argument + +[] +let ``Let read after a later assignment keeps the value it was bound to`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func(fun a -> + let mutable y = a + let x = y + 1 + y <- 100 + x + y) + @> + + Assert.Equal(131, run lambda 30) + +[] +let ``Let read after a property is set keeps the value it was bound to`` () = + let view = CustomerView(ViewAge = 1) + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func(fun a -> + let before = view.ViewAge + view.ViewAge <- before + a + before) + @> + + Assert.Equal(1, run lambda 5) + Assert.Equal(6, view.ViewAge) + +[] +let ``Mutable struct changed in place is one value`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func(fun a -> + let mutable tally = Tally() + tally.Add() + tally.Add() + tally.Count + a) + @> + + Assert.Equal(5, run lambda 3) + +[] +let ``Object bound by a let and changed is one object`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func(fun a -> + let items = Collections.Generic.List() + items.Add a + items.Count) + @> + + Assert.Equal(1, run lambda 7) + +[] +let ``Object bound by a let and read twice is one object`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun _ -> let o = obj () in obj.ReferenceEquals(o, o)) @> + + Assert.True(run lambda 0) + +[] +let ``Let that is never read is still evaluated`` () = + let counter = Counter() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun a -> let unused = counter.Next() in a) @> + + Assert.Equal(4, run lambda 4) + Assert.Equal(1, counter.Calls) + +[] +let ``Record fields written out of order are evaluated as written in a query over a sequence`` () = + let counter = Counter() + + let first = + LeafExpressionConverter.EvaluateQuotation + <@ query { for _ in [ 1; 2 ] do select { Second = counter.Next(); First = counter.Next() }; head } @> + :?> Reading + + Assert.Equal({ First = 2; Second = 1 }, first) + +type private BlockFinder() = + inherit ExpressionVisitor() + + member val Found = false with get, set + + override this.VisitBlock node = + this.Found <- true + base.VisitBlock node + +let private hasBlock (expression: Expression) = + let finder = BlockFinder() + finder.Visit expression |> ignore + finder.Found + +let addTo (items: Collections.Generic.List) (item: int) = + items.Add item + 1 + +[] +let ``Value of a let that a lambda reads is computed once however often the lambda runs`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func(fun a -> + let seen = Collections.Generic.HashSet() + Seq.length (Seq.filter (fun x -> seen.Add x) [| a; a; a + 1 |])) + @> + + Assert.Equal(2, run lambda 5) + +[] +let ``Call bound by a let and read twice runs once`` () = + let counter = Counter() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun a -> let v = counter.Next() in v + v + a) @> + + Assert.Equal(2, run lambda 0) + Assert.Equal(1, counter.Calls) + +[] +let ``Object behind a cast is one object when read twice`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func(fun _ -> + let items = Collections.Generic.List() :> Collections.Generic.IList + obj.ReferenceEquals(items, items)) + @> + + Assert.True(run lambda 0) + +[] +let ``Object reached through a second let is one object`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func(fun a -> + let made = Collections.Generic.List() + let items = made + addTo items a + items.Count) + @> + + Assert.Equal(2, run lambda 7) + +[] +let ``Member of an object read before a call keeps the value it had`` () = + let items = Collections.Generic.List() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun a -> let before = items.Count in addTo items a + before) @> + + Assert.Equal(1, run lambda 7) + +[] +let ``Statement anywhere in the lambda keeps every let where it is written`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func(fun view -> + let before = view.ViewAge + view.ViewAge <- before + 10 + before) + @> + + let view = CustomerView(ViewAge = 1) + Assert.Equal(1, lambda.Compile().Invoke view) + Assert.Equal(11, view.ViewAge) + +[] +let ``Enumerator bound by a let is one enumerator`` () = + let items = Collections.Generic.List([ 10; 20 ]) + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func(fun a -> + let position = (items :> seq).GetEnumerator() + if position.MoveNext() then position.Current + a else -1) + @> + + Assert.Equal(10, run lambda 0) + +[] +let ``Value built on a value that is read twice does not multiply`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func(fun a -> + let b = a + a + let c = b + b + let d = c + c + let e = d + d + let f = e + e + let g = f + f + let h = g + g + let i = h + h + i + i) + @> + + Assert.Equal(512, run lambda 1) + Assert.True((bodyOf lambda).ToString().Length < 400) + +[] +let ``Method of a string bound by a let is called at each read`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun s -> let upper = s.ToUpper() in upper + upper) @> + + Assert.False(hasBlock lambda) + Assert.Equal("ABAB", lambda.Compile().Invoke "ab") + +[] +let ``Call bound by a let and read once is made at that read`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun c -> let orders = Seq.length c.Orders in orders > c.Age + 1) @> + + Assert.False(hasBlock lambda) + +[] +let ``Record fields are not reordered around a statement in a query over a queryable`` () = + let source = System.Linq.Queryable.AsQueryable [ 1; 2 ] + + let rows = + query { + for x in source do + select ( + let mutable y = x + let second = (y <- y * 10; y) + { First = y; Second = second } + ) + } + |> Seq.toList + + Assert.Equal([ { First = 10; Second = 10 }; { First = 20; Second = 20 } ], rows) +[] +let ``Call bound by a let runs before a member that is read ahead of the variable`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func, int>(fun items -> let added = addTo items 5 in items.Count + added) @> + + Assert.Equal(2, lambda.Compile().Invoke(Collections.Generic.List())) + +[] +let ``Struct bound by a let is a copy`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun holder -> let tally = holder.Tally in tally.AddAndCount()) @> + + let holder = TallyHolder() + Assert.Equal(1, lambda.Compile().Invoke holder) + Assert.Equal(0, holder.Tally.Count) + +[] +let ``Boxed value bound by a let is one object`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun a -> let boxed = (a :> obj) in obj.ReferenceEquals(boxed, boxed)) @> + + Assert.True(run lambda 7) + +[] +let ``Record fields are not reordered next to a statement in a query over a queryable`` () = + let counter = Counter() + let view = CustomerView(ViewAge = 1) + let source = System.Linq.Queryable.AsQueryable [ 1 ] + + let rows = + query { + for x in source do + select ( + view.ViewAge <- x + { Second = counter.Next(); First = counter.Next() } + ) + } + |> Seq.toList + + Assert.Equal([ { First = 2; Second = 1 } ], rows) diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs index 0c7e32749cb..7f48a308ee6 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs @@ -3,15 +3,8 @@ module FSharp.Core.UnitTests.Linq.QueryKeyScenarios open System.Linq -open System.Linq.Expressions open Microsoft.FSharp.Linq.RuntimeHelpers -let private block = [ Violation.Statement ExpressionType.Block ] - -/// A tree is pinned with every block it has: one that goes from three to two is a change as well -let private blocks count = - List.replicate count (Violation.Statement ExpressionType.Block) - let private customers (data: Data) = Seq.ofList data.Customers /// Orders paired with the lines that share both the order number and, as quantity, the customer number. @@ -31,6 +24,12 @@ let private matchingLinesOrNone (data: Data) = | matches -> yield! matches } +/// The key of a line as the join scenarios spell it, made by a function instead of written in the query. +let lineKey (l: OrderLine) = {| Units = l.Quantity; Order = l.OrderId |} + +let private matchingLineCounts (data: Data) = + seq { for o in data.Orders -> o.Id, matchingLines data |> Seq.filter (fun (order, _) -> order = o.Id) |> Seq.length } + let private joins () = [ Scenario.rows "join on a scalar" @@ -72,7 +71,6 @@ let private joins () = select (o.Id, l.Id) }) matchingLines - |> Scenario.violating (blocks 2) Scenario.rows "join on an anonymous record, fields in alphabetical order" @@ -93,8 +91,16 @@ let private joins () = select (o.Id, l.Id) }) matchingLines - |> Scenario.violating (blocks 2) + Scenario.rows + "join on an anonymous record that one side gets from a function" + (fun db -> + query { + for o in db.Orders do + join l in db.Lines on ({| Units = o.CustomerId; Order = o.Id |} = lineKey l) + select (o.Id, l.Id) + }) + matchingLines Scenario.rows "groupJoin on a tuple" (fun db -> @@ -103,7 +109,27 @@ let private joins () = groupJoin l in db.Lines on ((o.Id, o.CustomerId) = (l.OrderId, l.Quantity)) into lines select (o.Id, lines.Count()) }) - (fun data -> seq { for o in data.Orders -> o.Id, matchingLines data |> Seq.filter (fun (order, _) -> order = o.Id) |> Seq.length }) + matchingLineCounts + + Scenario.rows + "groupJoin on a record, fields out of declaration order" + (fun db -> + query { + for o in db.Orders do + groupJoin l in db.Lines on ({ Units = o.CustomerId; OrderNumber = o.Id } = { Units = l.Quantity; OrderNumber = l.OrderId }) into lines + select (o.Id, lines.Count()) + }) + matchingLineCounts + + Scenario.rows + "groupJoin on an anonymous record, fields out of alphabetical order" + (fun db -> + query { + for o in db.Orders do + groupJoin l in db.Lines on ({| Units = o.CustomerId; Order = o.Id |} = {| Units = l.Quantity; Order = l.OrderId |}) into lines + select (o.Id, lines.Count()) + }) + matchingLineCounts Scenario.rows "leftOuterJoin on a tuple" @@ -114,6 +140,28 @@ let private joins () = for l in lines do select (o.Id, (if obj.ReferenceEquals(l, null) then -1 else l.Id)) }) + matchingLinesOrNone + + Scenario.rows + "leftOuterJoin on a record, fields out of declaration order" + (fun db -> + query { + for o in db.Orders do + leftOuterJoin l in db.Lines on ({ Units = o.CustomerId; OrderNumber = o.Id } = { Units = l.Quantity; OrderNumber = l.OrderId }) into lines + for l in lines do + select (o.Id, (if obj.ReferenceEquals(l, null) then -1 else l.Id)) + }) + matchingLinesOrNone + + Scenario.rows + "leftOuterJoin on an anonymous record, fields out of alphabetical order" + (fun db -> + query { + for o in db.Orders do + leftOuterJoin l in db.Lines on ({| Units = o.CustomerId; Order = o.Id |} = {| Units = l.Quantity; Order = l.OrderId |}) into lines + for l in lines do + select (o.Id, (if obj.ReferenceEquals(l, null) then -1 else l.Id)) + }) matchingLinesOrNone ] let private groupings () = @@ -152,7 +200,6 @@ let private groupings () = "groupBy a record, fields out of declaration order" (fun db -> query { for c in db.Customers do groupBy { KeyAge = c.Age / 10; KeyCity = c.City } into g; select (g.Key, g.Count()) }) (customers >> Seq.countBy (fun c -> { KeyCity = c.City; KeyAge = c.Age / 10 })) - |> Scenario.violating block Scenario.rows "groupBy an anonymous record, fields in alphabetical order" @@ -163,7 +210,6 @@ let private groupings () = "groupBy an anonymous record, fields out of alphabetical order" (fun db -> query { for c in db.Customers do groupBy {| Decade = c.Age / 10; City = c.City |} into g; select (g.Key, g.Count()) }) (customers >> Seq.countBy (fun c -> {| City = c.City; Decade = c.Age / 10 |})) - |> Scenario.violating block Scenario.rows "groupValBy a tuple of values" diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs index 760c9b78a49..18aa3c9e240 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs @@ -37,7 +37,6 @@ let cases () = let withLargeOrders = db.Orders.Where(fun o -> let total = o.Total in total > 4.0).Select(fun o -> o.CustomerId) db.Customers.Where(fun c -> withLargeOrders.Contains c.Id).Select(fun c -> c.Name)) (customers >> Seq.filter (fun c -> c.Orders |> Seq.exists (fun o -> o.Total > 4.0)) >> Seq.map _.Name) - |> Scenario.violating block Scenario.rows "Select a record, fields in declaration order" @@ -48,7 +47,6 @@ let cases () = "Select a record, fields out of declaration order" (fun db -> db.Customers.Select(fun c -> { PersonAge = c.Age; PersonName = c.Name })) (customers >> Seq.map (fun c -> { PersonName = c.Name; PersonAge = c.Age })) - |> Scenario.violating block Scenario.rows "Select an anonymous record, fields in alphabetical order" @@ -59,25 +57,53 @@ let cases () = "Select an anonymous record, fields out of alphabetical order" (fun db -> db.Customers.Select(fun c -> {| Name = c.Name; Age = c.Age |})) (customers >> Seq.map (fun c -> {| Age = c.Age; Name = c.Name |})) - |> Scenario.violating block + + Scenario.rows + "Select an anonymous record, method calls in fields out of order" + (fun db -> db.Customers.Select(fun c -> {| Upper = c.Name.ToUpper(); City = c.City.Trim() |})) + (customers >> Seq.map (fun c -> {| City = c.City.Trim(); Upper = c.Name.ToUpper() |})) + + Scenario.rows + "Select a struct anonymous record, fields out of alphabetical order" + (fun db -> db.Customers.Select(fun c -> struct {| Name = c.Name; Age = c.Age |})) + (customers >> Seq.map (fun c -> struct {| Age = c.Age; Name = c.Name |})) Scenario.rows "Select with a let used twice" (fun db -> db.Customers.Select(fun c -> let upper = c.Name.ToUpper() in upper + upper)) (customers >> Seq.map (fun c -> c.Name.ToUpper() + c.Name.ToUpper())) + + Scenario.rows + "Select with a count in a let used twice" + (fun db -> db.Customers.Select(fun c -> let orders = c.Orders.Count in orders + orders)) + (customers >> Seq.map (fun c -> c.Orders.Count * 2)) + + Scenario.rows + "Select with a call in a let used twice" + (fun db -> db.Customers.Select(fun c -> let orders = c.Orders.Count() in orders + orders)) + (customers >> Seq.map (fun c -> c.Orders.Count * 2)) |> Scenario.violating block + Scenario.rows + "Select with nested lets" + (fun db -> db.Customers.Select(fun c -> let age = c.Age in let next = age + 1 in next * 2)) + (customers >> Seq.map (fun c -> (c.Age + 1) * 2)) + + Scenario.rows + "Select on a nullable property" + (fun db -> db.Customers.Select(fun c -> c.Registered.HasValue)) + (customers >> Seq.map _.Registered.HasValue) Scenario.rows "Where taking a tuple apart" (fun db -> db.Customers.Select(fun c -> (c.Id, c.Name)).Where(fun (id, _) -> id > 1)) (customers >> Seq.map (fun c -> c.Id, c.Name) >> Seq.filter (fun (id, _) -> id > 1)) - |> Scenario.violating (reading "Tuple`2" @ blocks 2) + |> Scenario.violating (reading "Tuple`2") Scenario.rows "Where taking a struct tuple apart" (fun db -> db.Customers.Select(fun c -> struct (c.Id, c.Name)).Where(fun (struct (id, _)) -> id > 1)) (customers >> Seq.map (fun c -> struct (c.Id, c.Name)) >> Seq.filter (fun (struct (id, _)) -> id > 1)) - |> Scenario.violating (reading "ValueTuple`2" @ block) + |> Scenario.violating (reading "ValueTuple`2") Scenario.rows "Where taking the last of nine apart" @@ -87,7 +113,7 @@ let cases () = .Where(fun (_, _, _, _, _, _, _, _, last) -> last > 1) .Select(fun (_, name, _, _, _, _, _, _, _) -> name)) (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) - |> Scenario.violating (reading "Tuple`8" @ reading "Tuple`8" @ blocks 18) + |> Scenario.violating (reading "Tuple`8" @ reading "Tuple`8") Scenario.rows "Where taking the last of a struct nine apart" @@ -97,19 +123,18 @@ let cases () = .Where(fun (struct (_, _, _, _, _, _, _, _, last)) -> last > 1) .Select(fun (struct (_, name, _, _, _, _, _, _, _)) -> name)) (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) - |> Scenario.violating (reading "ValueTuple`8" @ reading "ValueTuple`8" @ blocks 2) + |> Scenario.violating (reading "ValueTuple`8" @ reading "ValueTuple`8") Scenario.rows "Where over a query that selects a tuple" (fun db -> (query { for c in db.Customers do select (c.Id, c.Name) }).Where(fun (id, _) -> id > 1)) (customers >> Seq.map (fun c -> c.Id, c.Name) >> Seq.filter (fun (id, _) -> id > 1)) - |> Scenario.violating (reading "Tuple`2" @ blocks 2) + |> Scenario.violating (reading "Tuple`2") Scenario.rows "Where on a nullable property" (fun db -> db.Customers.Where(fun c -> c.Registered.HasValue && c.Registered.Value.Year > 2020)) (customers >> Seq.filter (fun c -> c.Registered.HasValue && c.Registered.Value.Year > 2020)) - |> Scenario.violating (blocks 3) Scenario.rows "Where with a lambda over another source" diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryOperatorScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryOperatorScenarios.fs index 6b2339a996b..844eddba7b9 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryOperatorScenarios.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryOperatorScenarios.fs @@ -38,6 +38,11 @@ let private sequences () = (fun db -> query { for l in db.Lines do where (l.Discount ?> 1); select l.Id }) (lines >> Seq.filter (fun l -> l.Discount ?> 1) >> Seq.map _.Id) + Scenario.rows + "where with a let" + (fun db -> query { for c in db.Customers do where (let age = c.Age in age > 26 && age < 35); select c.Name }) + (customers >> Seq.filter (fun c -> c.Age > 26 && c.Age < 35) >> Seq.map _.Name) + Scenario.rows "if then" (fun db -> query { for c in db.Customers do if c.Age > 26 then yield c.Name }) @@ -131,6 +136,16 @@ let private values () = Scenario.value "exists" (fun db -> query { for c in db.Customers do exists (c.City = "Paris") }) (fun _ -> true) + Scenario.value + "exists on a nullable property" + (fun db -> query { for c in db.Customers do exists (c.Registered.HasValue && c.Registered.Value.Year > 2020) }) + (customers >> Seq.exists (fun c -> c.Registered.HasValue && c.Registered.Value.Year > 2020)) + + Scenario.value + "exists with a let" + (fun db -> query { for c in db.Customers do exists (let city = c.City in city = "Paris" || city = "Rome") }) + (customers >> Seq.exists (fun c -> c.City = "Paris" || c.City = "Rome")) + Scenario.value "all" (fun db -> query { for c in db.Customers do all (c.Age > 26) }) (fun _ -> false) Scenario.value "contains" (fun db -> query { for c in db.Customers do select c.Id; contains 2 }) (fun _ -> true) diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs index 8c1e684614e..b0c65d7aabf 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs @@ -2,10 +2,6 @@ module FSharp.Core.UnitTests.Linq.QueryProjectionScenarios -open System.Linq.Expressions - -let private block = [ Violation.Statement ExpressionType.Block ] - let private customers (data: Data) = Seq.ofList data.Customers let cases () = @@ -18,6 +14,11 @@ let cases () = (fun db -> query { for c in db.Customers do select (c.Name + "!", c.Age + 1, (if c.Age > 28 then "senior" else "junior")) }) (customers >> Seq.map (fun c -> c.Name + "!", c.Age + 1, (if c.Age > 28 then "senior" else "junior"))) + Scenario.rows + "let in select" + (fun db -> query { for c in db.Customers do select (let upper = c.Name.ToUpper() in upper + "!") }) + (customers >> Seq.map (fun c -> c.Name.ToUpper() + "!")) + Scenario.rows "tuple of two" (fun db -> query { for c in db.Customers do select (c.Id, c.Name) }) @@ -95,13 +96,22 @@ let cases () = "record, fields out of declaration order" (fun db -> query { for c in db.Customers do select { PersonAge = c.Age; PersonName = c.Name } }) (customers >> Seq.map (fun c -> { PersonName = c.Name; PersonAge = c.Age })) - |> Scenario.violating block + + Scenario.rows + "record, fields out of declaration order, read by the next where" + (fun db -> + query { + for c in db.Customers do + select { PersonAge = c.Age; PersonName = c.Name } into person + where (person.PersonAge > 26) + select person.PersonName + }) + (customers >> Seq.filter (fun c -> c.Age > 26) >> Seq.map _.Name) Scenario.rows "record of three, fields rotated" (fun db -> query { for c in db.Customers do select { Town = c.City; FullName = c.Name; Years = c.Age } }) (customers >> Seq.map (fun c -> { FullName = c.Name; Years = c.Age; Town = c.City })) - |> Scenario.violating block Scenario.rows "record holding a tuple" @@ -117,19 +127,27 @@ let cases () = "anonymous record, fields out of alphabetical order" (fun db -> query { for c in db.Customers do select {| Name = c.Name; Age = c.Age |} }) (customers >> Seq.map (fun c -> {| Age = c.Age; Name = c.Name |})) - |> Scenario.violating block + + Scenario.rows + "anonymous record, fields out of alphabetical order, read by the next where" + (fun db -> + query { + for c in db.Customers do + select {| Name = c.Name; Age = c.Age |} into person + where (person.Age > 26) + select person.Name + }) + (customers >> Seq.filter (fun c -> c.Age > 26) >> Seq.map _.Name) Scenario.rows "anonymous record, method calls in fields out of order" (fun db -> query { for c in db.Customers do select {| Upper = c.Name.ToUpper(); City = c.City.Trim() |} }) (customers >> Seq.map (fun c -> {| City = c.City.Trim(); Upper = c.Name.ToUpper() |})) - |> Scenario.violating block Scenario.rows "anonymous record inside an anonymous record" (fun db -> query { for c in db.Customers do select {| Person = {| Name = c.Name; Age = c.Age |} |} }) (customers >> Seq.map (fun c -> {| Person = {| Age = c.Age; Name = c.Name |} |})) - |> Scenario.violating block Scenario.rows "struct anonymous record, fields in alphabetical order" @@ -140,7 +158,6 @@ let cases () = "struct anonymous record, fields out of alphabetical order" (fun db -> query { for c in db.Customers do select (struct {| Name = c.Name; Age = c.Age |}) }) (customers >> Seq.map (fun c -> struct {| Age = c.Age; Name = c.Name |})) - |> Scenario.violating block Scenario.rows "CLIMutable record, fields in declaration order" @@ -151,7 +168,6 @@ let cases () = "CLIMutable record, fields out of declaration order" (fun db -> query { for c in db.Customers do select { DtoAge = c.Age; DtoName = c.Name } }) (customers >> Seq.map (fun c -> { DtoName = c.Name; DtoAge = c.Age })) - |> Scenario.violating block Scenario.rows "class with property initializers" From 813112d9d9392de31f7140eb3570af302ea08163 Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sun, 1 Nov 2026 22:14:31 +0100 Subject: [PATCH 10/19] Link the release note to the pull request Co-Authored-By: Claude Sonnet 5.5 --- docs/release-notes/.FSharp.Core/11.0.200.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes/.FSharp.Core/11.0.200.md b/docs/release-notes/.FSharp.Core/11.0.200.md index 9f12e38f378..050664fe9bc 100644 --- a/docs/release-notes/.FSharp.Core/11.0.200.md +++ b/docs/release-notes/.FSharp.Core/11.0.200.md @@ -8,7 +8,7 @@ * Quotation debug text from `Expr.ToString()`, `Expr.ToString(full)`, and `%A` now prints struct tuple construction as `NewStructTuple`; reference tuples remain `NewTuple`. ([Issue #20800](https://github.com/dotnet/fsharp/issues/20800), [PR #20814](https://github.com/dotnet/fsharp/pull/20814)) * Fix struct tuples in query expressions: selecting one no longer fails with "unreachable", and the items of one that a later operator reads are taken from its fields instead of failing with `ArgumentException: Struct tuples store their elements in fields, not properties`. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) * Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) -* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332)) +* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) * Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522)) ### Changed From 8dd6cdff4125923da3bdeaa2d407e92f36300c97 Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sun, 1 Nov 2026 22:17:54 +0100 Subject: [PATCH 11/19] Remove the duplicated release note line Co-Authored-By: Claude Sonnet 5.5 --- docs/release-notes/.FSharp.Core/11.0.200.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/release-notes/.FSharp.Core/11.0.200.md b/docs/release-notes/.FSharp.Core/11.0.200.md index 050664fe9bc..3994b58cd15 100644 --- a/docs/release-notes/.FSharp.Core/11.0.200.md +++ b/docs/release-notes/.FSharp.Core/11.0.200.md @@ -9,6 +9,5 @@ * Fix struct tuples in query expressions: selecting one no longer fails with "unreachable", and the items of one that a later operator reads are taken from its fields instead of failing with `ArgumentException: Struct tuples store their elements in fields, not properties`. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) * Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) * Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) -* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522)) ### Changed From 294558625f3c4e2711585e4cd1b793ab2499343d Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sun, 1 Nov 2026 22:56:17 +0100 Subject: [PATCH 12/19] Tell the lets of the compiler from the lets of the code A let is a record field that the compiler moved out of place only when its variable is named like the field whose place it takes. A let of the code with another name is evaluated first, like any other, even if a record follows it. A nullable is a plain value only when the type it holds is one, and the arguments of a method of a plain value must be plain too: a method can run code of the user's on an argument, once for every time it is made. A mutable let named like the copy the compiler makes of a struct gives way only for a plain value, so that a struct of the user's keeps its copy. Co-Authored-By: Claude Sonnet 5.5 --- src/FSharp.Core/Linq.fs | 44 +++++++--- .../ExpressionLambdaConversionTests.fs | 88 +++++++++++++++++++ 2 files changed, 119 insertions(+), 13 deletions(-) diff --git a/src/FSharp.Core/Linq.fs b/src/FSharp.Core/Linq.fs index 54a0aafe589..684f4781374 100644 --- a/src/FSharp.Core/Linq.fs +++ b/src/FSharp.Core/Linq.fs @@ -461,22 +461,35 @@ module LeafExpressionConverter = | ExprShape.ShapeLambda (_, body) -> countUses v body | ExprShape.ShapeCombination (_, args) -> List.sumBy (countUses v) args - /// The record that a chain of lets leads to + /// The record that a chain of lets leads to, with the arguments of its construction let rec (|NewRecordAfterLets|_|) (expr: Expr) = match expr with - | Let (_, _, NewRecordAfterLets args) -> Some args - | NewRecord (_, args) -> Some args + | Let (_, _, NewRecordAfterLets record) -> Some record + | NewRecord (recordType, args) -> Some (recordType, args) | _ -> None /// The compiler evaluates record fields in the order they are written, and binds the ones written before - /// their place to variables: 'let v = e in .. NewRecord (.., v, ..)'. This is such a 'let', and its body with - /// the value in the place of the variable, which evaluates the fields in the order they are declared. + /// their place to variables named after the field: 'let Second = e in .. NewRecord (.., Second, ..)'. + /// Whether the variable is such a one: the argument in the place of a field is the variable named like it. + /// A 'let' of the code with another name evaluates its value first, whatever the record that follows. + let isHoistedField (v: Var) (recordType: Type) (args: Expr list) = + let fields = FSharpType.GetRecordFields (recordType, showAll) + + args + |> List.indexed + |> List.exists (fun (index, arg) -> + match arg with + | Var used -> used = v && index < fields.Length && String.Equals (fields.[index].Name, v.Name, StringComparison.Ordinal) + | _ -> false) + + /// This is such a 'let', and its body with the value in the place of the variable, which evaluates the + /// fields in the order they are declared. let (|HoistedRecordField|_|) (enabled: bool) (expr: Expr) = match expr with | Let (v, e, body) when enabled && not v.IsMutable -> match body with - | NewRecordAfterLets args - when args |> List.exists (function Var arg -> arg = v | _ -> false) + | NewRecordAfterLets (recordType, args) + when isHoistedField v recordType args && countUses v body = 1 && not (hasStatements expr) -> Some (body.Substitute (fun used -> if used = v then Some e else None)) @@ -494,8 +507,8 @@ module LeafExpressionConverter = | _ -> false /// A type whose values cannot change and have no identity, so that its methods give an equal result for - /// equal arguments every time: a number, a string, a date and the like - let isPlainValueType (ty: Type) = + /// equal arguments every time: a number, a string, a date and the like, and a nullable of one of those + let rec isPlainValueType (ty: Type) = ty.IsPrimitive || ty.IsEnum || Type.op_Equality (ty, typeof) @@ -506,7 +519,7 @@ module LeafExpressionConverter = || Type.op_Equality (ty, typeof) || (ty.IsGenericType && Type.op_Equality (ty.GetGenericTypeDefinition (), typedefof>) - && ty.GetGenericArguments().[0].IsValueType) + && isPlainValueType (ty.GetGenericArguments().[0])) /// A method of a plain value that gives a plain value, like 'ToUpper' of a string let isPlainValueMethod (call: MethodCallExpression) = @@ -544,7 +557,11 @@ module LeafExpressionConverter = | :? MethodCallExpression as call -> isPlainValueMethod call && isRead letVariables call.Object - && call.Arguments |> Seq.forall (fun argument -> argument :? ConstantExpression || isElementRead letVariables argument) + // The method can run code of the user's on an argument that is no plain value, as often as it is made + && call.Arguments + |> Seq.forall (fun argument -> + isPlainValueType argument.Type + && (argument :? ConstantExpression || isElementRead letVariables argument)) | _ -> isElementRead letVariables expr /// Whether the read of the variable is the first thing that evaluating the expression does, on every path: @@ -988,11 +1005,12 @@ module LeafExpressionConverter = let isStructCopy = v.IsMutable && String.Equals (v.Name, "copyOfStruct", StringComparison.Ordinal) // A struct of the user's bound by a 'let' is a copy, and what is done to the copy must not reach - // the original, which the value would stand for. + // the original, which the value would stand for. The name the compiler gives to its copies makes no + // difference to that: only a copy of a plain value, which nothing is done to, gives way. let mayGiveWay = env.substituteLets && (not v.IsMutable || isStructCopy) - && (not v.Type.IsValueType || isPlainValueType v.Type || isStructCopy) + && (not v.Type.IsValueType || isPlainValueType v.Type) && not (Type.op_Equality (eP.Type, typeof)) // The Coerce case drops reference upcasts, and uses of the variable may need its own type. diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs index 607b5cb464f..af991c444e5 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs @@ -78,6 +78,29 @@ type TallyHolder() = type Reading = { First: int; Second: int } +let mutable private hashCalls = 0 + +/// A struct whose hash is the number of times a hash was asked for +[] +type CountedHash(id: int) = + member _.Id = id + + override _.GetHashCode() = + hashCalls <- hashCalls + 1 + hashCalls + + override _.Equals(_: obj) = false + +/// A formatting provider that counts how often it is asked for a format +type CountingProvider() = + let mutable calls = 0 + member _.Calls = calls + + interface IFormatProvider with + member _.GetFormat(_: Type) = + calls <- calls + 1 + null + let private run (lambda: Expression>) (argument: int) = lambda.Compile().Invoke argument [] @@ -361,3 +384,68 @@ let ``Record fields are not reordered next to a statement in a query over a quer |> Seq.toList Assert.Equal([ { First = 2; Second = 1 } ], rows) + +[] +let ``Let written by hand is evaluated before the fields that follow it`` () = + let counter = Counter() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun _ -> let second = counter.Next() in { First = counter.Next(); Second = second }) @> + + Assert.Equal({ First = 2; Second = 1 }, run lambda 0) + +[] +let ``Let written by hand is evaluated before the fields that follow it in a query over a queryable`` () = + let counter = Counter() + let source = System.Linq.Queryable.AsQueryable [ 1 ] + + let rows = + query { + for _ in source do + select ( + let second = counter.Next() + { First = counter.Next(); Second = second } + ) + } + |> Seq.toList + + Assert.Equal([ { First = 2; Second = 1 } ], rows) + +[] +let ``Hash of a nullable struct of the user's is computed once however often it is read`` () = + hashCalls <- 0 + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func, int>(fun n -> let hash = n.GetHashCode() in hash + hash) @> + + Assert.Equal(2, lambda.Compile().Invoke(Nullable(CountedHash 1))) + Assert.Equal(1, hashCalls) + +[] +let ``Text made with a formatting provider is made once however often it is read`` () = + let counting = CountingProvider() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun provider -> let text = (-1).ToString provider in text + text) @> + + // A negative number asks the provider for its sign + let once = (-1).ToString() + Assert.Equal(once + once, lambda.Compile().Invoke counting) + Assert.Equal(1, counting.Calls) + +[] +let ``Struct bound by a mutable let named like the copy the compiler makes is a copy`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func(fun holder -> + let mutable copyOfStruct = holder.Tally + copyOfStruct.AddAndCount()) + @> + + let holder = TallyHolder() + Assert.Equal(1, lambda.Compile().Invoke holder) + Assert.Equal(0, holder.Tally.Count) From cb58bf643911b9a72ba415162fc4c97930fdddd9 Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sun, 1 Nov 2026 23:10:42 +0100 Subject: [PATCH 13/19] Replace a let of plain arithmetic by its value A let that adds, subtracts or multiplies constants and the data a lambda is given gives way to its value, like a let that only names a member: let next = p.Age + 1 no longer makes a block, and a lambda nested in the one that has it reads the value through the parameter. The operands are constants and reads, never arithmetic themselves, so a value that is built on another one does not grow with every let. Co-Authored-By: Claude Sonnet 5.5 --- docs/release-notes/.FSharp.Core/11.0.200.md | 2 +- .../release-notes/PENDING_BREAKING_CHANGES.md | 4 +-- src/FSharp.Core/Linq.fs | 33 +++++++++++++++++-- .../ExpressionLambdaConversionTests.fs | 28 ++++++++++++++++ 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/docs/release-notes/.FSharp.Core/11.0.200.md b/docs/release-notes/.FSharp.Core/11.0.200.md index 3994b58cd15..5cb9c9e70bc 100644 --- a/docs/release-notes/.FSharp.Core/11.0.200.md +++ b/docs/release-notes/.FSharp.Core/11.0.200.md @@ -8,6 +8,6 @@ * Quotation debug text from `Expr.ToString()`, `Expr.ToString(full)`, and `%A` now prints struct tuple construction as `NewStructTuple`; reference tuples remain `NewTuple`. ([Issue #20800](https://github.com/dotnet/fsharp/issues/20800), [PR #20814](https://github.com/dotnet/fsharp/pull/20814)) * Fix struct tuples in query expressions: selecting one no longer fails with "unreachable", and the items of one that a later operator reads are taken from its fields instead of failing with `ArgumentException: Struct tuples store their elements in fields, not properties`. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) * Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) -* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) +* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member, a sum, difference or product of members, or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) ### Changed diff --git a/docs/release-notes/PENDING_BREAKING_CHANGES.md b/docs/release-notes/PENDING_BREAKING_CHANGES.md index 63691df6475..9653693c131 100644 --- a/docs/release-notes/PENDING_BREAKING_CHANGES.md +++ b/docs/release-notes/PENDING_BREAKING_CHANGES.md @@ -40,7 +40,7 @@ a.Equals(b) // Was: false, Now: true Inside a lambda handed to a LINQ provider (a query operator over `IQueryable`, or an F# lambda passed as `Expression>`) a let no longer makes a `Block` where leaving it out computes the same: -- its value only reads: a constant, the lambda's parameter or a member of it (`let age = p.Age`), or what a member or method of a string, number or date makes of those (`let upper = p.Name.ToUpper()`); +- its value only reads: a constant, the lambda's parameter or a member of it (`let age = p.Age`), what a member or method of a string, number or date makes of those (`let upper = p.Name.ToUpper()`), or an addition, subtraction or multiplication of those (`let next = p.Age + 1`); - or the variable is read once, as the first thing the rest of the lambda evaluates (`let orders = p.Orders.Count() in orders > 2`). This covers the lets the compiler introduces for tupled lambda parameters and struct copies. For record and anonymous record fields written out of order, the lets are left out as well: @@ -51,7 +51,7 @@ db.People.Select(fun p -> {| Name = p.Name; Age = p.Age |}) // Now: p => new <>f__AnonymousType(p.Age, p.Name) ``` -**Impact:** When such a lambda is compiled and run, the fields of a record written out of order are evaluated in declaration order instead of the order they are written in, which shows when they have side effects. A let that only reads, such as `let age = p.Age` or `let upper = p.Name.ToUpper()`, is computed where the variable is used: once for each use, at the moment of that use, and not at all on a path that does not use it. +**Impact:** When such a lambda is compiled and run, the fields of a record written out of order are evaluated in declaration order instead of the order they are written in, which shows when they have side effects. A let that only reads, such as `let age = p.Age`, `let upper = p.Name.ToUpper()` or `let next = p.Age + 1`, is computed where the variable is used: once for each use, at the moment of that use, and not at all on a path that does not use it. Every other let keeps its `Block` and is evaluated once, where it is written: a value that is computed by a call and read more than once, by a nested lambda or after something else has run, a struct of your own or a boxed value, a `mutable` variable, and every let of a lambda that assigns, sets a property or field, or runs a statement. `EvaluateQuotation` and `QuotationToExpression` are not affected. diff --git a/src/FSharp.Core/Linq.fs b/src/FSharp.Core/Linq.fs index 684f4781374..30ce0ae9fb8 100644 --- a/src/FSharp.Core/Linq.fs +++ b/src/FSharp.Core/Linq.fs @@ -541,12 +541,39 @@ module LeafExpressionConverter = | :? UnaryExpression as conversion -> isUpcast conversion && isElementRead letVariables conversion.Operand | _ -> false - /// Whether the expression only reads: a constant, the data a lambda is given, or what a member or method of a - /// plain value makes of those. Where a 'let' gives such a read a name, the read can be made wherever the name - /// is used, and as often as it is used. + /// An operand of arithmetic that a 'let' may give way to: a constant or the data a lambda is given, as a number + /// of another type if it has to be. Not arithmetic itself, so that a value does not grow with every 'let' that + /// builds on it. + let isOperand (letVariables: ParameterExpression list) (expr: Expression) = + match expr with + | :? ConstantExpression -> true + | :? UnaryExpression as conversion when conversion.NodeType = ExpressionType.Convert -> + (match conversion.Method with + | null -> true + | _ -> false) + && conversion.Type.IsPrimitive + && conversion.Operand.Type.IsPrimitive + && (conversion.Operand :? ConstantExpression || isElementRead letVariables conversion.Operand) + | _ -> isElementRead letVariables expr + + /// An addition, subtraction or multiplication of plain values: it runs no code of the user's + let isPlainArithmetic (operation: BinaryExpression) = + (operation.NodeType = ExpressionType.Add + || operation.NodeType = ExpressionType.Subtract + || operation.NodeType = ExpressionType.Multiply) + && isPlainValueType operation.Type + && (match operation.Method with + | null -> true + | meth -> isPlainValueType meth.DeclaringType) + + /// Whether the expression only reads: a constant, the data a lambda is given, what a member or method of a + /// plain value makes of those, or plain arithmetic on those. Where a 'let' gives such a read a name, the read + /// can be made wherever the name is used, and as often as it is used. let rec isRead (letVariables: ParameterExpression list) (expr: Expression) = match expr with | :? ConstantExpression -> true + | :? BinaryExpression as arithmetic when isPlainArithmetic arithmetic -> + isOperand letVariables arithmetic.Left && isOperand letVariables arithmetic.Right | :? MemberExpression as read when isPlainValueType read.Member.DeclaringType -> match read.Expression with | null -> false diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs index af991c444e5..e7bb85d31b2 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs @@ -449,3 +449,31 @@ let ``Struct bound by a mutable let named like the copy the compiler makes is a let holder = TallyHolder() Assert.Equal(1, lambda.Compile().Invoke holder) Assert.Equal(0, holder.Tally.Count) + +[] +let ``Let of an arithmetic on the parameter is replaced by its value`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun a -> let n = a + 1 in n * n) @> + + Assert.False(hasBlock lambda) + Assert.Equal(16, run lambda 3) + +[] +let ``Let of an arithmetic on a converted member is replaced by its value`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun c -> let total = float c.Age * 2.5 in total + total) @> + + Assert.False(hasBlock lambda) + Assert.Equal(15.0, lambda.Compile().Invoke(Customer(Age = 3))) + +[] +let ``Let of an arithmetic on a call keeps its block and runs the call once`` () = + let counter = Counter() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun a -> let n = counter.Next() + a in n + n) @> + + Assert.True(hasBlock lambda) + Assert.Equal(8, run lambda 3) + Assert.Equal(1, counter.Calls) From 7f35b4dd1f1c140905121a95e89b4a4d7ab3289d Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Mon, 2 Nov 2026 14:38:57 +0100 Subject: [PATCH 14/19] Evaluate record fields as written and keep a made string one string The lets that the compiler adds for record fields written out of order were left out whatever their value was, so a field with a side effect was evaluated after the fields that follow it: { Second = counter.Next(); First = counter.Next() } gave First = 1, and so did a let written by hand and named like the field it feeds. These lets are ordinary lets now. The construction of the anonymous object that stands for the record stays behind them, so a field that only reads still reaches the provider without a block. A let of a call that makes a string was replaced by the call at every use, so obj.ReferenceEquals(upper, upper) was false. It is replaced when it is used once, or when every use only looks at what the string holds. A division of reads by a constant other than 0 and -1 is a read as well, so the lets of a key like { KeyAge = c.Age / 10; KeyCity = c.City } stay out of the query. The note on breaking changes lists what else changes when and how often a read runs: in a nested lambda, after a call that changes its source, and on a path that does not use it. It says what EvaluateQuotation does with a let inside a lambda that goes to a provider. The release note names the join key that is built on one side only, and the part of #3782 that this fixes. Co-Authored-By: Claude Sonnet 5.5 --- docs/release-notes/.FSharp.Core/11.0.200.md | 3 +- .../release-notes/PENDING_BREAKING_CHANGES.md | 14 +- src/FSharp.Core/Linq.fs | 122 +++++++++------ src/FSharp.Core/Linq.fsi | 2 - src/FSharp.Core/Query.fs | 41 ++--- src/FSharp.Core/QueryExtensions.fs | 13 ++ .../ExpressionLambdaConversionTests.fs | 146 ++++++++++++++++++ .../QueryMethodSyntaxScenarios.fs | 3 - .../Microsoft.FSharp.Linq/QueryTests.fs | 22 +++ 9 files changed, 277 insertions(+), 89 deletions(-) diff --git a/docs/release-notes/.FSharp.Core/11.0.200.md b/docs/release-notes/.FSharp.Core/11.0.200.md index 5cb9c9e70bc..0e465a050d5 100644 --- a/docs/release-notes/.FSharp.Core/11.0.200.md +++ b/docs/release-notes/.FSharp.Core/11.0.200.md @@ -7,7 +7,6 @@ * Correct `FSharpValue.PreComputeTuplePropertyInfo` to report that struct tuples store elements in fields, not properties, and recommend supported tuple readers instead of reporting a valid index as out of range. ([Issue #20801](https://github.com/dotnet/fsharp/issues/20801), [PR #20815](https://github.com/dotnet/fsharp/pull/20815)) * Quotation debug text from `Expr.ToString()`, `Expr.ToString(full)`, and `%A` now prints struct tuple construction as `NewStructTuple`; reference tuples remain `NewTuple`. ([Issue #20800](https://github.com/dotnet/fsharp/issues/20800), [PR #20814](https://github.com/dotnet/fsharp/pull/20814)) * Fix struct tuples in query expressions: selecting one no longer fails with "unreachable", and the items of one that a later operator reads are taken from its fields instead of failing with `ArgumentException: Struct tuples store their elements in fields, not properties`. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) -* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) -* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member, a sum, difference or product of members, or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) +* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member, a sum, difference, product or quotient (by a constant) of members, or is read once now produce plain expressions. A `join` whose key one side builds as a tuple and the other side has ready-made, such as `join p in ps on ((k.KA, k.KB) = p.Pair)`, no longer fails with "Type mismatch when building 'ty'". ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [Issue #3782](https://github.com/dotnet/fsharp/issues/3782) for records with fields out of order, [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) ### Changed diff --git a/docs/release-notes/PENDING_BREAKING_CHANGES.md b/docs/release-notes/PENDING_BREAKING_CHANGES.md index 9653693c131..05fedaddabc 100644 --- a/docs/release-notes/PENDING_BREAKING_CHANGES.md +++ b/docs/release-notes/PENDING_BREAKING_CHANGES.md @@ -40,10 +40,10 @@ a.Equals(b) // Was: false, Now: true Inside a lambda handed to a LINQ provider (a query operator over `IQueryable`, or an F# lambda passed as `Expression>`) a let no longer makes a `Block` where leaving it out computes the same: -- its value only reads: a constant, the lambda's parameter or a member of it (`let age = p.Age`), what a member or method of a string, number or date makes of those (`let upper = p.Name.ToUpper()`), or an addition, subtraction or multiplication of those (`let next = p.Age + 1`); +- its value only reads: a constant, the lambda's parameter or a member of it (`let age = p.Age`), what a member or method of a string, number or date makes of those (`let upper = p.Name.ToUpper()`), an addition, subtraction or multiplication of those (`let next = p.Age + 1`), or a division by a constant other than 0 and -1 (`let decade = p.Age / 10`); - or the variable is read once, as the first thing the rest of the lambda evaluates (`let orders = p.Orders.Count() in orders > 2`). -This covers the lets the compiler introduces for tupled lambda parameters and struct copies. For record and anonymous record fields written out of order, the lets are left out as well: +This covers the lets the compiler introduces for tupled lambda parameters and struct copies. For record and anonymous record fields written out of order, the compiler adds a let for each field it moves, and that let is treated like any other: ```fsharp db.People.Select(fun p -> {| Name = p.Name; Age = p.Age |}) @@ -51,9 +51,15 @@ db.People.Select(fun p -> {| Name = p.Name; Age = p.Age |}) // Now: p => new <>f__AnonymousType(p.Age, p.Name) ``` -**Impact:** When such a lambda is compiled and run, the fields of a record written out of order are evaluated in declaration order instead of the order they are written in, which shows when they have side effects. A let that only reads, such as `let age = p.Age`, `let upper = p.Name.ToUpper()` or `let next = p.Age + 1`, is computed where the variable is used: once for each use, at the moment of that use, and not at all on a path that does not use it. +**Impact:** A let that only reads, such as `let age = p.Age`, `let upper = p.Name.ToUpper()` or `let next = p.Age + 1`, is computed where the variable is used. When such a lambda is compiled and run, that changes when and how often it runs: -Every other let keeps its `Block` and is evaluated once, where it is written: a value that is computed by a call and read more than once, by a nested lambda or after something else has run, a struct of your own or a boxed value, a `mutable` variable, and every let of a lambda that assigns, sets a property or field, or runs a statement. `EvaluateQuotation` and `QuotationToExpression` are not affected. +- once for each use, and once for each element when a nested lambda uses it, an arithmetic read like `let next = p.Age + 1` included: with `names` a `List`, `custs.Select(fun c -> let n = c.Name in names.Count(fun s -> s = n))` reads `Name` once for each name and customer, where it read it once for each customer before; +- at the moment of that use, so a member of the lambda's parameter that something in between changes gives the new value: `fun (items: List) -> let before = items.Count in addTo items 5 + before` gives 2 for an empty list where it gave 1; +- not at all on a path that does not use it: `let s = name.Substring 5 in if name.Length > 5 then s else ""` gives `""` for `"ab"` where it raised `ArgumentOutOfRangeException`. + +A record field written out of order whose value is more than a read, such as `{ Second = counter.Next(); First = counter.Next() }`, keeps its let, and the fields are evaluated in the order they are written, as they are in code. + +Every other let keeps its `Block` and is evaluated once, where it is written: a value that is computed by a call and read more than once, by a nested lambda or after something else has run, a struct of your own or a boxed value, a `mutable` variable, and every let of a lambda that assigns, sets a property or field, or runs a statement. A value that a call makes anew, like the `let upper = p.Name.ToUpper()` above, is kept as well when it is used more than once and a use can tell one object from another, as `obj.ReferenceEquals(upper, upper)` does. A let outside such a lambda keeps its `Block` in `EvaluateQuotation` and `QuotationToExpression`; inside a lambda that goes to a provider it is replaced the same way whichever of them converts it: `EvaluateQuotation <@ people.Select(fun x -> let n = x.Name in n + n).ToArray() @>` reads `Name` twice. ### Array access uses ArrayIndex diff --git a/src/FSharp.Core/Linq.fs b/src/FSharp.Core/Linq.fs index 30ce0ae9fb8..e27b5f7d2ea 100644 --- a/src/FSharp.Core/Linq.fs +++ b/src/FSharp.Core/Linq.fs @@ -455,47 +455,6 @@ module LeafExpressionConverter = | ExprShape.ShapeLambda (_, body) -> hasStatements body | ExprShape.ShapeCombination (_, args) -> List.exists hasStatements args - let rec countUses (v: Var) (expr: Expr) = - match expr with - | ExprShape.ShapeVar used -> if used = v then 1 else 0 - | ExprShape.ShapeLambda (_, body) -> countUses v body - | ExprShape.ShapeCombination (_, args) -> List.sumBy (countUses v) args - - /// The record that a chain of lets leads to, with the arguments of its construction - let rec (|NewRecordAfterLets|_|) (expr: Expr) = - match expr with - | Let (_, _, NewRecordAfterLets record) -> Some record - | NewRecord (recordType, args) -> Some (recordType, args) - | _ -> None - - /// The compiler evaluates record fields in the order they are written, and binds the ones written before - /// their place to variables named after the field: 'let Second = e in .. NewRecord (.., Second, ..)'. - /// Whether the variable is such a one: the argument in the place of a field is the variable named like it. - /// A 'let' of the code with another name evaluates its value first, whatever the record that follows. - let isHoistedField (v: Var) (recordType: Type) (args: Expr list) = - let fields = FSharpType.GetRecordFields (recordType, showAll) - - args - |> List.indexed - |> List.exists (fun (index, arg) -> - match arg with - | Var used -> used = v && index < fields.Length && String.Equals (fields.[index].Name, v.Name, StringComparison.Ordinal) - | _ -> false) - - /// This is such a 'let', and its body with the value in the place of the variable, which evaluates the - /// fields in the order they are declared. - let (|HoistedRecordField|_|) (enabled: bool) (expr: Expr) = - match expr with - | Let (v, e, body) when enabled && not v.IsMutable -> - match body with - | NewRecordAfterLets (recordType, args) - when isHoistedField v recordType args - && countUses v body = 1 - && not (hasStatements expr) -> - Some (body.Substitute (fun used -> if used = v then Some e else None)) - | _ -> None - | _ -> None - /// A conversion of a reference to a type its object already has, which runs no code, cannot fail and gives /// the same object. Boxing does not: each box is an object of its own. let isUpcast (conversion: UnaryExpression) = @@ -556,11 +515,26 @@ module LeafExpressionConverter = && (conversion.Operand :? ConstantExpression || isElementRead letVariables conversion.Operand) | _ -> isElementRead letVariables expr - /// An addition, subtraction or multiplication of plain values: it runs no code of the user's + /// A division or remainder that cannot fail: one of floating point numbers, which give infinity or NaN, or one of + /// integers by a constant other than 0 and -1, which overflows for the smallest integer + let isSafeDivision (operation: BinaryExpression) = + (operation.NodeType = ExpressionType.Divide || operation.NodeType = ExpressionType.Modulo) + && operation.Type.IsPrimitive + && (Type.op_Equality (operation.Type, typeof) + || Type.op_Equality (operation.Type, typeof) + || (match operation.Right with + | :? ConstantExpression as divisor when not (isNull divisor.Value) -> + let value = Convert.ToDecimal divisor.Value + value <> 0m && value <> -1m + | _ -> false)) + + /// An addition, subtraction or multiplication of plain values, or a division that cannot fail: it runs no code of + /// the user's let isPlainArithmetic (operation: BinaryExpression) = (operation.NodeType = ExpressionType.Add || operation.NodeType = ExpressionType.Subtract - || operation.NodeType = ExpressionType.Multiply) + || operation.NodeType = ExpressionType.Multiply + || isSafeDivision operation) && isPlainValueType operation.Type && (match operation.Method with | null -> true @@ -660,6 +634,46 @@ module LeafExpressionConverter = else node :> Expression + /// Whether evaluating the expression makes an object: a call or an addition with an object as its result, like the + /// upper case of a string. Evaluated twice, it gives two objects. + let makesNewObject (expr: Expression) = + not expr.Type.IsValueType && (expr :? MethodCallExpression || expr :? BinaryExpression) + + /// Finds the uses of a variable that can tell one object from another of the same value: everything but the object + /// being called on, or passed to, a method of a string, number or date, and the operands of an operator of one. + type ObjectIdentityUses (variable: ParameterExpression) = + inherit ExpressionVisitor () + + member val Found = false with get, set + + member private this.VisitOperands (operands: Expression list) = + for operand in operands do + match operand with + | null -> () + | operand when obj.ReferenceEquals (operand, variable) -> () + | operand -> this.Visit operand |> ignore + + override this.VisitMethodCall (node: MethodCallExpression) : Expression = + if isPlainValueType node.Method.DeclaringType then + this.VisitOperands (node.Object :: List.ofSeq node.Arguments) + node :> Expression + else + base.VisitMethodCall node + + override this.VisitBinary (node: BinaryExpression) : Expression = + match node.Method with + | null -> base.VisitBinary node + | operator when isPlainValueType operator.DeclaringType -> + this.VisitOperands [ node.Left; node.Right ] + node :> Expression + | _ -> base.VisitBinary node + + override this.VisitParameter (node: ParameterExpression) : Expression = + if obj.ReferenceEquals (node, variable) then + this.Found <- true + + node :> Expression + /// Convert F# quotations to LINQ expression trees. /// A more polished LINQ-Quotation translator will be published /// concert with later versions of LINQ. @@ -1015,9 +1029,6 @@ module LeafExpressionConverter = (Expression.NewArrayInit(typeof, [| for fv in fvs -> Expression.Convert(env.varEnv.[fv], typeof) |> asExpr |]) |> asExpr) |]) |> asExpr - | HoistedRecordField env.substituteLets inlined -> - ConvExprToLinqInContext env inlined - | Let (v, e, b) -> // Use Expression.Block to properly scope the variable. This: // 1. Evaluates the expression exactly once (preserving reference semantics) @@ -1043,8 +1054,11 @@ module LeafExpressionConverter = // The Coerce case drops reference upcasts, and uses of the variable may need its own type. let value () = if Type.op_Equality (eP.Type, v.Type) then eP else Expression.Convert (eP, v.Type) |> asExpr - if mayGiveWay && isRead env.letVariables eP then - ConvExprToLinqInContext { env with varEnv = Map.add v (value ()) env.varEnv } b + let isReadValue = mayGiveWay && isRead env.letVariables eP + let substituteValue () = ConvExprToLinqInContext { env with varEnv = Map.add v (value ()) env.varEnv } b + + if isReadValue && not (makesNewObject eP) then + substituteValue () else let envinner = { env with varEnv = Map.add v (vP |> asExpr) env.varEnv; letVariables = vP :: env.letVariables } let bodyP = ConvExprToLinqInContext envinner b @@ -1054,7 +1068,17 @@ module LeafExpressionConverter = let reads = VariableReads (vP, value ()) let replaced = reads.Visit bodyP - if reads.Count = 1 && readsFirst vP bodyP then + if isReadValue then + // The object is made anew at every use: with more than one, it takes uses that look at what + // it holds, and nothing that can tell one object from another + let identity = ObjectIdentityUses vP + identity.Visit bodyP |> ignore + + if reads.Count <= 1 || not identity.Found then + ValueSome (substituteValue ()) + else + ValueNone + elif reads.Count = 1 && readsFirst vP bodyP then ValueSome replaced else ValueNone diff --git a/src/FSharp.Core/Linq.fsi b/src/FSharp.Core/Linq.fsi index fcbcdb7221b..c4807ef98a1 100644 --- a/src/FSharp.Core/Linq.fsi +++ b/src/FSharp.Core/Linq.fsi @@ -91,5 +91,3 @@ module LeafExpressionConverter = RuntimeMethodHandle -> (Expr -> (Expr option * Reflection.MethodInfo * Expr list) option) val internal hasStatements: expr: Expr -> bool - - val internal (|HoistedRecordField|_|): enabled: bool -> expr: Expr -> Expr option diff --git a/src/FSharp.Core/Query.fs b/src/FSharp.Core/Query.fs index 73bf8e3cadf..07050078fdf 100644 --- a/src/FSharp.Core/Query.fs +++ b/src/FSharp.Core/Query.fs @@ -1043,23 +1043,6 @@ module Query = | MacroReduction reduced -> Some (walk reduced) | _ -> None) - /// Without the lets the compiler adds for record fields written out of order, the record reaches - /// ProduceMoreMutables as a record construction. - let rec InlineHoistedFields (p: Expr) = - match p with - | HoistedRecordField true inlined -> InlineHoistedFields inlined - | QuoteTyped _ - | QuoteRaw _ -> p - | ExprShape.ShapeCombination(_, []) -> p - | ExprShape.ShapeCombination(comb, args) -> ExprShape.RebuildShapeCombination(comb, List.map InlineHoistedFields args) - | ExprShape.ShapeLambda(v, body) -> Expr.Lambda(v, InlineHoistedFields body) - | ExprShape.ShapeVar _ -> p - - /// A provider reads the fields of a record in any order, so what goes to one loses the lets that keep the - /// written order. What is evaluated in memory keeps them, and with them the order. - let ProduceMoreMutablesFor isIQ tipf expr = - ProduceMoreMutables tipf (if isIQ && not (hasStatements expr) then InlineHoistedFields expr else expr) - let (|CallQueryBuilderRunQueryable|_|) : Expr -> _ = (|SpecificCallToMethod|_|) (methodhandleof (fun (b :QueryBuilder, v) -> b.Run v)) let (|CallQueryBuilderRunValue|_|) : Expr -> _ = (|SpecificCallToMethod|_|) (methodhandleof (fun (b : QueryBuilder, v: Expr<'a>) -> b.Run v) : 'a) @@ -1323,7 +1306,7 @@ module Query = | CallQueryBuilderYield(_, _, immutSelectorBody) -> let mutSelectorVar, mutSelectorBody = ConvertImmutableConsumerToMutableConsumer sourceConv (immutSelectorVar, MacroExpand immutSelectorBody) - let mutSelectorBody, selectorConv = ProduceMoreMutablesFor (qTyIsIQueryable qTy) TransInnerNoCheck mutSelectorBody + let mutSelectorBody, selectorConv = ProduceMoreMutables TransInnerNoCheck mutSelectorBody let mutSelectorBody = CleanupLeaf mutSelectorBody TransInnerResult.MakeSelect (canElim, qTyIsIQueryable qTy, mutSource, mutSelectorVar, mutSelectorBody), selectorConv @@ -1384,7 +1367,7 @@ module Query = | CallSelect (_, [_; qTy; _], mutSource, Lambda(immutSelectorVar, immutSelectorBody)) -> let mutSource, _sourceConv, mutSelectorVar, mutSelectorBody = TransInnerApplicative check mutSource (immutSelectorVar, immutSelectorBody) - let mutSelectorBody, selectorConv = ProduceMoreMutablesFor (qTyIsIQueryable qTy) TransInnerNoCheck mutSelectorBody + let mutSelectorBody, selectorConv = ProduceMoreMutables TransInnerNoCheck mutSelectorBody let mutSelectorBody = CleanupLeaf mutSelectorBody TransInnerResult.MakeSelect(canElim, qTyIsIQueryable qTy, mutSource, mutSelectorVar, mutSelectorBody), selectorConv @@ -1490,7 +1473,7 @@ module Query = let mutSource, sourceConv = TransInnerAndCommit CanEliminate.Yes check immutSource let mutVar2, mutKeySelector = ConvertImmutableConsumerToMutableConsumer sourceConv (immutVar2, MacroExpand immutKeySelector) let mutVar1, mutElementSelector = ConvertImmutableConsumerToMutableConsumer sourceConv (immutVar1, MacroExpand immutElementSelector) - let mutElementSelector, selectorConv = ProduceMoreMutablesFor (qTyIsIQueryable qTy) TransInnerNoCheck mutElementSelector + let mutElementSelector, selectorConv = ProduceMoreMutables TransInnerNoCheck mutElementSelector let mutElementSelector = CleanupLeaf mutElementSelector let conv = match selectorConv with @@ -1506,11 +1489,11 @@ module Query = LambdasNoDetupling([immutOuterResultGroupVar; immutInnerResultKeyVar], immutElementSelector)]) -> let (mutOuterSource, outerSourceConv, mutInnerSource, innerSourceConv, mutOuterKeyVar: Var, mutOuterKeySelector, mutInnerKeyVar: Var, mutInnerKeySelector: Expr) = - TransJoinInputs check (qTyIsIQueryable qTy) (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) + TransJoinInputs check (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) let mutOuterResultVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer outerSourceConv (immutOuterResultGroupVar, MacroExpand immutElementSelector) let mutInnerResultKeyVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer innerSourceConv (immutInnerResultKeyVar, mutElementSelector) - let mutElementSelector, elementSelectorConv = ProduceMoreMutablesFor (qTyIsIQueryable qTy) TransInnerNoCheck mutElementSelector + let mutElementSelector, elementSelectorConv = ProduceMoreMutables TransInnerNoCheck mutElementSelector let mutElementSelector = CleanupLeaf mutElementSelector let joinExpr = @@ -1530,12 +1513,12 @@ module Query = LambdasNoDetupling([immutOuterResultGroupVar; immutInnerResultGroupVar], immutElementSelector)]) -> let (mutOuterSource, outerSourceConv, mutInnerSource, innerSourceConv, mutOuterKeyVar: Var, mutOuterKeySelector, mutInnerKeyVar: Var, mutInnerKeySelector: Expr) = - TransJoinInputs check (qTyIsIQueryable qTy) (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) + TransJoinInputs check (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) let mutOuterResultGroupVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer outerSourceConv (immutOuterResultGroupVar, MacroExpand immutElementSelector) let innerGroupConv = MakeSeqConv innerSourceConv let mutInnerResultKeyVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer innerGroupConv (immutInnerResultGroupVar, mutElementSelector) - let mutElementSelector, elementSelectorConv = ProduceMoreMutablesFor (qTyIsIQueryable qTy) TransInnerNoCheck mutElementSelector + let mutElementSelector, elementSelectorConv = ProduceMoreMutables TransInnerNoCheck mutElementSelector let mutElementSelector = CleanupLeaf mutElementSelector let joinExpr = @@ -1559,11 +1542,11 @@ module Query = let immutElementSelector = immutElementSelector.Substitute (fun v -> if v = immutInnerResultGroupVar then Some (MakeDefaultIfEmpty ([immutInnerSourceTy], [Expr.Var immutInnerResultGroupVar])) else None) let (mutOuterSource, outerSourceConv, mutInnerSource, innerSourceConv, mutOuterKeyVar: Var, mutOuterKeySelector, mutInnerKeyVar: Var, mutInnerKeySelector: Expr) = - TransJoinInputs check (qTyIsIQueryable qTy) (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) + TransJoinInputs check (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) let mutOuterResultGroupVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer outerSourceConv (immutOuterResultGroupVar, MacroExpand immutElementSelector) let mutInnerResultKeyVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer innerSourceConv (immutInnerResultGroupVar, mutElementSelector) - let mutElementSelector, elementSelectorConv = ProduceMoreMutablesFor (qTyIsIQueryable qTy) TransInnerNoCheck mutElementSelector + let mutElementSelector, elementSelectorConv = ProduceMoreMutables TransInnerNoCheck mutElementSelector let mutElementSelector = CleanupLeaf mutElementSelector let joinExpr = @@ -1655,7 +1638,7 @@ module Query = | _ -> e, NoConv - and TransJoinInputs check isIQ (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) = + and TransJoinInputs check (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) = let mutOuterSource, outerSourceConv = TransInnerAndCommit CanEliminate.Yes check immutOuterSource let mutInnerSource, innerSourceConv = TransInnerAndCommit CanEliminate.Yes check immutInnerSource @@ -1663,8 +1646,8 @@ module Query = let mutInnerKeyVar, mutInnerKeySelector = ConvertImmutableConsumerToMutableConsumer innerSourceConv (immutInnerKeyVar, MacroExpand immutInnerKeySelector) // Keys may be composite tuples - convert them to be mutables. Note, if there is a tuple on one side, there must be a tuple on the other side. - let mutOuterKey, _ = ProduceMoreMutablesFor isIQ TransNone mutOuterKeySelector - let mutInnerKey, _ = ProduceMoreMutablesFor isIQ TransNone mutInnerKeySelector + let mutOuterKey, _ = ProduceMoreMutables TransNone mutOuterKeySelector + let mutInnerKey, _ = ProduceMoreMutables TransNone mutInnerKeySelector // A key that one side builds and the other gets ready-made changes its type on one side only. Both keep // the type they were written with then. diff --git a/src/FSharp.Core/QueryExtensions.fs b/src/FSharp.Core/QueryExtensions.fs index 18edd4b9b5b..b38d1e266c3 100644 --- a/src/FSharp.Core/QueryExtensions.fs +++ b/src/FSharp.Core/QueryExtensions.fs @@ -300,12 +300,25 @@ module internal Adapters = | RecordFieldGetSimplification newExpr -> newExpr | _ -> e + /// Whether the expression builds a record, possibly after some lets + let rec constructsRecordAfterLets (expr: Expr) = + match expr with + | Patterns.Let(_, _, body) -> constructsRecordAfterLets body + | Patterns.NewRecord(typ, _) -> isPartiallyImmutableRecord typ + | _ -> false + /// Given the expression part of a "yield" or "select" which produces a result in terms of immutable tuples or immutable records, /// generate an equivalent expression yielding anonymous objects. Also return the conversion for the immutable-to-mutable correspondence /// so we can reverse this later. let rec ProduceMoreMutables tipf expr = match expr with + // A record with fields written out of order comes after the lets that the compiler adds to keep the order they + // are written in. The lets stay where they are, in front of the construction. + | Patterns.Let(v, e, body) when constructsRecordAfterLets body -> + let body, conv = ProduceMoreMutables tipf body + Expr.Let(v, e, body), conv + // Replace immutable tuples by anonymous objects | Patterns.NewTuple exprs -> let argExprsNow, argScripts = diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs index e7bb85d31b2..7d040f6a43f 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs @@ -477,3 +477,149 @@ let ``Let of an arithmetic on a call keeps its block and runs the call once`` () Assert.True(hasBlock lambda) Assert.Equal(8, run lambda 3) Assert.Equal(1, counter.Calls) + +[] +let ``Record fields written out of order are evaluated as written in an expression lambda`` () = + let counter = Counter() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun _ -> { Second = counter.Next(); First = counter.Next() }) @> + + Assert.Equal({ First = 2; Second = 1 }, run lambda 0) + +[] +let ``Record fields written out of order are evaluated as written in a query over a queryable`` () = + let counter = Counter() + let source = System.Linq.Queryable.AsQueryable [ 1 ] + + let rows = + query { + for _ in source do + select { Second = counter.Next(); First = counter.Next() } + } + |> Seq.toList + + Assert.Equal([ { First = 2; Second = 1 } ], rows) + +[] +let ``Let named like the record field it feeds is evaluated before the fields that follow it`` () = + let counter = Counter() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun _ -> let Second = counter.Next() in { First = counter.Next(); Second = Second }) @> + + Assert.Equal({ First = 2; Second = 1 }, run lambda 0) + +[] +let ``Let named like the record field it feeds is evaluated before the fields that follow it in a query over a queryable`` () = + let counter = Counter() + let source = System.Linq.Queryable.AsQueryable [ 1 ] + + let rows = + query { + for _ in source do + select ( + let Second = counter.Next() + { First = counter.Next(); Second = Second } + ) + } + |> Seq.toList + + Assert.Equal([ { First = 2; Second = 1 } ], rows) + +[] +let ``String made by a call is one string where a use can tell one from another`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun s -> let upper = s.ToUpperInvariant() in obj.ReferenceEquals(upper, upper)) @> + + Assert.True(hasBlock lambda) + Assert.True(lambda.Compile().Invoke "ab") + +[] +let ``String made by a call is made at each use that only looks at it`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun s -> let upper = s.ToUpperInvariant() in upper.StartsWith "A" && upper.EndsWith "B") @> + + Assert.False(hasBlock lambda) + Assert.True(lambda.Compile().Invoke "ab") + +/// A parameter whose name counts how often it is read +type CountedPerson() = + let mutable nameReads = 0 + member _.NameReads = nameReads + + member _.Name = + nameReads <- nameReads + 1 + "ab" + +[] +let ``Member of the parameter that a let names is read at each use`` () = + let person = CountedPerson() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun p -> let n = p.Name in n + n) @> + + Assert.Equal("abab", lambda.Compile().Invoke person) + Assert.Equal(2, person.NameReads) + +[] +let ``Member of the parameter that a nested lambda uses is read for each element of the nested lambda`` () = + let person = CountedPerson() + let names = Collections.Generic.List([ "a"; "ab"; "b"; "ab" ]) + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun p -> let n = p.Name in System.Linq.Enumerable.Count(names, Func(fun s -> s = n))) @> + + Assert.Equal(2, lambda.Compile().Invoke person) + Assert.Equal(4, person.NameReads) + +[] +let ``Member of the parameter that a let names is read after a call that changes it`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func, int>(fun items -> let before = items.Count in addTo items 5 + before) @> + + // The read is made where the variable is used, after the call + Assert.Equal(2, lambda.Compile().Invoke(Collections.Generic.List())) + +[] +let ``Let whose value would throw is not evaluated on a path that does not use it`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun name -> let s = name.Substring 5 in if name.Length > 5 then s else "") @> + + Assert.Equal("", lambda.Compile().Invoke "ab") + +[] +let ``EvaluateQuotation replaces a let inside a lambda that goes to a provider`` () = + let person = CountedPerson() + let people = System.Linq.Queryable.AsQueryable [ person ] + + let names = + LeafExpressionConverter.EvaluateQuotation + <@ System.Linq.Enumerable.ToArray(System.Linq.Queryable.Select(people, fun x -> let n = x.Name in n + n)) @> + :?> string[] + + Assert.Equal([| "abab" |], names) + Assert.Equal(2, person.NameReads) + +[] +let ``Let of a division by a constant is replaced by its value`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun a -> let half = a / 2 in half + half) @> + + Assert.False(hasBlock lambda) + Assert.Equal(6, run lambda 7) + +[] +let ``Let of a division by a parameter keeps its block so that its failure is not skipped`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun a -> let quotient = 10 / a in if a > 100 then quotient else 0) @> + + Assert.True(hasBlock lambda) + Assert.Throws(fun () -> run lambda 0 |> ignore) |> ignore diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs index 18aa3c9e240..4ac90358101 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs @@ -7,9 +7,6 @@ open System.Linq.Expressions let private block = [ Violation.Statement ExpressionType.Block ] -/// A tree is pinned with every block it has: one that goes from three to two is a change as well -let private blocks count = - List.replicate count (Violation.Statement ExpressionType.Block) let private reading typeName = [ Violation.ReadsUnnamedMember typeName ] diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryTests.fs index faebf58c869..5d17f172e88 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryTests.fs @@ -42,10 +42,32 @@ module TestData = { Table2.Id1 = 4; Id2 = 40; Value2 = "Z" } ] + type PairedTable = { PairKey: int * int; PairValue: string } + + let pairedData = [ + { PairKey = (1, 10); PairValue = "P" } + { PairKey = (2, 20); PairValue = "Q" } + { PairKey = (4, 40); PairValue = "R" } + ] + open TestData type QueryTupleJoinTests() = + [] + member _.``Join of a key built on one side with a tuple property on the other``() = + let t1 = table1Data.AsQueryable() + let paired = pairedData.AsQueryable() + + let result = + query { + for a in t1 do + join p in paired on ((a.Id1, a.Id2) = p.PairKey) + select (a.Value1, p.PairValue) + } |> Seq.toList + + Assert.Equal<(string * string) list>([ ("A", "P"); ("B", "Q") ], result) + [] member _.``Inline tuple join returns correct matches - issue 7885``() = let t1 = table1Data.AsQueryable() From 46c0a57b13672ae8965486f3ddf82a084d18792f Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Mon, 2 Nov 2026 17:37:24 +0100 Subject: [PATCH 15/19] Say what distinct does with records that have an equality of their own A record selected with its fields written out of order is converted like one with its fields in order, and distinct compares the converted objects by their fields. For a record type with an Equals of its own that was already so when the fields were in order, and the test pins both orders to the same result. The breaking changes name it. Co-Authored-By: Claude Sonnet 5.5 --- .../release-notes/PENDING_BREAKING_CHANGES.md | 2 ++ .../ExpressionLambdaConversionTests.fs | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/docs/release-notes/PENDING_BREAKING_CHANGES.md b/docs/release-notes/PENDING_BREAKING_CHANGES.md index 05fedaddabc..87d400d3687 100644 --- a/docs/release-notes/PENDING_BREAKING_CHANGES.md +++ b/docs/release-notes/PENDING_BREAKING_CHANGES.md @@ -59,6 +59,8 @@ db.People.Select(fun p -> {| Name = p.Name; Age = p.Age |}) A record field written out of order whose value is more than a read, such as `{ Second = counter.Next(); First = counter.Next() }`, keeps its let, and the fields are evaluated in the order they are written, as they are in code. +A record that a `query` selects with its fields written out of order is converted like one with its fields in order: the query selects an anonymous object in its place. `distinct` then compares the fields of the record, also for a record type with an `Equals` of its own, as it already did when the fields were written in order. With `[] type HeadOnly = { Head: int; Tail: int }` that is equal when `Head` is, `select { Tail = x + 1; Head = 0 }` followed by `distinct` gives one row for each value of `Tail`, where it gave one row. + Every other let keeps its `Block` and is evaluated once, where it is written: a value that is computed by a call and read more than once, by a nested lambda or after something else has run, a struct of your own or a boxed value, a `mutable` variable, and every let of a lambda that assigns, sets a property or field, or runs a statement. A value that a call makes anew, like the `let upper = p.Name.ToUpper()` above, is kept as well when it is used more than once and a use can tell one object from another, as `obj.ReferenceEquals(upper, upper)` does. A let outside such a lambda keeps its `Block` in `EvaluateQuotation` and `QuotationToExpression`; inside a lambda that goes to a provider it is replaced the same way whichever of them converts it: `EvaluateQuotation <@ people.Select(fun x -> let n = x.Name in n + n).ToArray() @>` reads `Name` twice. ### Array access uses ArrayIndex diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs index 7d040f6a43f..1a6616a3212 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs @@ -502,6 +502,31 @@ let ``Record fields written out of order are evaluated as written in a query ove Assert.Equal([ { First = 2; Second = 1 } ], rows) +/// A record that is equal to another when its first field is +[] +type HeadOnly = + { Head: int + Tail: int } + + override x.Equals(other: obj) = + match other with + | :? HeadOnly as y -> x.Head = y.Head + | _ -> false + + override x.GetHashCode() = x.Head.GetHashCode() + +[] +let ``Distinct tells apart the records of a query by their fields, written in order or not`` () = + let source = System.Linq.Queryable.AsQueryable [ 1; 2 ] + + // The query makes anonymous objects of the records it selects, which are equal when their fields are. That holds + // for the records with an equality of their own too, and fields written out of order change nothing about it. + let inOrder = query { for x in source do select { Head = 0; Tail = x + 1 }; distinct } |> Seq.length + let outOfOrder = query { for x in source do select { Tail = x + 1; Head = 0 }; distinct } |> Seq.length + + Assert.Equal(2, inOrder) + Assert.Equal(2, outOfOrder) + [] let ``Let named like the record field it feeds is evaluated before the fields that follow it`` () = let counter = Counter() From a49f7d92f8884a1503dbfb8d003ec235c10957e8 Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Wed, 4 Nov 2026 10:36:00 +0100 Subject: [PATCH 16/19] Count the places a made string goes to, also through the strings its methods give A string that a call makes is made at each use only when no use can tell one object from another. Two ways to tell went unseen. A method of a string that gives a string may give the string it was called on (Substring 0, Trim, Clone, adding an empty string), so two such calls were taken for a look at what the string holds, and ReferenceEquals of their results changed from true to false. And the single use inside a nested lambda was taken for one use, where the lambda makes the string for every element, so the elements no longer shared one string. The rule now follows the object: a method, an operator or a member of a string, number or date that gives a number or a boolean only looks at it, one that gives an object may give the same object back, and that object is followed to where it goes in turn. The string stays in a variable when it goes to more than one place that can tell objects apart, or to one in a nested lambda. Adding a string to itself, or comparing the length of a trimmed copy, still replaces the let. Co-Authored-By: Claude Sonnet 5.5 --- .../release-notes/PENDING_BREAKING_CHANGES.md | 2 +- src/FSharp.Core/Linq.fs | 110 +++++++++++++----- .../ExpressionLambdaConversionTests.fs | 84 +++++++++++++ 3 files changed, 165 insertions(+), 31 deletions(-) diff --git a/docs/release-notes/PENDING_BREAKING_CHANGES.md b/docs/release-notes/PENDING_BREAKING_CHANGES.md index 87d400d3687..272975bd809 100644 --- a/docs/release-notes/PENDING_BREAKING_CHANGES.md +++ b/docs/release-notes/PENDING_BREAKING_CHANGES.md @@ -61,7 +61,7 @@ A record field written out of order whose value is more than a read, such as `{ A record that a `query` selects with its fields written out of order is converted like one with its fields in order: the query selects an anonymous object in its place. `distinct` then compares the fields of the record, also for a record type with an `Equals` of its own, as it already did when the fields were written in order. With `[] type HeadOnly = { Head: int; Tail: int }` that is equal when `Head` is, `select { Tail = x + 1; Head = 0 }` followed by `distinct` gives one row for each value of `Tail`, where it gave one row. -Every other let keeps its `Block` and is evaluated once, where it is written: a value that is computed by a call and read more than once, by a nested lambda or after something else has run, a struct of your own or a boxed value, a `mutable` variable, and every let of a lambda that assigns, sets a property or field, or runs a statement. A value that a call makes anew, like the `let upper = p.Name.ToUpper()` above, is kept as well when it is used more than once and a use can tell one object from another, as `obj.ReferenceEquals(upper, upper)` does. A let outside such a lambda keeps its `Block` in `EvaluateQuotation` and `QuotationToExpression`; inside a lambda that goes to a provider it is replaced the same way whichever of them converts it: `EvaluateQuotation <@ people.Select(fun x -> let n = x.Name in n + n).ToArray() @>` reads `Name` twice. +Every other let keeps its `Block` and is evaluated once, where it is written: a value that is computed by a call and read more than once, by a nested lambda or after something else has run, a struct of your own or a boxed value, a `mutable` variable, and every let of a lambda that assigns, sets a property or field, or runs a statement. A value that a call makes anew, like the `let upper = p.Name.ToUpper()` above, is kept as well when the string goes to more than one place that can tell one object from another, or to one in a nested lambda: `obj.ReferenceEquals(upper, upper)` does, and so do `upper.Substring(0)` and `upper.Trim()`, which can give back the string they are called on, and `fun _ -> upper` in a nested lambda, which gives the same string for every element. A method or a member of a string that gives a number or a boolean, like `upper.StartsWith "A"` or `upper.Length`, only looks at what the string holds, and so does a string that goes to one place only, like `upper + upper`: those are made at each use, also in a nested lambda. A let outside such a lambda keeps its `Block` in `EvaluateQuotation` and `QuotationToExpression`; inside a lambda that goes to a provider it is replaced the same way whichever of them converts it: `EvaluateQuotation <@ people.Select(fun x -> let n = x.Name in n + n).ToArray() @>` reads `Name` twice. ### Array access uses ArrayIndex diff --git a/src/FSharp.Core/Linq.fs b/src/FSharp.Core/Linq.fs index e27b5f7d2ea..f87e40ca2fa 100644 --- a/src/FSharp.Core/Linq.fs +++ b/src/FSharp.Core/Linq.fs @@ -639,40 +639,89 @@ module LeafExpressionConverter = let makesNewObject (expr: Expression) = not expr.Type.IsValueType && (expr :? MethodCallExpression || expr :? BinaryExpression) - /// Finds the uses of a variable that can tell one object from another of the same value: everything but the object - /// being called on, or passed to, a method of a string, number or date, and the operands of an operator of one. + /// Whether the expression may give the object that the variable holds: the variable, a method or an operator of a + /// string, number or date that gives an object, which can be the one it got ('Substring 0', 'Trim' and '+ ""' do give + /// it back), and a choice or a block of those + let rec mayGiveObjectOf (variable: ParameterExpression) (expr: Expression) = + match expr with + | null -> false + | :? ParameterExpression as parameter -> obj.ReferenceEquals (parameter, variable) + | :? MethodCallExpression as call when isPlainValueType call.Method.DeclaringType && not call.Type.IsValueType -> + mayGiveObjectOf variable call.Object || call.Arguments |> Seq.exists (mayGiveObjectOf variable) + | :? BinaryExpression as operation when + not operation.Type.IsValueType + && (match operation.Method with + | null -> false + | operator -> isPlainValueType operator.DeclaringType) + -> + mayGiveObjectOf variable operation.Left || mayGiveObjectOf variable operation.Right + | :? UnaryExpression as conversion when isUpcast conversion -> mayGiveObjectOf variable conversion.Operand + | :? ConditionalExpression as choice -> mayGiveObjectOf variable choice.IfTrue || mayGiveObjectOf variable choice.IfFalse + | :? BlockExpression as block -> mayGiveObjectOf variable block.Result + | _ -> false + + /// Finds where the object of a variable goes to a place that can tell one object from another of the same value, and + /// counts those places. A method, an operator or a member of a string, number or date only looks at what the object + /// holds when it gives a number or a boolean; when it gives an object, that can be the one it got, and then the + /// place is where that object goes in turn. Every other place can tell: a method of the user's, a constructor, an + /// argument of 'ReferenceEquals', the result of a lambda, a variable. type ObjectIdentityUses (variable: ParameterExpression) = inherit ExpressionVisitor () - member val Found = false with get, set - - member private this.VisitOperands (operands: Expression list) = - for operand in operands do - match operand with - | null -> () - | operand when obj.ReferenceEquals (operand, variable) -> () - | operand -> this.Visit operand |> ignore + let parents = Stack () + let mutable lambdas = 0 + + /// The number of places that can tell one object from another + member val Places = 0 with get, set + /// One of them is in a lambda inside the scope of the variable, which runs as often as it is called + member val PlaceInLambda = false with get, set + + member private this.Place (count: int) = + this.Places <- this.Places + count + + if lambdas > 0 then + this.PlaceInLambda <- true + + /// Whether the object only goes on to the parent, or is only looked at there + member private this.GoesOn (parent: Expression) (child: Expression) = + match parent with + | :? MethodCallExpression as call -> isPlainValueType call.Method.DeclaringType + | :? BinaryExpression as operation -> + match operation.Method with + | null -> false + | operator -> isPlainValueType operator.DeclaringType + | :? MemberExpression as read -> isPlainValueType read.Member.DeclaringType + | :? UnaryExpression as conversion -> isUpcast conversion + | :? ConditionalExpression as choice -> not (obj.ReferenceEquals (choice.Test, child)) + | :? BlockExpression -> true + | _ -> false - override this.VisitMethodCall (node: MethodCallExpression) : Expression = - if isPlainValueType node.Method.DeclaringType then - this.VisitOperands (node.Object :: List.ofSeq node.Arguments) - node :> Expression - else - base.VisitMethodCall node + override this.Visit (node: Expression) : Expression = + match node with + | null -> null + | _ -> + if mayGiveObjectOf variable node then + if parents.Count = 0 then + this.Place 1 + else + let parent = parents.Peek () - override this.VisitBinary (node: BinaryExpression) : Expression = - match node.Method with - | null -> base.VisitBinary node - | operator when isPlainValueType operator.DeclaringType -> - this.VisitOperands [ node.Left; node.Right ] - node :> Expression - | _ -> base.VisitBinary node + if not (this.GoesOn parent node) then + match parent with + // The object is kept in a variable, which can be used any number of times + | :? BinaryExpression as assignment when assignment.NodeType = ExpressionType.Assign -> this.Place 2 + | _ -> this.Place 1 - override this.VisitParameter (node: ParameterExpression) : Expression = - if obj.ReferenceEquals (node, variable) then - this.Found <- true + parents.Push node + let visited = base.Visit node + parents.Pop () |> ignore + visited - node :> Expression + override this.VisitLambda<'T> (node: Expression<'T>) : Expression = + lambdas <- lambdas + 1 + let visited = base.VisitLambda node + lambdas <- lambdas - 1 + visited /// Convert F# quotations to LINQ expression trees. /// A more polished LINQ-Quotation translator will be published @@ -1069,12 +1118,13 @@ module LeafExpressionConverter = let replaced = reads.Visit bodyP if isReadValue then - // The object is made anew at every use: with more than one, it takes uses that look at what - // it holds, and nothing that can tell one object from another + // The object is made anew at every use, and a lambda inside the scope makes it as often as it + // runs: it can go to one place that tells one object from another, outside a lambda, and + // to any number of places that only look at what it holds let identity = ObjectIdentityUses vP identity.Visit bodyP |> ignore - if reads.Count <= 1 || not identity.Found then + if identity.Places = 0 || (identity.Places = 1 && not identity.PlaceInLambda) then ValueSome (substituteValue ()) else ValueNone diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs index 1a6616a3212..c0c28789047 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs @@ -572,6 +572,90 @@ let ``String made by a call is made at each use that only looks at it`` () = Assert.False(hasBlock lambda) Assert.True(lambda.Compile().Invoke "ab") +[] +let ``String made by a call is one string where a method of the string hands it on`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun s -> + let upper = s.ToUpperInvariant() + obj.ReferenceEquals(upper.Substring(0), upper.Substring(0))) @> + + // Substring 0 gives the string it is called on + Assert.True(hasBlock lambda) + Assert.True(lambda.Compile().Invoke "ab") + +[] +let ``String made by a call is one string where Clone hands it on`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun s -> + let upper = s.ToUpperInvariant() + obj.ReferenceEquals(upper.Clone(), upper.Clone())) @> + + Assert.True(hasBlock lambda) + Assert.True(lambda.Compile().Invoke "ab") + +[] +let ``String made by a call is one string for every element of a nested lambda that hands it on`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func>(fun s -> + let upper = s.ToUpperInvariant() + System.Linq.Enumerable.Select([| 1; 2 |], Func(fun _ -> upper))) @> + + let items = Seq.toArray (lambda.Compile().Invoke "ab") + + Assert.True(hasBlock lambda) + Assert.True(obj.ReferenceEquals(items.[0], items.[1])) + +[] +let ``String made by a call is made for every element of a nested lambda that only looks at it`` () = + let names = Collections.Generic.List([ "a"; "AB"; "b"; "AB" ]) + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun s -> + let upper = s.ToUpperInvariant() + System.Linq.Enumerable.Count(names, Func(fun n -> n.StartsWith(upper)))) @> + + Assert.False(hasBlock lambda) + Assert.Equal(2, lambda.Compile().Invoke "ab") + +[] +let ``String made by a call is one string where an operator hands it on`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun s -> + let upper = s.ToUpperInvariant() + obj.ReferenceEquals(upper + "", upper + "")) @> + + // Adding an empty string gives the string it is added to + Assert.True(hasBlock lambda) + Assert.True(lambda.Compile().Invoke "ab") + +[] +let ``String made by a call is made at each use where a method gives only a number or a boolean of a string made from it`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun s -> + let upper = s.ToUpperInvariant() + upper.Trim().Length > 1 && upper.Substring(1).Length < 9 && upper.StartsWith "A") @> + + Assert.False(hasBlock lambda) + Assert.True(lambda.Compile().Invoke "ab") + +[] +let ``String made by a call is one string where one use hands it on and another only looks at it`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func(fun s -> + let upper = s.ToUpperInvariant() + upper, upper.StartsWith "A") @> + + // Only one place can tell one object from another + Assert.False(hasBlock lambda) + Assert.Equal(("AB", true), lambda.Compile().Invoke "ab") + /// A parameter whose name counts how often it is read type CountedPerson() = let mutable nameReads = 0 From 6df150ce26ca246209304eee4ca489d69b1f2d1a Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sun, 27 Sep 2026 20:37:00 +0200 Subject: [PATCH 17/19] Quote lambdas that read a parameter of the lambda they are in A lambda passed as Expression> inside another lambda that goes to a LINQ provider - db.Orders.Any(fun o -> o.CustomerId = c.Id) in a query where, a second for over db.Orders.Where(..) - reached the provider as a call that converts the inner lambda's quotation when the tree runs, with the values of the variables it captures: QuotationToLambdaExpression(SubstHelper(, [c], [c])). A provider does not run the tree, so c never gets a value: EF Core failed on it and Cosmos rejected it. Such a lambda is now converted with the outer one and quoted, which is what the C# compiler emits for a nested expression lambda, so the provider sees an ordinary subquery whose parameter it knows. A nested lambda that reads no parameter of the outer lambda keeps the call. It depends on nothing the provider has to translate, so EF Core and Cosmos evaluate it before translating, as they do any captured value, and a method of the user's own that takes an expression keeps working inside a query. A nested lambda that cannot be converted keeps the call as well, whatever the conversion fails with: the call fails the same way, and only if the tree gets to run it. The scenario check stops reporting a call that a provider computes before it translates - one that holds no parameter, lambda, query or statement - and checks the lambda such a call gives. It also looks at the operator chains inside lambdas for members read from tuples built without naming them, which were only reported through the call before. Fixes #2758 Co-Authored-By: Claude Opus 5.5 --- docs/release-notes/.FSharp.Core/11.0.200.md | 6 ++ .../release-notes/PENDING_BREAKING_CHANGES.md | 12 +++ src/FSharp.Core/Linq.fs | 53 ++++++++++ .../ExpressionLambdaConversionTests.fs | 96 +++++++++++++++++++ .../QueryMethodSyntaxScenarios.fs | 13 ++- .../QueryProviderShape.fs | 66 ++++++++++++- .../QueryStructureScenarios.fs | 58 ++++++++++- 7 files changed, 294 insertions(+), 10 deletions(-) diff --git a/docs/release-notes/.FSharp.Core/11.0.200.md b/docs/release-notes/.FSharp.Core/11.0.200.md index 0e465a050d5..cdb9b9cbac1 100644 --- a/docs/release-notes/.FSharp.Core/11.0.200.md +++ b/docs/release-notes/.FSharp.Core/11.0.200.md @@ -8,5 +8,11 @@ * Quotation debug text from `Expr.ToString()`, `Expr.ToString(full)`, and `%A` now prints struct tuple construction as `NewStructTuple`; reference tuples remain `NewTuple`. ([Issue #20800](https://github.com/dotnet/fsharp/issues/20800), [PR #20814](https://github.com/dotnet/fsharp/pull/20814)) * Fix struct tuples in query expressions: selecting one no longer fails with "unreachable", and the items of one that a later operator reads are taken from its fields instead of failing with `ArgumentException: Struct tuples store their elements in fields, not properties`. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) * Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member, a sum, difference, product or quotient (by a constant) of members, or is read once now produce plain expressions. A `join` whose key one side builds as a tuple and the other side has ready-made, such as `join p in ps on ((k.KA, k.KB) = p.Pair)`, no longer fails with "Type mismatch when building 'ty'". ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [Issue #3782](https://github.com/dotnet/fsharp/issues/3782) for records with fields out of order, [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) +* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) +* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member, a sum, difference or product of members, or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) +* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) +* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332)) +* Fix a lambda nested in another lambda that goes to a LINQ provider, such as `db.Orders.Any(fun o -> o.CustomerId = c.Id)` inside a query `where`: when it reads a parameter of the outer lambda, it becomes a quoted lambda, as in C#, instead of a run-time call to `QuotationToLambdaExpression` that no provider can translate. ([Issue #2758](https://github.com/dotnet/fsharp/issues/2758)) +* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522)) ### Changed diff --git a/docs/release-notes/PENDING_BREAKING_CHANGES.md b/docs/release-notes/PENDING_BREAKING_CHANGES.md index 272975bd809..4d6a87b6b94 100644 --- a/docs/release-notes/PENDING_BREAKING_CHANGES.md +++ b/docs/release-notes/PENDING_BREAKING_CHANGES.md @@ -63,6 +63,18 @@ A record that a `query` selects with its fields written out of order is converte Every other let keeps its `Block` and is evaluated once, where it is written: a value that is computed by a call and read more than once, by a nested lambda or after something else has run, a struct of your own or a boxed value, a `mutable` variable, and every let of a lambda that assigns, sets a property or field, or runs a statement. A value that a call makes anew, like the `let upper = p.Name.ToUpper()` above, is kept as well when the string goes to more than one place that can tell one object from another, or to one in a nested lambda: `obj.ReferenceEquals(upper, upper)` does, and so do `upper.Substring(0)` and `upper.Trim()`, which can give back the string they are called on, and `fun _ -> upper` in a nested lambda, which gives the same string for every element. A method or a member of a string that gives a number or a boolean, like `upper.StartsWith "A"` or `upper.Length`, only looks at what the string holds, and so does a string that goes to one place only, like `upper + upper`: those are made at each use, also in a nested lambda. A let outside such a lambda keeps its `Block` in `EvaluateQuotation` and `QuotationToExpression`; inside a lambda that goes to a provider it is replaced the same way whichever of them converts it: `EvaluateQuotation <@ people.Select(fun x -> let n = x.Name in n + n).ToArray() @>` reads `Name` twice. +### Nested lambdas that read a parameter of the outer lambda are quoted + +Inside a lambda handed to a LINQ provider, a lambda passed as `Expression>` that reads a parameter of the outer lambda is now a `Quote` of the converted lambda, as in C#, instead of a call that converts its quotation when the outer tree runs. + +```fsharp +db.Customers.Where(fun c -> db.Orders.Any(fun o -> o.CustomerId = c.Id)) +// Was: c => Orders.Any(QuotationToLambdaExpression(SubstHelper(, [c], [c]))) +// Now: c => Orders.Any(o => o.CustomerId == c.Id) +``` + +**Impact:** When such a tree is compiled and run, the nested lambda reads the outer parameter through a closure where it used to get its value as a constant. A let of the outer lambda that names a member of its parameter is read by the nested lambda when that lambda runs. A nested lambda that reads no parameter of the outer lambda is converted as before, and so is one that cannot be converted. + ### Array access uses ArrayIndex ```fsharp diff --git a/src/FSharp.Core/Linq.fs b/src/FSharp.Core/Linq.fs index f87e40ca2fa..4a89608f18c 100644 --- a/src/FSharp.Core/Linq.fs +++ b/src/FSharp.Core/Linq.fs @@ -405,6 +405,8 @@ module LeafExpressionConverter = let (|MemberInitializationHelperQ|_|) = (|SpecificCallToMethod|_|) (methodhandleof MemberInitializationHelper) let (|NewAnonymousObjectHelperQ|_|) = (|SpecificCallToMethod|_|) (methodhandleof NewAnonymousObjectHelper) let (|ArrayLookupQ|_|) = (|SpecificCallToMethod|_|) (methodhandleof (fun (x, y) -> LanguagePrimitives.IntrinsicFunctions.GetArray x y)) + // QuotationToLambdaExpression is defined after the converter, so it cannot be named with methodhandleof here. + let (|QuotationToLambdaExpressionQ|_|) = SpecificCallToMethodInfo (typeof.DeclaringType.GetMethod "QuotationToLambdaExpression") //let (|ArrayAssignQ|_|) = (|SpecificCallToMethod|_|) (methodhandleof (fun -> LanguagePrimitives.IntrinsicFunctions.SetArray : int array -> int -> int -> unit)) //let (|ArrayTypeQ|_|) (ty:System.Type) = if ty.IsArray && ty.GetArrayRank() = 1 then Some (ty.GetElementType()) else None @@ -723,6 +725,44 @@ module LeafExpressionConverter = lambdas <- lambdas - 1 visited + /// Finds a parameter that an expression reads without being the one to bind it. The variables of the lets + /// around stand for values, and are no such parameters. + type FreeParameterFinder (letVariables: ParameterExpression list) = + inherit ExpressionVisitor () + + let bound = HashSet (letVariables, HashIdentity.Reference) + + member val Found = false with get, set + + override _.VisitLambda<'T> (node: Expression<'T>) : Expression = + for parameter in node.Parameters do + bound.Add parameter |> ignore + + base.VisitLambda node + + override _.VisitBlock (node: BlockExpression) : Expression = + for variable in node.Variables do + bound.Add variable |> ignore + + base.VisitBlock node + + override this.VisitParameter (node: ParameterExpression) : Expression = + if not (bound.Contains node) then + this.Found <- true + + node :> Expression + + /// Whether the lambda reads a parameter of a lambda in the tree being built, directly or through a 'let' whose + /// value took its place. A lambda inside such a value has parameters of its own, as the witness of 'abs' does. + let usesEnclosingParameter (env: ConvEnv) (lambda: Expr) = + lambda.GetFreeVars() + |> Seq.exists (fun v -> + match env.varEnv.TryFind v with + | Some bound -> + let finder = FreeParameterFinder env.letVariables + finder.Visit bound |> ignore + finder.Found + | None -> false) /// Convert F# quotations to LINQ expression trees. /// A more polished LINQ-Quotation translator will be published /// concert with later versions of LINQ. @@ -939,6 +979,19 @@ module LeafExpressionConverter = let substituteLets = (env.substituteLets || not env.inExpressionLambda) && not (hasStatements x1) ConvExprToLinqInContext { env with inExpressionLambda = true; substituteLets = substituteLets } x1 + // The compiler turns a lambda passed where Expression is expected into a call that converts + // its quotation when the tree runs, with the values of the variables it captures. A provider does + // not run the tree, so a nested lambda that reads a parameter of the tree never gets a value: + // that one is quoted, as C# does. Any other keeps the call, which a provider can evaluate, and so + // does one that cannot be converted, whatever the conversion fails with: its call fails the same + // way, and only when the tree gets to run it. + | QuotationToLambdaExpressionQ (None, _, [ QuoteTyped (NewDelegate _ as lambda) ]) + when env.inExpressionLambda && usesEnclosingParameter env lambda -> + try + Expression.Quote(ConvExprToLinqInContext env lambda) |> asExpr + with _ -> + Expression.Call(minfo, ConvExprsToLinq env args) |> asExpr + // Use witnesses if they are available | CallWithWitnesses (objArgOpt, _, minfo2, witnessArgs, args) -> let fullArgs = witnessArgs @ args diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs index c0c28789047..0668ad00dd5 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs @@ -7,6 +7,9 @@ open System.Linq.Expressions open Xunit open Microsoft.FSharp.Linq.RuntimeHelpers +type LinqExpr = + static member FromFunc<'T, 'U>(e: Expression>) = e + type private Counter() = let mutable calls = 0 member _.Calls = calls @@ -54,6 +57,55 @@ let ``Assigned mutable variable stays a block in an expression lambda`` () = Assert.Equal(ExpressionType.Block, (bodyOf lambda).NodeType) Assert.Equal(6, lambda.Compile().Invoke 5) +[] +let ``Lambda nested in an expression lambda is quoted`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func>>(fun x -> LinqExpr.FromFunc(fun y -> y + x)) @> + + match bodyOf lambda with + | :? MethodCallExpression as call -> + Assert.Equal(nameof LinqExpr.FromFunc, call.Method.Name) + Assert.Equal(ExpressionType.Quote, call.Arguments.[0].NodeType) + Assert.Equal(ExpressionType.Lambda, (call.Arguments.[0] :?> UnaryExpression).Operand.NodeType) + | body -> Assert.Fail $"Expected a call of FromFunc, got {body}" + + Assert.Equal(3, lambda.Compile().Invoke(1).Compile().Invoke 2) + +[] +let ``Nested lambda that reads no parameter of the tree stays a call`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func>>(fun _ -> LinqExpr.FromFunc(fun y -> y + 1)) @> + + match bodyOf lambda with + | :? MethodCallExpression as call -> Assert.Equal(ExpressionType.Call, call.Arguments.[0].NodeType) + | body -> Assert.Fail $"Expected a call of FromFunc, got {body}" + + Assert.Equal(3, lambda.Compile().Invoke(0).Compile().Invoke 2) + +[] +let ``Let read only by a nested lambda is evaluated once for each outer element`` () = + let counter = Counter() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func>>(fun _ -> let n = counter.Next() in LinqExpr.FromFunc(fun y -> y + n)) @> + + let add = lambda.Compile().Invoke(0).Compile() + Assert.Equal(3, add.Invoke 2) + Assert.Equal(4, add.Invoke 3) + Assert.Equal(1, counter.Calls) + +[] +let ``Nested lambda in an evaluated quotation gets the values it captures`` () = + let inner = + LeafExpressionConverter.EvaluateQuotation <@ let x = 5 in LinqExpr.FromFunc(fun y -> y + x) @> :?> Expression> + + match inner.Body with + | :? BinaryExpression as sum -> Assert.Equal(ExpressionType.Constant, sum.Right.NodeType) + | body -> Assert.Fail $"Expected a sum, got {body}" + + Assert.Equal(7, inner.Compile().Invoke 2) + [] let ``EvaluateQuotation still evaluates a let once`` () = let counter = Counter() @@ -478,6 +530,50 @@ let ``Let of an arithmetic on a call keeps its block and runs the call once`` () Assert.Equal(8, run lambda 3) Assert.Equal(1, counter.Calls) +[] +let ``Let read by a nested lambda that also reads the parameter is evaluated once for each outer element`` () = + let counter = Counter() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func>>(fun x -> let n = counter.Next() in LinqExpr.FromFunc(fun y -> y + n + x)) @> + + let add = lambda.Compile().Invoke(0).Compile() + Assert.Equal(3, add.Invoke 2) + Assert.Equal(4, add.Invoke 3) + Assert.Equal(1, counter.Calls) + +[] +let ``Lambda inside the value of a let does not make a nested lambda read the tree`` () = + let counter = Counter() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func>>(fun _ -> let n = abs (counter.Next()) in LinqExpr.FromFunc(fun y -> y + n)) @> + + match bodyOf lambda with + | :? MethodCallExpression as call -> Assert.Equal(ExpressionType.Call, call.Arguments.[0].NodeType) + | body -> Assert.Fail $"Expected a call of FromFunc, got {body}" + + let add = lambda.Compile().Invoke(0).Compile() + Assert.Equal(3, add.Invoke 2) + Assert.Equal(4, add.Invoke 3) + Assert.Equal(1, counter.Calls) + +[] +let ``Nested lambda that cannot be converted fails only where it is reached`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func>>(fun x -> + if x < 0 then + LinqExpr.FromFunc(fun y -> (try y = x with _ -> false)) + else + LinqExpr.FromFunc(fun y -> y = x)) + @> + + Assert.True(lambda.Compile().Invoke(1).Compile().Invoke 1) + [] let ``Record fields written out of order are evaluated as written in an expression lambda`` () = let counter = Counter() diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs index 4ac90358101..6b5e3a92303 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs @@ -137,9 +137,16 @@ let cases () = "Where with a lambda over another source" (fun db -> db.Customers.Where(fun c -> db.Orders.Any(fun o -> o.CustomerId = c.Id))) (customers >> Seq.filter (fun c -> c.Orders.Count > 0)) - |> Scenario.violating - [ Violation.RuntimeHelperCall "LeafExpressionConverter.QuotationToLambdaExpression" - Violation.RuntimeHelperCall "LeafExpressionConverter.SubstHelper" ] + + Scenario.rows + "Select with a lambda over another source" + (fun db -> db.Customers.Select(fun c -> (c.Name, db.Orders.Where(fun o -> o.CustomerId = c.Id).Sum(fun o -> o.Total)))) + (customers >> Seq.map (fun c -> c.Name, c.Orders |> Seq.sumBy _.Total)) + + Scenario.rows + "SelectMany with a lambda over another source" + (fun db -> db.Customers.SelectMany(fun c -> db.Orders.Where(fun o -> o.CustomerId = c.Id) :> seq).Select(fun o -> o.Id)) + (fun data -> data.Orders |> Seq.map _.Id) Scenario.rows "OrderBy, ThenByDescending" diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs index 8709087cd59..fc04ebab2dd 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs @@ -157,6 +157,53 @@ module ProviderShape = | "GroupJoin" -> true | _ -> false) + /// Finds what keeps a provider from computing an expression before it translates: a parameter, a lambda, a + /// query, a statement, or a value that is different every time it is asked for. + type private TranslatedPartFinder() = + inherit ExpressionVisitor() + + member val Found = false with get, set + + override this.VisitLambda<'T>(node: Expression<'T>) = + this.Found <- true + node + + override this.VisitParameter node = + this.Found <- true + node + + override this.VisitMember node = + let owner = node.Member.DeclaringType + + if + (owner = typeof || owner = typeof) + && (node.Member.Name = "Now" || node.Member.Name = "UtcNow" || node.Member.Name = "Today") + then + this.Found <- true + + base.VisitMember node + + override this.VisitMethodCall node = + if node.Method.DeclaringType = typeof && node.Method.Name = "NewGuid" then + this.Found <- true + + base.VisitMethodCall node + + override this.Visit node = + match node with + | null -> null + | node -> + if isStatement node.NodeType || typeof.IsAssignableFrom node.Type then + this.Found <- true + + base.Visit node + + /// A provider computes such an expression before it translates, as it does a captured value. + let private isComputedByProvider (node: Expression) = + let finder = TranslatedPartFinder() + finder.Visit node |> ignore + not finder.Found + type private ShapeVisitor() = inherit ExpressionVisitor() @@ -233,13 +280,28 @@ module ProviderShape = base.VisitNew node override this.VisitMethodCall node = - if node.Method.DeclaringType.Assembly = fsharpCore then + let inFSharpCore = node.Method.DeclaringType.Assembly = fsharpCore + // Whatever a call the provider computes holds is run, not translated + let computed = isComputedByProvider node + + if inFSharpCore && not computed then found.Add(Violation.RuntimeHelperCall $"{node.Method.DeclaringType.Name}.{node.Method.Name}") if collectionSelectorDepth > 0 && flattensOrJoins node then found.Add(Violation.NestedInCollectionSelector node.Method.Name) - if isLinqOperator node && node.Method.Name = "SelectMany" then + if computed then + // A lambda it gives is then part of what the provider translates + if typeof.IsAssignableFrom node.Type then + try + match Expression.Lambda(node).Compile().DynamicInvoke() with + | :? Expression as produced -> this.Visit produced |> ignore + | _ -> () + with error -> + found.Add(Violation.Throws(error.GetBaseException().GetType().Name)) + + node + elif isLinqOperator node && node.Method.Name = "SelectMany" then this.Visit node.Object |> ignore this.Visit node.Arguments.[0] |> ignore collectionSelectorDepth <- collectionSelectorDepth + 1 diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs index 6f806f4bb43..019ae751e85 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs @@ -2,11 +2,12 @@ module FSharp.Core.UnitTests.Linq.QueryStructureScenarios +open System open System.Linq +open System.Linq.Expressions -let private quotedLambda = - [ Violation.RuntimeHelperCall "LeafExpressionConverter.QuotationToLambdaExpression" - Violation.RuntimeHelperCall "LeafExpressionConverter.SubstHelper" ] +type Compiled = + static member Run(lambda: Expression>, argument: int) = lambda.Compile().Invoke argument let private nested operators = operators |> List.map Violation.NestedInCollectionSelector @@ -30,6 +31,11 @@ let private ordersWithLines (data: Data) = let private customersWithOrders (data: Data) = seq { for c in data.Customers do for o in c.Orders -> c.Name, o.Id } +let private absWithItsWitness = + [ Violation.RuntimeHelperCall "Operators.Abs$W" + Violation.RuntimeHelperCall "FuncConvert.ToFSharpFunc" + Violation.RuntimeHelperCall "Operators.Abs" ] + let private leftJoinedOrders (data: Data) = seq { for c in data.Customers do for o in orDefault c.Orders -> c.Name, idOrNone o } @@ -185,13 +191,55 @@ let private subqueries () = "second for over a source filtered by a lambda" (fun db -> query { for c in db.Customers do for o in db.Orders.Where(fun o -> o.CustomerId = c.Id) do select (c.Name, o.Id) }) customersWithOrders - |> Scenario.violating quotedLambda Scenario.rows "where with a lambda over another source" (fun db -> query { for c in db.Customers do where (db.Orders.Any(fun o -> o.CustomerId = c.Id)); select c.Name }) (fun data -> seq { for c in data.Customers do if c.Orders.Count > 0 then yield c.Name }) - |> Scenario.violating quotedLambda ] + + Scenario.rows + "where with All over another source" + (fun db -> query { for c in db.Customers do where (db.Orders.Where(fun o -> o.CustomerId = c.Id).All(fun o -> o.Total > 1.0)); select c.Name }) + (fun data -> seq { for c in data.Customers do if c.Orders |> Seq.forall (fun o -> o.Total > 1.0) then yield c.Name }) + + Scenario.rows + "select with Count over another source" + (fun db -> query { for c in db.Customers do select (c.Name, db.Orders.Count(fun o -> o.CustomerId = c.Id)) }) + (fun data -> seq { for c in data.Customers -> c.Name, c.Orders.Count }) + + Scenario.rows + "lambda over another source capturing a let" + (fun db -> query { for c in db.Customers do where (let id = c.Id in db.Orders.Any(fun o -> o.CustomerId = id)); select c.Name }) + (fun data -> seq { for c in data.Customers do if c.Orders.Count > 0 then yield c.Name }) + + Scenario.rows + "lambda nested two levels deep" + (fun db -> + query { + for c in db.Customers do + where (db.Orders.Any(fun o -> o.CustomerId = c.Id && db.Lines.Any(fun l -> l.OrderId = o.Id && l.Quantity > 1))) + select c.Name + }) + (fun data -> seq { for c in data.Customers do if c.Orders |> Seq.exists (fun o -> o.Lines |> Seq.exists (fun l -> l.Quantity > 1)) then yield c.Name }) + + Scenario.rows + "where with a method whose lambda reads nothing of the query" + (fun db -> query { for c in db.Customers do where (c.Age > Compiled.Run((fun x -> x + 1), 26)); select c.Name }) + (fun data -> seq { for c in data.Customers do if c.Age > 27 then yield c.Name }) + + // What a provider computes before it translates is checked too: it holds a lambda the provider cannot + // run, or gives one whose body the provider then has to translate. + Scenario.rows + "where with abs of a constant" + (fun db -> query { for c in db.Customers do where (c.Age > abs (-27)); select c.Name }) + (fun data -> seq { for c in data.Customers do if c.Age > 27 then yield c.Name }) + |> Scenario.violating absWithItsWitness + + Scenario.rows + "where with a lambda that reads nothing of the query and calls abs" + (fun db -> query { for c in db.Customers do where (c.Age > 26 && db.Orders.Any(fun o -> abs (o.Total - 10.0) < 6.0)); select c.Name }) + (fun data -> seq { for c in data.Customers do if c.Age > 26 && data.Orders |> Seq.exists (fun o -> abs (o.Total - 10.0) < 6.0) then yield c.Name }) + |> Scenario.violating absWithItsWitness ] let private leftJoins () = [ Scenario.rows From a026e19c8a43d7e4f0019674cb84ff53b5ee2329 Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sun, 1 Nov 2026 22:15:20 +0100 Subject: [PATCH 18/19] Link the release note to the pull request Co-Authored-By: Claude Sonnet 5.5 --- docs/release-notes/.FSharp.Core/11.0.200.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes/.FSharp.Core/11.0.200.md b/docs/release-notes/.FSharp.Core/11.0.200.md index cdb9b9cbac1..8c28700239f 100644 --- a/docs/release-notes/.FSharp.Core/11.0.200.md +++ b/docs/release-notes/.FSharp.Core/11.0.200.md @@ -12,7 +12,7 @@ * Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member, a sum, difference or product of members, or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) * Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) * Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332)) -* Fix a lambda nested in another lambda that goes to a LINQ provider, such as `db.Orders.Any(fun o -> o.CustomerId = c.Id)` inside a query `where`: when it reads a parameter of the outer lambda, it becomes a quoted lambda, as in C#, instead of a run-time call to `QuotationToLambdaExpression` that no provider can translate. ([Issue #2758](https://github.com/dotnet/fsharp/issues/2758)) +* Fix a lambda nested in another lambda that goes to a LINQ provider, such as `db.Orders.Any(fun o -> o.CustomerId = c.Id)` inside a query `where`: when it reads a parameter of the outer lambda, it becomes a quoted lambda, as in C#, instead of a run-time call to `QuotationToLambdaExpression` that no provider can translate. ([Issue #2758](https://github.com/dotnet/fsharp/issues/2758), [PR #20896](https://github.com/dotnet/fsharp/pull/20896)) * Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522)) ### Changed From 84abac23c8bb75f884e6056380567deb4fbbfa81 Mon Sep 17 00:00:00 2001 From: XperiAndri Date: Sun, 1 Nov 2026 23:36:24 +0100 Subject: [PATCH 19/19] Check the selectors a provider computes, and cover nested lambdas that read a let A selector that the provider computes before it translates, which the compiler makes of a lambda that reads nothing of the tree, is evaluated and checked like a written one, and one that is not a lambda is passed over: the check no longer fails on `select (db.Orders.Select(fun o -> o.Id))`. Tests: that query as a scenario, a hand-built tree with a computed predicate, and a nested lambda that reads a let of arithmetic, which is quoted, and one that reads a let of arithmetic on a call, which is evaluated once. Co-Authored-By: Claude Sonnet 5.5 --- docs/release-notes/.FSharp.Core/11.0.200.md | 5 -- .../ExpressionLambdaConversionTests.fs | 88 ++++++++++++------- .../QueryProviderShape.fs | 20 +++++ .../Microsoft.FSharp.Linq/QueryShapeTests.fs | 17 ++++ .../QueryStructureScenarios.fs | 6 ++ 5 files changed, 100 insertions(+), 36 deletions(-) diff --git a/docs/release-notes/.FSharp.Core/11.0.200.md b/docs/release-notes/.FSharp.Core/11.0.200.md index 8c28700239f..44fb93c0a2e 100644 --- a/docs/release-notes/.FSharp.Core/11.0.200.md +++ b/docs/release-notes/.FSharp.Core/11.0.200.md @@ -8,11 +8,6 @@ * Quotation debug text from `Expr.ToString()`, `Expr.ToString(full)`, and `%A` now prints struct tuple construction as `NewStructTuple`; reference tuples remain `NewTuple`. ([Issue #20800](https://github.com/dotnet/fsharp/issues/20800), [PR #20814](https://github.com/dotnet/fsharp/pull/20814)) * Fix struct tuples in query expressions: selecting one no longer fails with "unreachable", and the items of one that a later operator reads are taken from its fields instead of failing with `ArgumentException: Struct tuples store their elements in fields, not properties`. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) * Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member, a sum, difference, product or quotient (by a constant) of members, or is read once now produce plain expressions. A `join` whose key one side builds as a tuple and the other side has ready-made, such as `join p in ps on ((k.KA, k.KB) = p.Pair)`, no longer fails with "Type mismatch when building 'ty'". ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [Issue #3782](https://github.com/dotnet/fsharp/issues/3782) for records with fields out of order, [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) -* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) -* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member, a sum, difference or product of members, or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) -* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332), [PR #20895](https://github.com/dotnet/fsharp/pull/20895)) -* Fix `let` bindings reaching LINQ providers as `Expression.Block`, which no provider translates: in lambdas handed to a provider, records and anonymous records with fields written out of order, tupled lambda parameters, struct copies such as `x.Nullable.HasValue`, and a `let` that only names a member or is read once now produce plain expressions. ([Issue #11131](https://github.com/dotnet/fsharp/issues/11131), [Issue #15648](https://github.com/dotnet/fsharp/issues/15648), [Issue #11332](https://github.com/dotnet/fsharp/issues/11332)) * Fix a lambda nested in another lambda that goes to a LINQ provider, such as `db.Orders.Any(fun o -> o.CustomerId = c.Id)` inside a query `where`: when it reads a parameter of the outer lambda, it becomes a quoted lambda, as in C#, instead of a run-time call to `QuotationToLambdaExpression` that no provider can translate. ([Issue #2758](https://github.com/dotnet/fsharp/issues/2758), [PR #20896](https://github.com/dotnet/fsharp/pull/20896)) -* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522)) ### Changed diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs index 0668ad00dd5..37eddb6488f 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/ExpressionLambdaConversionTests.fs @@ -437,6 +437,50 @@ let ``Record fields are not reordered next to a statement in a query over a quer Assert.Equal([ { First = 2; Second = 1 } ], rows) +[] +let ``Let read by a nested lambda that also reads the parameter is evaluated once for each outer element`` () = + let counter = Counter() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func>>(fun x -> let n = counter.Next() in LinqExpr.FromFunc(fun y -> y + n + x)) @> + + let add = lambda.Compile().Invoke(0).Compile() + Assert.Equal(3, add.Invoke 2) + Assert.Equal(4, add.Invoke 3) + Assert.Equal(1, counter.Calls) + +[] +let ``Lambda inside the value of a let does not make a nested lambda read the tree`` () = + let counter = Counter() + + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ Func>>(fun _ -> let n = abs (counter.Next()) in LinqExpr.FromFunc(fun y -> y + n)) @> + + match bodyOf lambda with + | :? MethodCallExpression as call -> Assert.Equal(ExpressionType.Call, call.Arguments.[0].NodeType) + | body -> Assert.Fail $"Expected a call of FromFunc, got {body}" + + let add = lambda.Compile().Invoke(0).Compile() + Assert.Equal(3, add.Invoke 2) + Assert.Equal(4, add.Invoke 3) + Assert.Equal(1, counter.Calls) + +[] +let ``Nested lambda that cannot be converted fails only where it is reached`` () = + let lambda = + LeafExpressionConverter.QuotationToLambdaExpression + <@ + Func>>(fun x -> + if x < 0 then + LinqExpr.FromFunc(fun y -> (try y = x with _ -> false)) + else + LinqExpr.FromFunc(fun y -> y = x)) + @> + + Assert.True(lambda.Compile().Invoke(1).Compile().Invoke 1) + [] let ``Let written by hand is evaluated before the fields that follow it`` () = let counter = Counter() @@ -531,48 +575,30 @@ let ``Let of an arithmetic on a call keeps its block and runs the call once`` () Assert.Equal(1, counter.Calls) [] -let ``Let read by a nested lambda that also reads the parameter is evaluated once for each outer element`` () = - let counter = Counter() - +let ``Lambda nested in an expression lambda is quoted when it reads a let of arithmetic on the parameter`` () = let lambda = LeafExpressionConverter.QuotationToLambdaExpression - <@ Func>>(fun x -> let n = counter.Next() in LinqExpr.FromFunc(fun y -> y + n + x)) @> - - let add = lambda.Compile().Invoke(0).Compile() - Assert.Equal(3, add.Invoke 2) - Assert.Equal(4, add.Invoke 3) - Assert.Equal(1, counter.Calls) - -[] -let ``Lambda inside the value of a let does not make a nested lambda read the tree`` () = - let counter = Counter() - - let lambda = - LeafExpressionConverter.QuotationToLambdaExpression - <@ Func>>(fun _ -> let n = abs (counter.Next()) in LinqExpr.FromFunc(fun y -> y + n)) @> + <@ Func>>(fun x -> let n = x + 1 in LinqExpr.FromFunc(fun y -> y + n)) @> match bodyOf lambda with - | :? MethodCallExpression as call -> Assert.Equal(ExpressionType.Call, call.Arguments.[0].NodeType) + | :? MethodCallExpression as call -> Assert.Equal(ExpressionType.Quote, call.Arguments.[0].NodeType) | body -> Assert.Fail $"Expected a call of FromFunc, got {body}" - let add = lambda.Compile().Invoke(0).Compile() - Assert.Equal(3, add.Invoke 2) - Assert.Equal(4, add.Invoke 3) - Assert.Equal(1, counter.Calls) + Assert.False(hasBlock lambda) + Assert.Equal(6, lambda.Compile().Invoke(2).Compile().Invoke 3) [] -let ``Nested lambda that cannot be converted fails only where it is reached`` () = +let ``Let of an arithmetic on a call read by a nested lambda is evaluated once for each outer element`` () = + let counter = Counter() + let lambda = LeafExpressionConverter.QuotationToLambdaExpression - <@ - Func>>(fun x -> - if x < 0 then - LinqExpr.FromFunc(fun y -> (try y = x with _ -> false)) - else - LinqExpr.FromFunc(fun y -> y = x)) - @> + <@ Func>>(fun x -> let n = counter.Next() + x in LinqExpr.FromFunc(fun y -> y + n)) @> - Assert.True(lambda.Compile().Invoke(1).Compile().Invoke 1) + let add = lambda.Compile().Invoke(10).Compile() + Assert.Equal(14, add.Invoke 3) + Assert.Equal(15, add.Invoke 4) + Assert.Equal(1, counter.Calls) [] let ``Record fields written out of order are evaluated as written in an expression lambda`` () = diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs index fc04ebab2dd..9edae187b70 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs @@ -374,6 +374,20 @@ module ProviderShape = | :? ConstantExpression as root -> obj.ReferenceEquals(root.Value, query) | _ -> false + /// The lambda that a call gives when the provider computes it before it translates, as it does for a selector + /// that reads nothing of the tree: a nested lambda that the compiler turned into a call. Nothing, if the call + /// fails. + let private computedLambda (node: Expression) = + if typeof.IsAssignableFrom node.Type && isComputedByProvider node then + try + match Expression.Lambda(node).Compile().DynamicInvoke() with + | :? LambdaExpression as lambda -> ValueSome lambda + | _ -> ValueNone + with _ -> + ValueNone + else + ValueNone + /// Finds members read from elements that an earlier operator built without naming them: the provider has to /// take apart an object it was not told how to. Every chain of operators is followed, the ones inside lambdas /// and the ones of a query captured from earlier too, and each parameter of a lambda is tied to the sequence @@ -390,6 +404,7 @@ module ProviderShape = if index < call.Arguments.Count then match call.Arguments.[index] with | Lambda lambda -> ValueSome lambda + | :? MethodCallExpression as computed -> computedLambda computed | _ -> ValueNone else ValueNone @@ -533,6 +548,11 @@ module ProviderShape = if isLinqOperator node then this.Check node + // The lambda such a call gives is part of what the provider translates + match computedLambda node with + | ValueSome lambda -> this.Visit lambda |> ignore + | ValueNone -> () + base.VisitMethodCall node // A query built earlier and used in this one is part of what the provider translates. diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs index a9cc9d9535f..a0a95f574db 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs @@ -8,6 +8,7 @@ open System.Linq open System.Linq.Expressions open System.Reflection open Xunit +open Microsoft.FSharp.Linq.RuntimeHelpers type QueryShapeTests() = @@ -35,6 +36,11 @@ type QueryShapeTests() = [] member _.``Method syntax``(scenario: Scenario) = Scenario.check scenario +/// Selectors that an expression gives when it is run, as the compiler turns a lambda nested in a lambda into a call +type ComputedSelectors = + static member PositiveItem1() : Expression> = + LeafExpressionConverter.QuotationToLambdaExpression <@ Func(fun (a, _) -> a > 0) @> + /// Trees built by hand, the way a C# compiler builds them, to check the validator itself. They do not depend on how /// FSharp.Core translates a query, so they mean the same in every branch. module private ValidatorTrees = @@ -72,6 +78,10 @@ module private ValidatorTrees = call "Select" [ typeof; orderPair ] [ orders.Expression; quote (Expression.Lambda(body, o)) ] + /// source.Where(ComputedSelectors.PositiveItem1()), a call that gives t => t.Item1 > 0 + let whereComputedItem1Positive (source: Expression) = + call "Where" [ orderPair ] [ source; Expression.Call(typeof.GetMethod "PositiveItem1") :> Expression ] + /// source.Where(t => t.Item1 > 0) let whereItem1Positive (source: Expression) (tuple: Type) = let t = Expression.Parameter(tuple, "t") @@ -149,6 +159,13 @@ type ProviderShapeValidatorTests() = Assert.Equal(unnamedRead, reported (capturing false)) Assert.Empty(reported (capturing true)) + [] + member _.``A selector that the provider computes is checked``() = + let computed named = ValidatorTrees.whereComputedItem1Positive (ValidatorTrees.orderPairs named) + + Assert.Equal(unnamedRead, reported (computed false)) + Assert.Empty(reported (computed true)) + [] member _.``An identity Select keeps the elements unnamed``() = let t = Expression.Parameter(ValidatorTrees.customerPair, "t") diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs index 019ae751e85..b5a7a944590 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs @@ -207,6 +207,12 @@ let private subqueries () = (fun db -> query { for c in db.Customers do select (c.Name, db.Orders.Count(fun o -> o.CustomerId = c.Id)) }) (fun data -> seq { for c in data.Customers -> c.Name, c.Orders.Count }) + // The selector of the inner query reads nothing of the element, so it stays a call that the provider computes + Scenario.rows + "select of another source projected without reading the element" + (fun db -> query { for c in db.Customers do select (db.Orders.Select(fun o -> o.Id)) }) + (fun data -> seq { for _ in data.Customers -> (seq { for o in data.Orders -> o.Id }).AsQueryable() }) + Scenario.rows "lambda over another source capturing a let" (fun db -> query { for c in db.Customers do where (let id = c.Id in db.Orders.Any(fun o -> o.CustomerId = id)); select c.Name })