From 4e3137a2a183f57118f9f8be7038cdc4c39cd428 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Wed, 12 Oct 2022 15:49:35 +0200 Subject: [PATCH 1/8] Generating cascading project --- .../FSharp.Compiler.Benchmarks.fsproj | 1 + .../FileCascadeBenchmarks.fs | 75 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs diff --git a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FSharp.Compiler.Benchmarks.fsproj b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FSharp.Compiler.Benchmarks.fsproj index 4bdb142e62b..d1187649935 100644 --- a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FSharp.Compiler.Benchmarks.fsproj +++ b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FSharp.Compiler.Benchmarks.fsproj @@ -17,6 +17,7 @@ + diff --git a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs new file mode 100644 index 00000000000..684aaa0ac86 --- /dev/null +++ b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs @@ -0,0 +1,75 @@ +namespace FSharp.Compiler.Benchmarks + +open System +open System.IO +open System.Text +open FSharp.Compiler.CodeAnalysis +open FSharp.Compiler.Diagnostics +open FSharp.Compiler.EditorServices +open FSharp.Compiler.Text +open FSharp.Compiler.AbstractIL.IL +open FSharp.Compiler.AbstractIL.ILBinaryReader +open BenchmarkDotNet.Attributes +open FSharp.Compiler.Benchmarks +open Microsoft.CodeAnalysis.Text + +[] +module private Helpers = + + let createProject files projectFilename = + { + ProjectFileName = projectFilename + ProjectId = None + SourceFiles = files + OtherOptions = [|"--optimize+" |] + ReferencedProjects = [||] + IsIncompleteTypeCheckEnvironment = false + UseScriptResolutionRules = false + LoadTime = DateTime() + UnresolvedReferences = None + OriginalLoadReferences = [] + Stamp = Some 0L (* set the stamp to 0L on each run so we don't evaluate the whole project again *) + } + + let baselineModule = + $""" +module Benchmark0 +let returnValue = 5 +let myFunc0 () = 5""" + + let generateSourceCode number = + $""" +module Benchmark%i{number} +open Benchmark%i{number-1} +let myFunc%i{number} () = myFunc%i{number-1}()""" + +[] +type FileCascadeBenchmarks() = + let mutable project : FSharpProjectOptions option = None + + let filesToCreate = 100 + + [] + member _.Setup() = + match project with + | Some p -> p + | None -> + let projectFolder = Directory.CreateDirectory(Path.Combine(Directory.GetCurrentDirectory(),"CascadeBenchmarkProject")) + if(projectFolder.Exists) then + do projectFolder.Delete(recursive=true) + do Directory.CreateDirectory(projectFolder.FullName) |> ignore + + let inProjectFolder fileName = Path.Combine(projectFolder.FullName,fileName) + + File.WriteAllText(inProjectFolder "Benchmark0.fs",baselineModule) + for i = 1 to filesToCreate do + File.WriteAllText(inProjectFolder $"Benchmark%i{i}.fs", generateSourceCode i) + + let dllFileName = inProjectFolder "CascadingBenchMark.dll" + let allSourceCodeFiles = [| for i in 0 .. 100 -> inProjectFolder $"Benchmark%i{i}.fs"|] + let x = createProject allSourceCodeFiles dllFileName + project <- Some x + x + + + From 8a8499c294a06a6060f5a5b22b03d40751985e67 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Wed, 12 Oct 2022 17:25:27 +0200 Subject: [PATCH 2/8] Benchmark attributes added --- .../FileCascadeBenchmarks.fs | 95 ++++++++++++++----- 1 file changed, 69 insertions(+), 26 deletions(-) diff --git a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs index 684aaa0ac86..746f81065b7 100644 --- a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs +++ b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs @@ -12,9 +12,11 @@ open FSharp.Compiler.AbstractIL.ILBinaryReader open BenchmarkDotNet.Attributes open FSharp.Compiler.Benchmarks open Microsoft.CodeAnalysis.Text +open BenchmarkDotNet.Order +open BenchmarkDotNet.Mathematics [] -module private Helpers = +module private CascadeProjectHelpers = let createProject files projectFilename = { @@ -41,35 +43,76 @@ let myFunc0 () = 5""" $""" module Benchmark%i{number} open Benchmark%i{number-1} -let myFunc%i{number} () = myFunc%i{number-1}()""" +let myFunc%i{number} () = myFunc%i{number-1}() +//$COMMENTAREA$""" [] +[] +[] +[] type FileCascadeBenchmarks() = - let mutable project : FSharpProjectOptions option = None + let mutable project : FSharpProjectOptions option = None - let filesToCreate = 100 + let getProject() = project.Value + + let checker = FSharpChecker.Create(projectCacheSize = 5, enableParallelCheckingWithSignatureFiles = true, parallelReferenceResolution = true) + let filesToCreate = 64 + + let mutable finalFileContents = SourceText.ofString "" - [] - member _.Setup() = - match project with - | Some p -> p - | None -> - let projectFolder = Directory.CreateDirectory(Path.Combine(Directory.GetCurrentDirectory(),"CascadeBenchmarkProject")) - if(projectFolder.Exists) then - do projectFolder.Delete(recursive=true) - do Directory.CreateDirectory(projectFolder.FullName) |> ignore - - let inProjectFolder fileName = Path.Combine(projectFolder.FullName,fileName) - - File.WriteAllText(inProjectFolder "Benchmark0.fs",baselineModule) - for i = 1 to filesToCreate do - File.WriteAllText(inProjectFolder $"Benchmark%i{i}.fs", generateSourceCode i) - - let dllFileName = inProjectFolder "CascadingBenchMark.dll" - let allSourceCodeFiles = [| for i in 0 .. 100 -> inProjectFolder $"Benchmark%i{i}.fs"|] - let x = createProject allSourceCodeFiles dllFileName - project <- Some x - x - + [] + member _.Setup() = + let projectFolder = Directory.CreateDirectory(Path.Combine(Directory.GetCurrentDirectory(),"CascadeBenchmarkProject")) + if(projectFolder.Exists) then + do projectFolder.Delete(recursive=true) + do Directory.CreateDirectory(projectFolder.FullName) |> ignore + + let inProjectFolder fileName = Path.Combine(projectFolder.FullName,fileName) + + File.WriteAllText(inProjectFolder "Benchmark0.fs",baselineModule) + for i = 1 to filesToCreate do + File.WriteAllText(inProjectFolder $"Benchmark%i{i}.fs", generateSourceCode i) + + let dllFileName = inProjectFolder "CascadingBenchMark.dll" + let allSourceCodeFiles = [| for i in 0 .. filesToCreate -> inProjectFolder $"Benchmark%i{i}.fs"|] + let x = createProject allSourceCodeFiles dllFileName + project <- Some x + finalFileContents <- generateSourceCode filesToCreate |> SourceText.ofString + + + member x.ChangeFile(fileIndex:int, action) = + let project = getProject() + let fileName = project.SourceFiles.[fileIndex] + let fullOriginalSource = File.ReadAllText(fileName) + try + File.WriteAllText(fileName, fullOriginalSource.Replace("$COMMENTAREA$","$FILEMODIFIED")) + action() + finally + File.WriteAllText(fileName,fullOriginalSource) + + member x.CheckFinalFile () = + let project = getProject() + let lastFile = project.SourceFiles |> Array.last + checker.ParseAndCheckFileInProject(lastFile,999,finalFileContents,project) + + [] + member x.ParseProjectAsIs() = + x.CheckFinalFile() + + [] + member x.ParseProjectWithFullCacheClear() = + checker.ClearCache([getProject()]) + checker.InvalidateConfiguration(getProject()) + x.CheckFinalFile() + + [] + member x.ParseProjectWithChangingFirstFile() = + x.ChangeFile(fileIndex=0, action= fun () -> x.CheckFinalFile()) + [] + member x.ParseProjectWithChanging25thPercentileFile() = + x.ChangeFile(fileIndex=filesToCreate/4, action= fun () -> x.CheckFinalFile()) + [] + member x.ParseProjectWithChangingMiddleFile() = + x.ChangeFile(fileIndex=filesToCreate/2, action= fun () -> x.CheckFinalFile()) \ No newline at end of file From 1fb219a2eb69d2f86674157578516164577552f1 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Thu, 13 Oct 2022 11:13:11 +0200 Subject: [PATCH 3/8] More agressive cache cleaning in benchmarks --- .../CompilerServiceBenchmarks/FileCascadeBenchmarks.fs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs index 746f81065b7..c836eb6c0da 100644 --- a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs +++ b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs @@ -101,6 +101,7 @@ type FileCascadeBenchmarks() = [] member x.ParseProjectWithFullCacheClear() = + checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() checker.ClearCache([getProject()]) checker.InvalidateConfiguration(getProject()) x.CheckFinalFile() From 5a7ad01d50b01bcf73385d74b18e198e5dd91494 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Thu, 13 Oct 2022 15:53:39 +0200 Subject: [PATCH 4/8] Better naming, waiting for async, further benchmark case --- .gitignore | 1 + VisualFSharp.sln | 21 ++++++++++++++++--- .../FileCascadeBenchmarks.fs | 21 ++++++++++++------- .../Fsharp.ProfilingStartpointProject.fsproj | 16 ++++++++++++++ .../Program.fs | 17 +++++++++++++++ 5 files changed, 65 insertions(+), 11 deletions(-) create mode 100644 tests/benchmarks/Fsharp.ProfilingStartpointProject/Fsharp.ProfilingStartpointProject.fsproj create mode 100644 tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs diff --git a/.gitignore b/.gitignore index 95851a4bc83..3cfeb6c1cb3 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,4 @@ nCrunchTemp_* /test.fsx tests/FSharp.Compiler.Service.Tests/FSharp.CompilerService.SurfaceArea.netstandard.actual +*.vsp diff --git a/VisualFSharp.sln b/VisualFSharp.sln index 4f4947d3491..343c976316e 100644 --- a/VisualFSharp.sln +++ b/VisualFSharp.sln @@ -184,7 +184,7 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "fsiAnyCpu", "src\fsi\fsiAny EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "fsiArm64", "src\fsi\fsiArm64Project\fsiArm64.fsproj", "{EB015235-1E07-4CDA-9CC6-3FBCC27910D1}" EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "HistoricalBenchmark", "tests\benchmarks\FCSBenchmarks\BenchmarkComparison\HistoricalBenchmark.fsproj", "{583182E1-3484-4A8F-AC06-7C0D232C0CA4}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "HistoricalBenchmark", "tests\benchmarks\FCSBenchmarks\BenchmarkComparison\HistoricalBenchmark.fsproj", "{583182E1-3484-4A8F-AC06-7C0D232C0CA4}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "FCSBenchmarks", "FCSBenchmarks", "{39CDF34B-FB23-49AE-AB27-0975DA379BB5}" ProjectSection(SolutionItems) = preProject @@ -193,6 +193,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "FCSBenchmarks", "FCSBenchma tests\benchmarks\FCSBenchmarks\SmokeTestAllBenchmarks.ps1 = tests\benchmarks\FCSBenchmarks\SmokeTestAllBenchmarks.ps1 EndProjectSection EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Fsharp.ProfilingStartpointProject", "tests\benchmarks\Fsharp.ProfilingStartpointProject\Fsharp.ProfilingStartpointProject.fsproj", "{FE23BB65-276A-4E41-8CC7-F7752241DEBA}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1019,6 +1021,18 @@ Global {583182E1-3484-4A8F-AC06-7C0D232C0CA4}.Release|Any CPU.Build.0 = Release|Any CPU {583182E1-3484-4A8F-AC06-7C0D232C0CA4}.Release|x86.ActiveCfg = Release|Any CPU {583182E1-3484-4A8F-AC06-7C0D232C0CA4}.Release|x86.Build.0 = Release|Any CPU + {FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|x86.ActiveCfg = Debug|Any CPU + {FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|x86.Build.0 = Debug|Any CPU + {FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|Any CPU.ActiveCfg = Debug|Any CPU + {FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|Any CPU.Build.0 = Debug|Any CPU + {FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|x86.ActiveCfg = Debug|Any CPU + {FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|x86.Build.0 = Debug|Any CPU + {FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|Any CPU.Build.0 = Release|Any CPU + {FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|x86.ActiveCfg = Release|Any CPU + {FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1089,6 +1103,7 @@ Global {B5A9BBD9-2F45-4722-A6CA-BAE3C64CD4E2} = {3881429D-A97A-49EB-B7AE-A82BA5FE9C77} {14F3D3D6-5C8E-43C2-98A2-17EA704D4DEA} = {CFE3259A-2D30-4EB0-80D5-E8B5F3D01449} {A422D673-8E3B-4924-821B-DD3174173426} = {4C7B48D7-19AF-4AE7-9D1D-3BB289D5480D} + {564E7DC5-11CB-4FCF-ABDD-23AD93AF3A61} = {39CDF34B-FB23-49AE-AB27-0975DA379BB5} {B1E30F2C-894F-47A9-9C8A-3324831E7D26} = {4C7B48D7-19AF-4AE7-9D1D-3BB289D5480D} {597D9896-4B90-4E9E-9C99-445C2CB9FF60} = {B8DDA694-7939-42E3-95E5-265C2217C142} {0973C362-585C-4838-9459-D7E45C6B784B} = {B8DDA694-7939-42E3-95E5-265C2217C142} @@ -1096,9 +1111,9 @@ Global {511C95D9-3BA6-451F-B6F8-F033F40878A5} = {B8DDA694-7939-42E3-95E5-265C2217C142} {37EB3E54-ABC6-4CF5-8273-7CE4B61A42C1} = {B8DDA694-7939-42E3-95E5-265C2217C142} {EB015235-1E07-4CDA-9CC6-3FBCC27910D1} = {B8DDA694-7939-42E3-95E5-265C2217C142} - {39CDF34B-FB23-49AE-AB27-0975DA379BB5} = {DFB6ADD7-3149-43D9-AFA0-FC4A818B472B} - {564E7DC5-11CB-4FCF-ABDD-23AD93AF3A61} = {39CDF34B-FB23-49AE-AB27-0975DA379BB5} {583182E1-3484-4A8F-AC06-7C0D232C0CA4} = {39CDF34B-FB23-49AE-AB27-0975DA379BB5} + {39CDF34B-FB23-49AE-AB27-0975DA379BB5} = {DFB6ADD7-3149-43D9-AFA0-FC4A818B472B} + {FE23BB65-276A-4E41-8CC7-F7752241DEBA} = {39CDF34B-FB23-49AE-AB27-0975DA379BB5} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {48EDBBBE-C8EE-4E3C-8B19-97184A487B37} diff --git a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs index c836eb6c0da..00fecc74ae8 100644 --- a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs +++ b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs @@ -90,30 +90,35 @@ type FileCascadeBenchmarks() = finally File.WriteAllText(fileName,fullOriginalSource) - member x.CheckFinalFile () = + member x.ParseAndCheckLastFileInTheProject () = let project = getProject() let lastFile = project.SourceFiles |> Array.last - checker.ParseAndCheckFileInProject(lastFile,999,finalFileContents,project) + checker.ParseAndCheckFileInProject(lastFile,999,finalFileContents,project) |> Async.RunSynchronously [] - member x.ParseProjectAsIs() = - x.CheckFinalFile() + member x.ParseAndCheckLastFileProjectAsIs() = + x.ParseAndCheckLastFileInTheProject() [] member x.ParseProjectWithFullCacheClear() = checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() checker.ClearCache([getProject()]) checker.InvalidateConfiguration(getProject()) - x.CheckFinalFile() + checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() + x.ParseAndCheckLastFileInTheProject() [] member x.ParseProjectWithChangingFirstFile() = - x.ChangeFile(fileIndex=0, action= fun () -> x.CheckFinalFile()) + x.ChangeFile(fileIndex=0, action= fun () -> x.ParseAndCheckLastFileInTheProject()) [] member x.ParseProjectWithChanging25thPercentileFile() = - x.ChangeFile(fileIndex=filesToCreate/4, action= fun () -> x.CheckFinalFile()) + x.ChangeFile(fileIndex=filesToCreate/4, action= fun () -> x.ParseAndCheckLastFileInTheProject()) [] member x.ParseProjectWithChangingMiddleFile() = - x.ChangeFile(fileIndex=filesToCreate/2, action= fun () -> x.CheckFinalFile()) \ No newline at end of file + x.ChangeFile(fileIndex=filesToCreate/2, action= fun () -> x.ParseAndCheckLastFileInTheProject()) + + [] + member x.ParseProjectWithChangingPenultimateFile() = + x.ChangeFile(fileIndex=(filesToCreate-2), action= fun () -> x.ParseAndCheckLastFileInTheProject()) \ No newline at end of file diff --git a/tests/benchmarks/Fsharp.ProfilingStartpointProject/Fsharp.ProfilingStartpointProject.fsproj b/tests/benchmarks/Fsharp.ProfilingStartpointProject/Fsharp.ProfilingStartpointProject.fsproj new file mode 100644 index 00000000000..7ed3d17abc9 --- /dev/null +++ b/tests/benchmarks/Fsharp.ProfilingStartpointProject/Fsharp.ProfilingStartpointProject.fsproj @@ -0,0 +1,16 @@ + + + + Exe + net7.0 + + + + + + + + + + + diff --git a/tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs b/tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs new file mode 100644 index 00000000000..77d52967618 --- /dev/null +++ b/tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs @@ -0,0 +1,17 @@ +open FSharp.Compiler.Benchmarks + +let bench = new FileCascadeBenchmarks() +do bench.Setup() + +[] +let main args = + match args |> Array.toList with + | ["cache-clear"] -> + for i=1 to 1000 do + bench.ParseProjectWithFullCacheClear() |> ignore + | ["mid-change"] -> + for id=1 to 1000 do + bench.ParseProjectWithChangingMiddleFile() |> ignore + | _ -> failwith "Invalid args. Use cache-clear or mid-change" + |> ignore + 0 From 403f6eeec721c12fe46ce83c67b5736b350851d2 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 14 Oct 2022 12:51:04 +0200 Subject: [PATCH 5/8] Expanding the benchmark --- .../FileCascadeBenchmarks.fs | 32 ++++++++++++++++--- .../Program.fs | 14 +++++--- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs index 00fecc74ae8..83d5991bb63 100644 --- a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs +++ b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs @@ -37,13 +37,23 @@ module private CascadeProjectHelpers = $""" module Benchmark0 let returnValue = 5 -let myFunc0 () = 5""" +let myFunc0 () = returnValue +type MyType0 = MyType0 of string +type MyOtherType0 = MyOtherType0 of int""" let generateSourceCode number = $""" module Benchmark%i{number} open Benchmark%i{number-1} let myFunc%i{number} () = myFunc%i{number-1}() +type MyType{number} = MyType{number} of string +type MyOtherType{number} = MyOtherType{number} of int +type MyFunctionType{number} = MyType{number} -> MyOtherType{number} + +let processFunc{number} (x) (func:MyFunctionType{number}) = + async {{ + return func(x) + }} //$COMMENTAREA$""" [] @@ -56,7 +66,7 @@ type FileCascadeBenchmarks() = let getProject() = project.Value let checker = FSharpChecker.Create(projectCacheSize = 5, enableParallelCheckingWithSignatureFiles = true, parallelReferenceResolution = true) - let filesToCreate = 64 + let filesToCreate = 1024 let mutable finalFileContents = SourceText.ofString "" @@ -97,7 +107,14 @@ type FileCascadeBenchmarks() = [] member x.ParseAndCheckLastFileProjectAsIs() = - x.ParseAndCheckLastFileInTheProject() + let parse,check = x.ParseAndCheckLastFileInTheProject() + printfn $"ParseHadErrors = {parse.ParseHadErrors}" + let checkResult = + match check with + | FSharpCheckFileAnswer.Aborted -> "abort" + | FSharpCheckFileAnswer.Succeeded a -> + $"Signature.Size = {a.GenerateSignature() |> Option.map (fun s -> s.Length) |> Option.defaultValue 0}" + printfn $"Check = {checkResult }" [] member x.ParseProjectWithFullCacheClear() = @@ -117,7 +134,14 @@ type FileCascadeBenchmarks() = [] member x.ParseProjectWithChangingMiddleFile() = - x.ChangeFile(fileIndex=filesToCreate/2, action= fun () -> x.ParseAndCheckLastFileInTheProject()) + let parse,check = x.ChangeFile(fileIndex=filesToCreate/2, action= fun () -> x.ParseAndCheckLastFileInTheProject()) + printfn $"ParseHadErrors = {parse.ParseHadErrors}" + let checkResult = + match check with + | FSharpCheckFileAnswer.Aborted -> "abort" + | FSharpCheckFileAnswer.Succeeded a -> + $"Signature.Size = {a.GenerateSignature() |> Option.map (fun s -> s.Length) |> Option.defaultValue 0}" + printfn $"Check = {checkResult }" [] member x.ParseProjectWithChangingPenultimateFile() = diff --git a/tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs b/tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs index 77d52967618..9f06d35e95a 100644 --- a/tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs +++ b/tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs @@ -6,11 +6,17 @@ do bench.Setup() [] let main args = match args |> Array.toList with - | ["cache-clear"] -> - for i=1 to 1000 do - bench.ParseProjectWithFullCacheClear() |> ignore + | ["no-change"] -> + for i=1 to 256 do + printfn "***************************" + printfn "ITERATION %i" i + printfn "***************************" + bench.ParseAndCheckLastFileProjectAsIs() |> ignore | ["mid-change"] -> - for id=1 to 1000 do + for i=1 to 16 do + printfn "***************************" + printfn "ITERATION %i" i + printfn "***************************" bench.ParseProjectWithChangingMiddleFile() |> ignore | _ -> failwith "Invalid args. Use cache-clear or mid-change" |> ignore From 29f4690b914aadee1c036d02e753d6154ece1f63 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 18 Oct 2022 13:45:08 +0200 Subject: [PATCH 6/8] Benchmark for incremental compiling with or without .fsi files --- .../FSharp.Compiler.Benchmarks.fsproj | 1 + .../FileCascadeBenchmarks.fs | 122 ++++++++++++------ .../Program.fs | 26 +++- 3 files changed, 105 insertions(+), 44 deletions(-) diff --git a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FSharp.Compiler.Benchmarks.fsproj b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FSharp.Compiler.Benchmarks.fsproj index d1187649935..0d5fe5df518 100644 --- a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FSharp.Compiler.Benchmarks.fsproj +++ b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FSharp.Compiler.Benchmarks.fsproj @@ -11,6 +11,7 @@ Microsoft.NETCore.App.Host.win-x64 [6.0.2], Microsoft.NETCore.App.Host.win-x64 [6.0.2], Microsoft.NETCore.App.Host.win-x64 [6.0.2], Microsoft.NETCore.App.Host.win-x64 [6.0.2]. --> $(NoWarn);NU1505 + false diff --git a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs index 83d5991bb63..1953f25090a 100644 --- a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs +++ b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs @@ -1,4 +1,5 @@ -namespace FSharp.Compiler.Benchmarks +#PARALLEL_COMPILATION_NAMEOFTHEGROUPLIKE_UNITTESTS +namespace FSharp.Compiler.Benchmarks open System open System.IO @@ -41,6 +42,18 @@ let myFunc0 () = returnValue type MyType0 = MyType0 of string type MyOtherType0 = MyOtherType0 of int""" + let baselineFsi = + """ +module Benchmark0 + +val returnValue: int + +val myFunc0: unit -> int + +type MyType0 = | MyType0 of string + +type MyOtherType0 = | MyOtherType0 of int""" + let generateSourceCode number = $""" module Benchmark%i{number} @@ -56,43 +69,82 @@ let processFunc{number} (x) (func:MyFunctionType{number}) = }} //$COMMENTAREA$""" +/// Code create using FSharpCheckFileResults.GenerateSignature() + let generateFsi number = + $""" +module Benchmark{number} + +val myFunc{number}: unit -> int + +type MyType{number} = | MyType6 of string + +type MyOtherType{number} = | MyOtherType{number} of int + +type MyFunctionType{number} = MyType{number} -> MyOtherType{number} + +val processFunc{number}: x: MyType{number} -> func: MyFunctionType{number} -> Async""" + [] -[] +[] [] [] type FileCascadeBenchmarks() = - let mutable project : FSharpProjectOptions option = None - - let getProject() = project.Value - - let checker = FSharpChecker.Create(projectCacheSize = 5, enableParallelCheckingWithSignatureFiles = true, parallelReferenceResolution = true) - let filesToCreate = 1024 - - let mutable finalFileContents = SourceText.ofString "" + let mutable project : FSharpProjectOptions option = None + let filesToCreate = 128 + member val FinalFileContents = SourceText.ofString "" with get,set + + [] + member val PartialCheck = true with get,set + [] + member val ParaChecking = true with get,set + [] + member val GenerateFSI = true with get,set + + member val Checker = Unchecked.defaultof with get, set + member this.GetProject() = project.Value [] - member _.Setup() = + member this.Setup() = + printfn $"Running Setup(). Partial = {this.PartialCheck}, Para = {this.ParaChecking}, FSIGen = {this.GenerateFSI}" + this.Checker <- FSharpChecker.Create( + projectCacheSize = 5, + enablePartialTypeChecking = this.PartialCheck, + enableParallelCheckingWithSignatureFiles = this.ParaChecking) + let projectFolder = Directory.CreateDirectory(Path.Combine(Directory.GetCurrentDirectory(),"CascadeBenchmarkProject")) if(projectFolder.Exists) then do projectFolder.Delete(recursive=true) do Directory.CreateDirectory(projectFolder.FullName) |> ignore let inProjectFolder fileName = Path.Combine(projectFolder.FullName,fileName) - + + if this.GenerateFSI then + File.WriteAllText(inProjectFolder "Benchmark0.fsi",baselineFsi) File.WriteAllText(inProjectFolder "Benchmark0.fs",baselineModule) - for i = 1 to filesToCreate do + + for i = 1 to filesToCreate do + if this.GenerateFSI then + File.WriteAllText(inProjectFolder $"Benchmark%i{i}.fsi", generateFsi i) File.WriteAllText(inProjectFolder $"Benchmark%i{i}.fs", generateSourceCode i) let dllFileName = inProjectFolder "CascadingBenchMark.dll" - let allSourceCodeFiles = [| for i in 0 .. filesToCreate -> inProjectFolder $"Benchmark%i{i}.fs"|] + let allSourceCodeFiles = + [| for i in 0 .. filesToCreate do + if this.GenerateFSI then + yield (inProjectFolder $"Benchmark%i{i}.fsi") + yield (inProjectFolder $"Benchmark%i{i}.fs") + |] let x = createProject allSourceCodeFiles dllFileName project <- Some x - finalFileContents <- generateSourceCode filesToCreate |> SourceText.ofString + this.FinalFileContents <- generateSourceCode filesToCreate |> SourceText.ofString + + () member x.ChangeFile(fileIndex:int, action) = - let project = getProject() - let fileName = project.SourceFiles.[fileIndex] + let project = x.GetProject() + let fileIndexAdapted = if x.GenerateFSI then fileIndex * 2 else fileIndex + let fileName = project.SourceFiles.[fileIndexAdapted] let fullOriginalSource = File.ReadAllText(fileName) try File.WriteAllText(fileName, fullOriginalSource.Replace("$COMMENTAREA$","$FILEMODIFIED")) @@ -101,47 +153,31 @@ type FileCascadeBenchmarks() = File.WriteAllText(fileName,fullOriginalSource) member x.ParseAndCheckLastFileInTheProject () = - let project = getProject() + let project = x.GetProject() let lastFile = project.SourceFiles |> Array.last - checker.ParseAndCheckFileInProject(lastFile,999,finalFileContents,project) |> Async.RunSynchronously + let pr,cr = x.Checker.ParseAndCheckFileInProject(lastFile,999,x.FinalFileContents,project) |> Async.RunSynchronously + printfn $"ParseError = {pr.ParseHadErrors}; Check = {match cr with | FSharpCheckFileAnswer.Succeeded _ -> '+' | _ -> '-' }" + for e in pr.Diagnostics do + printfn "Error:= %s" (e.ToString()) [] member x.ParseAndCheckLastFileProjectAsIs() = - let parse,check = x.ParseAndCheckLastFileInTheProject() - printfn $"ParseHadErrors = {parse.ParseHadErrors}" - let checkResult = - match check with - | FSharpCheckFileAnswer.Aborted -> "abort" - | FSharpCheckFileAnswer.Succeeded a -> - $"Signature.Size = {a.GenerateSignature() |> Option.map (fun s -> s.Length) |> Option.defaultValue 0}" - printfn $"Check = {checkResult }" + x.ParseAndCheckLastFileInTheProject() [] member x.ParseProjectWithFullCacheClear() = - checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() - checker.ClearCache([getProject()]) - checker.InvalidateConfiguration(getProject()) - checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() + x.Checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() + x.Checker.ClearCache([x.GetProject()]) + x.Checker.InvalidateConfiguration(x.GetProject()) x.ParseAndCheckLastFileInTheProject() [] member x.ParseProjectWithChangingFirstFile() = x.ChangeFile(fileIndex=0, action= fun () -> x.ParseAndCheckLastFileInTheProject()) - [] - member x.ParseProjectWithChanging25thPercentileFile() = - x.ChangeFile(fileIndex=filesToCreate/4, action= fun () -> x.ParseAndCheckLastFileInTheProject()) - [] member x.ParseProjectWithChangingMiddleFile() = - let parse,check = x.ChangeFile(fileIndex=filesToCreate/2, action= fun () -> x.ParseAndCheckLastFileInTheProject()) - printfn $"ParseHadErrors = {parse.ParseHadErrors}" - let checkResult = - match check with - | FSharpCheckFileAnswer.Aborted -> "abort" - | FSharpCheckFileAnswer.Succeeded a -> - $"Signature.Size = {a.GenerateSignature() |> Option.map (fun s -> s.Length) |> Option.defaultValue 0}" - printfn $"Check = {checkResult }" + x.ChangeFile(fileIndex=filesToCreate/2, action= fun () -> x.ParseAndCheckLastFileInTheProject()) [] member x.ParseProjectWithChangingPenultimateFile() = diff --git a/tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs b/tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs index 9f06d35e95a..5d851e60865 100644 --- a/tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs +++ b/tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs @@ -1,10 +1,34 @@ open FSharp.Compiler.Benchmarks +open System +open System.IO +open System.Text +open FSharp.Compiler.CodeAnalysis +open FSharp.Compiler.Diagnostics +open FSharp.Compiler.EditorServices +open FSharp.Compiler.Text +open FSharp.Compiler.AbstractIL.IL +open FSharp.Compiler.AbstractIL.ILBinaryReader +open BenchmarkDotNet.Attributes +open FSharp.Compiler.Benchmarks +open Microsoft.CodeAnalysis.Text +open BenchmarkDotNet.Order +open BenchmarkDotNet.Mathematics + let bench = new FileCascadeBenchmarks() +bench.GenerateFSI <- true do bench.Setup() +(* +This project was created as an easy entry point for low-level profiling of FCS operations. +The only purpose is the easy of setup (simply set as startup project and launch) so that a profiler can be connected. +There is definitely no harm in deleting it if it starts bothering anyone. +*) + + [] -let main args = +let main args = + match args |> Array.toList with | ["no-change"] -> for i=1 to 256 do From 510b4e870a07355b1642a1cc5b7cd89f1f0b73ab Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 18 Oct 2022 17:15:32 +0200 Subject: [PATCH 7/8] Update tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs Co-authored-by: Petr Pokorny --- .../CompilerServiceBenchmarks/FileCascadeBenchmarks.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs index 9be9c3e1eef..accffc76c12 100644 --- a/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs +++ b/tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs @@ -75,7 +75,7 @@ module Benchmark{number} val myFunc{number}: unit -> int -type MyType{number} = | MyType6 of string +type MyType{number} = | MyType{number} of string type MyOtherType{number} = | MyOtherType{number} of int From 7126be9404c23d54e33a8d62c40f9fb436f5d7cb Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 18 Oct 2022 17:30:59 +0200 Subject: [PATCH 8/8] New project added to assemblycheck ignore list --- buildtools/AssemblyCheck/SkipVerifyEmbeddedPdb.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/buildtools/AssemblyCheck/SkipVerifyEmbeddedPdb.txt b/buildtools/AssemblyCheck/SkipVerifyEmbeddedPdb.txt index f823017f4a1..9ea06757477 100644 --- a/buildtools/AssemblyCheck/SkipVerifyEmbeddedPdb.txt +++ b/buildtools/AssemblyCheck/SkipVerifyEmbeddedPdb.txt @@ -1,5 +1,6 @@ FSharp.Build.UnitTests.dll FSharp.Compiler.Benchmarks.dll +Fsharp.ProfilingStartpointProject.dll FSharp.Compiler.ComponentTests.dll FSharp.Test.Utilities.dll FSharp.Compiler.Private.Scripting.UnitTests.dll