Project with cascading files benchmarks - compare effect on signature files on incremental build

.gitignore

@@ -125,3 +125,4 @@ nCrunchTemp_*
 /test.fsx
 
 tests/FSharp.Compiler.Service.Tests/FSharp.CompilerService.SurfaceArea.netstandard.actual
+*.vsp

VisualFSharp.sln

@@ -184,7 +184,7 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "fsiAnyCpu", "src\fsi\fsiAny
 EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "fsiArm64", "src\fsi\fsiArm64Project\fsiArm64.fsproj", "{EB015235-1E07-4CDA-9CC6-3FBCC27910D1}"
 EndProject
-Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "HistoricalBenchmark", "tests\benchmarks\FCSBenchmarks\BenchmarkComparison\HistoricalBenchmark.fsproj", "{583182E1-3484-4A8F-AC06-7C0D232C0CA4}"
+Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "HistoricalBenchmark", "tests\benchmarks\FCSBenchmarks\BenchmarkComparison\HistoricalBenchmark.fsproj", "{583182E1-3484-4A8F-AC06-7C0D232C0CA4}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "FCSBenchmarks", "FCSBenchmarks", "{39CDF34B-FB23-49AE-AB27-0975DA379BB5}"
 	ProjectSection(SolutionItems) = preProject
@@ -193,6 +193,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "FCSBenchmarks", "FCSBenchma
 		tests\benchmarks\FCSBenchmarks\SmokeTestAllBenchmarks.ps1 = tests\benchmarks\FCSBenchmarks\SmokeTestAllBenchmarks.ps1
 	EndProjectSection
 EndProject
+Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Fsharp.ProfilingStartpointProject", "tests\benchmarks\Fsharp.ProfilingStartpointProject\Fsharp.ProfilingStartpointProject.fsproj", "{FE23BB65-276A-4E41-8CC7-F7752241DEBA}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -1019,6 +1021,18 @@ Global
 		{583182E1-3484-4A8F-AC06-7C0D232C0CA4}.Release|Any CPU.Build.0 = Release|Any CPU
 		{583182E1-3484-4A8F-AC06-7C0D232C0CA4}.Release|x86.ActiveCfg = Release|Any CPU
 		{583182E1-3484-4A8F-AC06-7C0D232C0CA4}.Release|x86.Build.0 = Release|Any CPU
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|x86.Build.0 = Debug|Any CPU
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|Any CPU.ActiveCfg = Debug|Any CPU
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|Any CPU.Build.0 = Debug|Any CPU
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|x86.ActiveCfg = Debug|Any CPU
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|x86.Build.0 = Debug|Any CPU
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|x86.ActiveCfg = Release|Any CPU
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1089,16 +1103,17 @@ Global
 		{B5A9BBD9-2F45-4722-A6CA-BAE3C64CD4E2} = {3881429D-A97A-49EB-B7AE-A82BA5FE9C77}
 		{14F3D3D6-5C8E-43C2-98A2-17EA704D4DEA} = {CFE3259A-2D30-4EB0-80D5-E8B5F3D01449}
 		{A422D673-8E3B-4924-821B-DD3174173426} = {4C7B48D7-19AF-4AE7-9D1D-3BB289D5480D}
+		{564E7DC5-11CB-4FCF-ABDD-23AD93AF3A61} = {39CDF34B-FB23-49AE-AB27-0975DA379BB5}
 		{B1E30F2C-894F-47A9-9C8A-3324831E7D26} = {4C7B48D7-19AF-4AE7-9D1D-3BB289D5480D}
 		{597D9896-4B90-4E9E-9C99-445C2CB9FF60} = {B8DDA694-7939-42E3-95E5-265C2217C142}
 		{0973C362-585C-4838-9459-D7E45C6B784B} = {B8DDA694-7939-42E3-95E5-265C2217C142}
 		{E54456F4-D51A-4334-B225-92EBBED92B40} = {B8DDA694-7939-42E3-95E5-265C2217C142}
 		{511C95D9-3BA6-451F-B6F8-F033F40878A5} = {B8DDA694-7939-42E3-95E5-265C2217C142}
 		{37EB3E54-ABC6-4CF5-8273-7CE4B61A42C1} = {B8DDA694-7939-42E3-95E5-265C2217C142}
 		{EB015235-1E07-4CDA-9CC6-3FBCC27910D1} = {B8DDA694-7939-42E3-95E5-265C2217C142}
-		{39CDF34B-FB23-49AE-AB27-0975DA379BB5} = {DFB6ADD7-3149-43D9-AFA0-FC4A818B472B}
-		{564E7DC5-11CB-4FCF-ABDD-23AD93AF3A61} = {39CDF34B-FB23-49AE-AB27-0975DA379BB5}
 		{583182E1-3484-4A8F-AC06-7C0D232C0CA4} = {39CDF34B-FB23-49AE-AB27-0975DA379BB5}
+		{39CDF34B-FB23-49AE-AB27-0975DA379BB5} = {DFB6ADD7-3149-43D9-AFA0-FC4A818B472B}
+		{FE23BB65-276A-4E41-8CC7-F7752241DEBA} = {39CDF34B-FB23-49AE-AB27-0975DA379BB5}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {48EDBBBE-C8EE-4E3C-8B19-97184A487B37}

buildtools/AssemblyCheck/SkipVerifyEmbeddedPdb.txt

@@ -1,5 +1,6 @@
 FSharp.Build.UnitTests.dll
 FSharp.Compiler.Benchmarks.dll
+Fsharp.ProfilingStartpointProject.dll
 FSharp.Compiler.ComponentTests.dll
 FSharp.Test.Utilities.dll
 FSharp.Compiler.Private.Scripting.UnitTests.dll

tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FSharp.Compiler.Benchmarks.fsproj

@@ -11,6 +11,7 @@
           Microsoft.NETCore.App.Host.win-x64 [6.0.2], Microsoft.NETCore.App.Host.win-x64 [6.0.2], Microsoft.NETCore.App.Host.win-x64 [6.0.2], Microsoft.NETCore.App.Host.win-x64 [6.0.2].
     -->
     <NoWarn>$(NoWarn);NU1505</NoWarn>
+    <ProduceReferenceAssembly>false</ProduceReferenceAssembly>
   </PropertyGroup>
 
   <ItemGroup>

tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FileCascadeBenchmarks.fs

@@ -37,82 +37,148 @@ module private CascadeProjectHelpers =
         $"""
 module Benchmark0
 let returnValue = 5
-let myFunc0 () = 5"""
+let myFunc0 () = returnValue
+type MyType0 = MyType0 of string
+type MyOtherType0 = MyOtherType0 of int"""
+
+    let baselineFsi = 
+        """
+module Benchmark0
+
+val returnValue: int
+
+val myFunc0: unit -> int
+
+type MyType0 = | MyType0 of string
+
+type MyOtherType0 = | MyOtherType0 of int"""
 
     let generateSourceCode number =
         $"""
 module Benchmark%i{number}
 open Benchmark%i{number-1}
 let myFunc%i{number} () = myFunc%i{number-1}()
+type MyType{number} = MyType{number} of string
+type MyOtherType{number} = MyOtherType{number} of int
+type MyFunctionType{number} = MyType{number} -> MyOtherType{number}
+
+let processFunc{number} (x) (func:MyFunctionType{number}) = 
+    async {{
+        return func(x)
+    }}
 //$COMMENTAREA$"""
 
+/// Code create using FSharpCheckFileResults.GenerateSignature()
+    let generateFsi number = 
+        $"""
+module Benchmark{number}
+
+val myFunc{number}: unit -> int
+
+type MyType{number} = | MyType{number} of string
+
+type MyOtherType{number} = | MyOtherType{number} of int
+
+type MyFunctionType{number} = MyType{number} -> MyOtherType{number}
+
+val processFunc{number}: x: MyType{number} -> func: MyFunctionType{number} -> Async<MyOtherType{number}>"""
+
 [<MemoryDiagnoser>]
-[<LongRunJob>]
+[<ShortRunJob>]
 [<Orderer(SummaryOrderPolicy.FastestToSlowest)>]
 [<RankColumn(NumeralSystem.Roman)>]
 type FileCascadeBenchmarks() = 
-    let mutable project : FSharpProjectOptions option = None
-
-    let getProject() = project.Value
-
-    let checker = FSharpChecker.Create(projectCacheSize = 5, enableParallelCheckingWithSignatureFiles = true, parallelReferenceResolution = true)
-    let filesToCreate = 64
-
-    let mutable finalFileContents = SourceText.ofString ""
+    let mutable project : FSharpProjectOptions option = None   
+    let filesToCreate = 128
+    member val FinalFileContents = SourceText.ofString "" with get,set
+
+    [<ParamsAllValues>]
+    member val PartialCheck = true with get,set
+    [<ParamsAllValues>]
+    member val ParaChecking = true with get,set
+    [<ParamsAllValues>]
+    member val GenerateFSI = true with get,set
+
+    member val Checker = Unchecked.defaultof<FSharpChecker> with get, set
+    member this.GetProject() = project.Value
 
     [<GlobalSetup>]
-    member _.Setup() =
+    member this.Setup() =
+        printfn $"Running Setup(). Partial = {this.PartialCheck}, Para = {this.ParaChecking}, FSIGen = {this.GenerateFSI}"
+        this.Checker <- FSharpChecker.Create(
+                projectCacheSize = 5, 
+                enablePartialTypeChecking = this.PartialCheck,
+                enableParallelCheckingWithSignatureFiles = this.ParaChecking)
+
+
         let projectFolder = Directory.CreateDirectory(Path.Combine(Directory.GetCurrentDirectory(),"CascadeBenchmarkProject"))                    
         if(projectFolder.Exists) then
             do projectFolder.Delete(recursive=true)
         do Directory.CreateDirectory(projectFolder.FullName) |> ignore
 
         let inProjectFolder fileName = Path.Combine(projectFolder.FullName,fileName)
-
+
+        if this.GenerateFSI then
+            File.WriteAllText(inProjectFolder "Benchmark0.fsi",baselineFsi)
         File.WriteAllText(inProjectFolder "Benchmark0.fs",baselineModule)
-        for i = 1 to filesToCreate do
+
+        for i = 1 to filesToCreate do            
+            if this.GenerateFSI then
+                File.WriteAllText(inProjectFolder $"Benchmark%i{i}.fsi", generateFsi i)
             File.WriteAllText(inProjectFolder $"Benchmark%i{i}.fs", generateSourceCode i)
 
         let dllFileName = inProjectFolder "CascadingBenchMark.dll"
-        let allSourceCodeFiles = [| for i in 0 .. filesToCreate -> inProjectFolder $"Benchmark%i{i}.fs"|]
+        let allSourceCodeFiles = 
+            [| for i in 0 .. filesToCreate do                
+                if this.GenerateFSI then
+                    yield (inProjectFolder $"Benchmark%i{i}.fsi")
+                yield (inProjectFolder $"Benchmark%i{i}.fs")
+            |]
         let x = createProject allSourceCodeFiles dllFileName
         project <- Some x
-        finalFileContents <- generateSourceCode filesToCreate |> SourceText.ofString
+        this.FinalFileContents <- generateSourceCode filesToCreate |> SourceText.ofString                
+
+        ()
 
 
     member x.ChangeFile(fileIndex:int, action) = 
-        let project = getProject()
-        let fileName = project.SourceFiles.[fileIndex]
+        let project = x.GetProject()
+        let fileIndexAdapted = if x.GenerateFSI then fileIndex * 2 else fileIndex
+        let fileName = project.SourceFiles.[fileIndexAdapted]
         let fullOriginalSource = File.ReadAllText(fileName)
         try
             File.WriteAllText(fileName, fullOriginalSource.Replace("$COMMENTAREA$","$FILEMODIFIED"))
             action()
         finally
             File.WriteAllText(fileName,fullOriginalSource)
 
-    member x.CheckFinalFile () =
-        let project = getProject()
+    member x.ParseAndCheckLastFileInTheProject () =
+        let project = x.GetProject()
         let lastFile = project.SourceFiles |> Array.last
-        checker.ParseAndCheckFileInProject(lastFile,999,finalFileContents,project)
+        let pr,cr = x.Checker.ParseAndCheckFileInProject(lastFile,999,x.FinalFileContents,project) |> Async.RunSynchronously
+        printfn $"ParseError = {pr.ParseHadErrors}; Check = {match cr with | FSharpCheckFileAnswer.Succeeded _ -> '+' | _ -> '-' }"
+        for e in pr.Diagnostics do
+            printfn "Error:= %s" (e.ToString())
 
     [<Benchmark>]
-    member x.ParseProjectAsIs() =
-        x.CheckFinalFile()
+    member x.ParseAndCheckLastFileProjectAsIs() =
+        x.ParseAndCheckLastFileInTheProject()
 
     [<Benchmark(Baseline=true)>]
     member x.ParseProjectWithFullCacheClear() =
-        checker.ClearCache([getProject()])
-        checker.InvalidateConfiguration(getProject())
-        x.CheckFinalFile()
+        x.Checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients()
+        x.Checker.ClearCache([x.GetProject()])
+        x.Checker.InvalidateConfiguration(x.GetProject())     
+        x.ParseAndCheckLastFileInTheProject()
 
     [<Benchmark>]
     member x.ParseProjectWithChangingFirstFile() =
-        x.ChangeFile(fileIndex=0, action= fun () -> x.CheckFinalFile())  
+        x.ChangeFile(fileIndex=0, action= fun () -> x.ParseAndCheckLastFileInTheProject())  
 
     [<Benchmark>]
-    member x.ParseProjectWithChanging25thPercentileFile() =
-        x.ChangeFile(fileIndex=filesToCreate/4, action= fun () -> x.CheckFinalFile())  
+    member x.ParseProjectWithChangingMiddleFile() =
+        x.ChangeFile(fileIndex=filesToCreate/2, action= fun () -> x.ParseAndCheckLastFileInTheProject())        
 
     [<Benchmark>]
-    member x.ParseProjectWithChangingMiddleFile() =
-        x.ChangeFile(fileIndex=filesToCreate/2, action= fun () -> x.CheckFinalFile())  
+    member x.ParseProjectWithChangingPenultimateFile() =
+        x.ChangeFile(fileIndex=(filesToCreate-2), action= fun () -> x.ParseAndCheckLastFileInTheProject()) 

tests/benchmarks/Fsharp.ProfilingStartpointProject/Fsharp.ProfilingStartpointProject.fsproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="Program.fs" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\FCSBenchmarks\CompilerServiceBenchmarks\FSharp.Compiler.Benchmarks.fsproj" />
+  </ItemGroup>
+
+</Project>

tests/benchmarks/Fsharp.ProfilingStartpointProject/Program.fs

@@ -0,0 +1,47 @@
+open FSharp.Compiler.Benchmarks
+
+open System
+open System.IO
+open System.Text
+open FSharp.Compiler.CodeAnalysis
+open FSharp.Compiler.Diagnostics
+open FSharp.Compiler.EditorServices
+open FSharp.Compiler.Text
+open FSharp.Compiler.AbstractIL.IL
+open FSharp.Compiler.AbstractIL.ILBinaryReader
+open BenchmarkDotNet.Attributes
+open FSharp.Compiler.Benchmarks
+open Microsoft.CodeAnalysis.Text
+open BenchmarkDotNet.Order
+open BenchmarkDotNet.Mathematics
+
+let bench = new FileCascadeBenchmarks()
+bench.GenerateFSI <- true
+do bench.Setup()
+
+(*
+This project was created as an easy entry point for low-level profiling of FCS operations.
+The only purpose is the easy of setup (simply set as startup project and launch) so that a profiler can be connected.
+There is definitely no harm in deleting it if it starts bothering anyone.
+*)
+
+
+[<EntryPoint>]
+let main args =   
+
+    match args |> Array.toList with
+    | ["no-change"] -> 
+        for i=1 to 256 do
+            printfn "***************************"
+            printfn "ITERATION %i" i
+            printfn "***************************"
+            bench.ParseAndCheckLastFileProjectAsIs()  |> ignore         
+    | ["mid-change"] ->
+        for i=1 to 16 do
+            printfn "***************************"
+            printfn "ITERATION %i" i
+            printfn "***************************"
+            bench.ParseProjectWithChangingMiddleFile() |> ignore
+    | _ -> failwith "Invalid args. Use cache-clear or mid-change"
+    |> ignore
+    0