From 4c8fb160492d4ee5712be9af1f2e0023df781ff6 Mon Sep 17 00:00:00 2001 From: nojaf Date: Tue, 30 Jan 2024 09:31:07 +0100 Subject: [PATCH 1/5] Run TransparentCompiler unit tests with local response files. --- .../FSharpChecker/TransparentCompiler.fs | 116 +++++++++++++++++- .../ProjectGeneration.fs | 25 ++-- 2 files changed, 131 insertions(+), 10 deletions(-) diff --git a/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs b/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs index 60c7e8be881..40a6b16d8f8 100644 --- a/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs +++ b/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs @@ -494,7 +494,7 @@ let fuzzingTest seed (project: SyntheticProject) = task { let checker = builder.Checker // Force creation and caching of options - do! SaveAndCheckProject project checker |> Async.Ignore + do! SaveAndCheckProject project checker false |> Async.Ignore let projectAgent = MailboxProcessor.Start(fun (inbox: MailboxProcessor) -> let rec loop project = @@ -800,4 +800,116 @@ module Stuff = //Assert.Equal(hash, hash2) - () \ No newline at end of file + () + +let mkWorkflowBuilderForResponseFile + (useTransparentCompiler:bool) + (responseFile: FileInfo) + : SyntheticProject * ProjectWorkflowBuilder + = + if not responseFile.Exists then + failwith $"%s{responseFile.FullName} does not exist" + + let compilerArgs = File.ReadAllLines responseFile.FullName + + let fsharpFileExtensions = set [| ".fs" ; ".fsi" ; ".fsx" |] + + let isFSharpFile (file : string) = + Set.exists (fun (ext : string) -> file.EndsWith (ext, StringComparison.Ordinal)) fsharpFileExtensions + + let fsharpFiles = + compilerArgs + |> Array.choose (fun (line : string) -> + if not (isFSharpFile line) then + None + else + + let fullPath = Path.Combine (responseFile.DirectoryName, line) + if not (File.Exists fullPath) then + None + else + Some fullPath + ) + |> Array.toList + + let signatureFiles, implementationFiles = + fsharpFiles |> List.partition (fun path -> path.EndsWith ".fsi") + + let signatureFiles = set signatureFiles + + let sourceFiles = + implementationFiles + |> List.map (fun implPath -> + { + Id = implPath + PublicVersion = 1 + InternalVersion = 1 + DependsOn = [] + FunctionName = "f" + SignatureFile = + let sigPath = $"%s{implPath}i" in + if signatureFiles.Contains sigPath then Custom(File.ReadAllText sigPath) else No + HasErrors = false + Source = File.ReadAllText implPath + ExtraSource = "" + EntryPoint = false + } + ) + + let otherOptions = + compilerArgs + |> Array.filter (fun line -> not (isFSharpFile line)) + |> Array.toList + + let syntheticProject: SyntheticProject = + { + Name = Path.GetFileNameWithoutExtension responseFile.Name + ProjectDir = responseFile.DirectoryName + SourceFiles = sourceFiles + DependsOn = List.empty + RecursiveNamespace = false + OtherOptions = otherOptions + AutoAddModules = false + NugetReferences = List.empty + FrameworkReferences = List.empty + SkipInitialCheck = false + } + + let builder = + ProjectWorkflowBuilder( + syntheticProject, + isExistingProject = true, + useTransparentCompiler = useTransparentCompiler + ) + + syntheticProject, builder + +/// Update these paths to a local response file with compiler arguments of existing F# projects. +/// References projects are expected to have been built. +let localResponseFiles = + [| + @"C:\Projects\fantomas\src\Fantomas.Core.Tests\Fantomas.Core.Tests.rsp" + |] + |> Array.collect (fun f -> + [| + [| true :> obj; f:> obj |] + [| false :> obj; f :> obj|] + |] + ) + +// Uncomment this attribute if you want run this test against local response files. +// [] +[] +let ``TypeCheck last file in project with transparent compiler`` useTransparent responseFile = + let responseFile = FileInfo responseFile + let project, workflow = mkWorkflowBuilderForResponseFile useTransparent responseFile + + let lastFile = + project.SourceFiles + |> List.last + |> fun sf -> sf.Id + + workflow { + clearCache + checkFile lastFile expectOk + } diff --git a/tests/FSharp.Test.Utilities/ProjectGeneration.fs b/tests/FSharp.Test.Utilities/ProjectGeneration.fs index 54c00ebe544..e66d0238562 100644 --- a/tests/FSharp.Test.Utilities/ProjectGeneration.fs +++ b/tests/FSharp.Test.Utilities/ProjectGeneration.fs @@ -195,7 +195,9 @@ type SyntheticSourceFile = EntryPoint: bool } - member this.FileName = $"File{this.Id}.fs" + member this.FileName = + if File.Exists this.Id then this.Id else $"File%s{this.Id}.fs" + member this.SignatureFileName = $"{this.FileName}i" member this.TypeName = $"T{this.Id}V_{this.PublicVersion}" member this.ModuleName = $"Module{this.Id}" @@ -792,12 +794,14 @@ type WorkflowContext = Signatures: Map Cursor: FSharpSymbolUse option } -let SaveAndCheckProject project checker = +let SaveAndCheckProject project checker isExistingProject = async { use _ = Activity.start "SaveAndCheckProject" [ Activity.Tags.project, project.Name ] - do! saveProject project true checker + // Don't save the project if it is a real world project that exists on disk. + if not isExistingProject then + do! saveProject project true checker let options = project.GetProjectOptions checker let! snapshot = FSharpProjectSnapshot.FromOptions(options, getFileSnapshot project) @@ -834,13 +838,15 @@ type ProjectWorkflowBuilder ?useSyntaxTreeCache, ?useTransparentCompiler, ?runTimeout, - ?autoStart + ?autoStart, + ?isExistingProject ) = let useTransparentCompiler = defaultArg useTransparentCompiler FSharp.Compiler.CompilerConfig.FSharpExperimentalFeaturesEnabledAutomatically let useGetSource = not useTransparentCompiler && defaultArg useGetSource false let useChangeNotifications = not useTransparentCompiler && defaultArg useChangeNotifications false let autoStart = defaultArg autoStart true + let isExistingProject = defaultArg isExistingProject false let mutable latestProject = initialProject let mutable activity = None @@ -874,7 +880,7 @@ type ProjectWorkflowBuilder let getInitialContext() = match initialContext with | Some ctx -> async.Return ctx - | None -> SaveAndCheckProject initialProject checker + | None -> SaveAndCheckProject initialProject checker isExistingProject /// Creates a ProjectWorkflowBuilder which will already have the project /// saved and checked so time won't be spent on that. @@ -915,7 +921,7 @@ type ProjectWorkflowBuilder try Async.RunSynchronously(workflow, timeout = defaultArg runTimeout 600_000) finally - if initialContext.IsNone then + if initialContext.IsNone && not isExistingProject then this.DeleteProjectDir() activity |> Option.iter (fun x -> x.Dispose()) tracerProvider |> Option.iter (fun x -> @@ -1021,10 +1027,13 @@ type ProjectWorkflowBuilder async { let! ctx = workflow - use _ = + use activity = Activity.start "ProjectWorkflowBuilder.CheckFile" [ Activity.Tags.project, initialProject.Name; "fileId", fileId ] - let! results = checkFile fileId ctx.Project checker + let! results = + checkFile fileId ctx.Project checker + + activity.Dispose() let oldSignature = ctx.Signatures[fileId] let newSignature = getSignature results From 97942614d17a25c26e3a6ca41dbb1ec063954193 Mon Sep 17 00:00:00 2001 From: nojaf Date: Tue, 30 Jan 2024 10:40:45 +0100 Subject: [PATCH 2/5] Move mkSyntheticProjectForResponseFile to ProjectGeneration. --- .../FSharpChecker/TransparentCompiler.fs | 95 ++----------------- .../ProjectGeneration.fs | 68 +++++++++++++ 2 files changed, 78 insertions(+), 85 deletions(-) diff --git a/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs b/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs index 40a6b16d8f8..007197877c2 100644 --- a/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs +++ b/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs @@ -802,88 +802,6 @@ module Stuff = () -let mkWorkflowBuilderForResponseFile - (useTransparentCompiler:bool) - (responseFile: FileInfo) - : SyntheticProject * ProjectWorkflowBuilder - = - if not responseFile.Exists then - failwith $"%s{responseFile.FullName} does not exist" - - let compilerArgs = File.ReadAllLines responseFile.FullName - - let fsharpFileExtensions = set [| ".fs" ; ".fsi" ; ".fsx" |] - - let isFSharpFile (file : string) = - Set.exists (fun (ext : string) -> file.EndsWith (ext, StringComparison.Ordinal)) fsharpFileExtensions - - let fsharpFiles = - compilerArgs - |> Array.choose (fun (line : string) -> - if not (isFSharpFile line) then - None - else - - let fullPath = Path.Combine (responseFile.DirectoryName, line) - if not (File.Exists fullPath) then - None - else - Some fullPath - ) - |> Array.toList - - let signatureFiles, implementationFiles = - fsharpFiles |> List.partition (fun path -> path.EndsWith ".fsi") - - let signatureFiles = set signatureFiles - - let sourceFiles = - implementationFiles - |> List.map (fun implPath -> - { - Id = implPath - PublicVersion = 1 - InternalVersion = 1 - DependsOn = [] - FunctionName = "f" - SignatureFile = - let sigPath = $"%s{implPath}i" in - if signatureFiles.Contains sigPath then Custom(File.ReadAllText sigPath) else No - HasErrors = false - Source = File.ReadAllText implPath - ExtraSource = "" - EntryPoint = false - } - ) - - let otherOptions = - compilerArgs - |> Array.filter (fun line -> not (isFSharpFile line)) - |> Array.toList - - let syntheticProject: SyntheticProject = - { - Name = Path.GetFileNameWithoutExtension responseFile.Name - ProjectDir = responseFile.DirectoryName - SourceFiles = sourceFiles - DependsOn = List.empty - RecursiveNamespace = false - OtherOptions = otherOptions - AutoAddModules = false - NugetReferences = List.empty - FrameworkReferences = List.empty - SkipInitialCheck = false - } - - let builder = - ProjectWorkflowBuilder( - syntheticProject, - isExistingProject = true, - useTransparentCompiler = useTransparentCompiler - ) - - syntheticProject, builder - /// Update these paths to a local response file with compiler arguments of existing F# projects. /// References projects are expected to have been built. let localResponseFiles = @@ -900,12 +818,19 @@ let localResponseFiles = // Uncomment this attribute if you want run this test against local response files. // [] [] -let ``TypeCheck last file in project with transparent compiler`` useTransparent responseFile = +let ``TypeCheck last file in project with transparent compiler`` useTransparentCompiler responseFile = let responseFile = FileInfo responseFile - let project, workflow = mkWorkflowBuilderForResponseFile useTransparent responseFile + let syntheticProject = mkSyntheticProjectForResponseFile responseFile + + let workflow = + ProjectWorkflowBuilder( + syntheticProject, + isExistingProject = true, + useTransparentCompiler = useTransparentCompiler + ) let lastFile = - project.SourceFiles + syntheticProject.SourceFiles |> List.last |> fun sf -> sf.Id diff --git a/tests/FSharp.Test.Utilities/ProjectGeneration.fs b/tests/FSharp.Test.Utilities/ProjectGeneration.fs index e66d0238562..285d99926ab 100644 --- a/tests/FSharp.Test.Utilities/ProjectGeneration.fs +++ b/tests/FSharp.Test.Utilities/ProjectGeneration.fs @@ -470,6 +470,74 @@ let private writeFile (p: SyntheticProject) (f: SyntheticSourceFile) = let content = renderSourceFile p f writeFileIfChanged fileName content +/// Creates a SyntheticProject from the compiler arguments found in the response file. +let mkSyntheticProjectForResponseFile (responseFile: FileInfo) : SyntheticProject = + if not responseFile.Exists then + failwith $"%s{responseFile.FullName} does not exist" + + let compilerArgs = File.ReadAllLines responseFile.FullName + + let fsharpFileExtensions = set [| ".fs" ; ".fsi" ; ".fsx" |] + + let isFSharpFile (file : string) = + Set.exists (fun (ext : string) -> file.EndsWith (ext, StringComparison.Ordinal)) fsharpFileExtensions + + let fsharpFiles = + compilerArgs + |> Array.choose (fun (line : string) -> + if not (isFSharpFile line) then + None + else + + let fullPath = Path.Combine (responseFile.DirectoryName, line) + if not (File.Exists fullPath) then + None + else + Some fullPath + ) + |> Array.toList + + let signatureFiles, implementationFiles = + fsharpFiles |> List.partition (fun path -> path.EndsWith ".fsi") + + let signatureFiles = set signatureFiles + + let sourceFiles = + implementationFiles + |> List.map (fun implPath -> + { + Id = implPath + PublicVersion = 1 + InternalVersion = 1 + DependsOn = [] + FunctionName = "f" + SignatureFile = + let sigPath = $"%s{implPath}i" in + if signatureFiles.Contains sigPath then Custom(File.ReadAllText sigPath) else No + HasErrors = false + Source = File.ReadAllText implPath + ExtraSource = "" + EntryPoint = false + } + ) + + let otherOptions = + compilerArgs + |> Array.filter (fun line -> not (isFSharpFile line)) + |> Array.toList + + { + Name = Path.GetFileNameWithoutExtension responseFile.Name + ProjectDir = responseFile.DirectoryName + SourceFiles = sourceFiles + DependsOn = List.empty + RecursiveNamespace = false + OtherOptions = otherOptions + AutoAddModules = false + NugetReferences = List.empty + FrameworkReferences = List.empty + SkipInitialCheck = false + } [] module ProjectOperations = From 1749ffee4fddca9e583bfde86bb90b4723bd8e07 Mon Sep 17 00:00:00 2001 From: nojaf Date: Tue, 30 Jan 2024 14:52:43 +0100 Subject: [PATCH 3/5] Use the file name relative to the response file without the extension. --- .../FSharpChecker/TransparentCompiler.fs | 8 ++++++-- .../ProjectGeneration.fs | 19 ++++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs b/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs index 007197877c2..90dc85c4a33 100644 --- a/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs +++ b/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs @@ -831,8 +831,12 @@ let ``TypeCheck last file in project with transparent compiler`` useTransparentC let lastFile = syntheticProject.SourceFiles - |> List.last - |> fun sf -> sf.Id + |> List.tryLast + |> Option.map (fun sf -> sf.Id) + + match lastFile with + | None -> failwithf "Last file of project could not be found" + | Some lastFile -> workflow { clearCache diff --git a/tests/FSharp.Test.Utilities/ProjectGeneration.fs b/tests/FSharp.Test.Utilities/ProjectGeneration.fs index 285d99926ab..d669b8441af 100644 --- a/tests/FSharp.Test.Utilities/ProjectGeneration.fs +++ b/tests/FSharp.Test.Utilities/ProjectGeneration.fs @@ -193,10 +193,14 @@ type SyntheticSourceFile = Source: string ExtraSource: string EntryPoint: bool + /// The absolute path of an existing F# file. + PhysicalFileName: string option } member this.FileName = - if File.Exists this.Id then this.Id else $"File%s{this.Id}.fs" + match this.PhysicalFileName with + | Some f -> f + | None -> $"File%s{this.Id}.fs" member this.SignatureFileName = $"{this.FileName}i" member this.TypeName = $"T{this.Id}V_{this.PublicVersion}" @@ -218,7 +222,8 @@ let sourceFile fileId deps = HasErrors = false Source = "" ExtraSource = "" - EntryPoint = false } + EntryPoint = false + PhysicalFileName = None } let OptionsCache = ConcurrentDictionary() @@ -505,8 +510,15 @@ let mkSyntheticProjectForResponseFile (responseFile: FileInfo) : SyntheticProjec let sourceFiles = implementationFiles |> List.map (fun implPath -> + let id = + let fileNameWithoutExtension = Path.GetFileNameWithoutExtension implPath + let directoryOfFile = FileInfo(implPath).DirectoryName + let relativeUri = Uri(responseFile.FullName).MakeRelativeUri(Uri(directoryOfFile)) + let relativeFolderPath = Uri.UnescapeDataString(relativeUri.ToString()).Replace('/', Path.DirectorySeparatorChar) + Path.Combine(relativeFolderPath, fileNameWithoutExtension) + { - Id = implPath + Id = id PublicVersion = 1 InternalVersion = 1 DependsOn = [] @@ -518,6 +530,7 @@ let mkSyntheticProjectForResponseFile (responseFile: FileInfo) : SyntheticProjec Source = File.ReadAllText implPath ExtraSource = "" EntryPoint = false + PhysicalFileName = Some implPath } ) From 582f387fb2f7d05691094da4e3a81cf37253f2de Mon Sep 17 00:00:00 2001 From: nojaf Date: Tue, 30 Jan 2024 14:58:47 +0100 Subject: [PATCH 4/5] Don't prefix if IsPhysicalFile. --- tests/FSharp.Test.Utilities/ProjectGeneration.fs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/FSharp.Test.Utilities/ProjectGeneration.fs b/tests/FSharp.Test.Utilities/ProjectGeneration.fs index d669b8441af..c591dcc4b70 100644 --- a/tests/FSharp.Test.Utilities/ProjectGeneration.fs +++ b/tests/FSharp.Test.Utilities/ProjectGeneration.fs @@ -193,14 +193,12 @@ type SyntheticSourceFile = Source: string ExtraSource: string EntryPoint: bool - /// The absolute path of an existing F# file. - PhysicalFileName: string option + /// Indicates whether this is an existing F# file on disk. + IsPhysicalFile: bool } member this.FileName = - match this.PhysicalFileName with - | Some f -> f - | None -> $"File%s{this.Id}.fs" + if this.IsPhysicalFile then $"%s{this.Id}.fs" else $"File%s{this.Id}.fs" member this.SignatureFileName = $"{this.FileName}i" member this.TypeName = $"T{this.Id}V_{this.PublicVersion}" @@ -223,7 +221,7 @@ let sourceFile fileId deps = Source = "" ExtraSource = "" EntryPoint = false - PhysicalFileName = None } + IsPhysicalFile = false } let OptionsCache = ConcurrentDictionary() @@ -530,7 +528,7 @@ let mkSyntheticProjectForResponseFile (responseFile: FileInfo) : SyntheticProjec Source = File.ReadAllText implPath ExtraSource = "" EntryPoint = false - PhysicalFileName = Some implPath + IsPhysicalFile = true } ) From 91cf9c102c054d4a7673563c4cbab4cee835c599 Mon Sep 17 00:00:00 2001 From: nojaf Date: Tue, 30 Jan 2024 15:06:52 +0100 Subject: [PATCH 5/5] Use SyntheticProject.Create --- tests/FSharp.Test.Utilities/ProjectGeneration.fs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/FSharp.Test.Utilities/ProjectGeneration.fs b/tests/FSharp.Test.Utilities/ProjectGeneration.fs index c591dcc4b70..22dcc12c184 100644 --- a/tests/FSharp.Test.Utilities/ProjectGeneration.fs +++ b/tests/FSharp.Test.Utilities/ProjectGeneration.fs @@ -536,18 +536,12 @@ let mkSyntheticProjectForResponseFile (responseFile: FileInfo) : SyntheticProjec compilerArgs |> Array.filter (fun line -> not (isFSharpFile line)) |> Array.toList - - { - Name = Path.GetFileNameWithoutExtension responseFile.Name + + { SyntheticProject.Create(Path.GetFileNameWithoutExtension responseFile.Name) with ProjectDir = responseFile.DirectoryName SourceFiles = sourceFiles - DependsOn = List.empty - RecursiveNamespace = false OtherOptions = otherOptions AutoAddModules = false - NugetReferences = List.empty - FrameworkReferences = List.empty - SkipInitialCheck = false } []