From d73ab48a28da4bea2846cd03c10305b1f962966c Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 22 Nov 2022 15:04:40 +0100 Subject: [PATCH 01/44] Revive ilxgen para --- src/Compiler/CodeGen/IlxGen.fs | 75 +++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index a5dad6c8907..7e89bea5c53 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -6,6 +6,7 @@ module internal FSharp.Compiler.IlxGen open System.IO open System.Reflection open System.Collections.Generic +open System.Collections.Immutable open FSharp.Compiler.IO open Internal.Utilities @@ -42,6 +43,8 @@ open FSharp.Compiler.TypeRelations let IlxGenStackGuardDepth = StackGuard.GetDepthOption "IlxGen" +let getEmptyStackGuard() = StackGuard(IlxGenStackGuardDepth, "IlxAssemblyGenerator") + let IsNonErasedTypar (tp: Typar) = not tp.IsErased let DropErasedTypars (tps: Typar list) = tps |> List.filter IsNonErasedTypar @@ -333,6 +336,12 @@ type cenv = /// Used to apply forced inlining optimizations to witnesses generated late during codegen mutable optimizeDuringCodeGen: bool -> Expr -> Expr + /// What depth are we at when generating an expression? + //mutable exprRecursionDepth: int + + /// Delayed Method Generation - prevents stack overflows when we need to generate methods that are split into many methods by the optimizer. + delayedGenMethods: Queue unit> + /// Guard the stack and move to a new one if necessary mutable stackGuard: StackGuard @@ -1229,6 +1238,12 @@ and IlxGenEnv = /// Indicates that the .locals init flag should be set on a method and all its nested methods and lambdas initLocals: bool + + /// Delay code gen for files. + delayCodeGen: bool + + /// Collection of code-gen functions where each function represents a file. + delayedFileGen: ImmutableArray<(cenv -> unit) []> } override _.ToString() = "" @@ -3109,6 +3124,19 @@ and CodeGenMethodForExpr cenv mgbuf (entryPointInfo, methodName, eenv, alreadyUs code +and DelayCodeGenMethodForExpr cenv mgbuf (entryPointInfo, methodName, eenv, alreadyUsedArgs, selfArgOpt, expr0, sequel0) = + let ilLazyCode = + lazy + CodeGenMethodForExpr { cenv with stackGuard = getEmptyStackGuard(); delayedGenMethods = Queue() } mgbuf (entryPointInfo, methodName, eenv, alreadyUsedArgs, selfArgOpt, expr0, sequel0) + + if (* cenv.exprRecursionDepth > 0 || *) eenv.delayCodeGen then + cenv.delayedGenMethods.Enqueue(fun _ -> ilLazyCode.Force() |> ignore) + else + // Eagerly codegen if we are not in an expression depth. + ilLazyCode.Force() |> ignore + + ilLazyCode + //-------------------------------------------------------------------------- // Generate sequels //-------------------------------------------------------------------------- @@ -8349,15 +8377,27 @@ and GenBindingAfterDebugPoint cenv cgbuf eenv bind isStateVar startMarkOpt = cgbuf.mgbuf.AddOrMergePropertyDef(ilGetterMethSpec.MethodRef.DeclaringTypeRef, ilPropDef, m) let ilMethodDef = - let ilCode = - CodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) - - let ilMethodBody = MethodBody.IL(lazy ilCode) + let ilLazyCode = + if eenv.delayCodeGen then + DelayCodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) + else + let ilCode = CodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) + lazy ilCode - (mkILStaticMethod ([], ilGetterMethSpec.Name, access, [], mkILReturn ilTy, ilMethodBody)) - .WithSpecialName + let ilMethodBody = MethodBody.IL(ilLazyCode) + (mkILStaticMethod ([], ilGetterMethSpec.Name, access, [], mkILReturn ilTy, ilMethodBody)).WithSpecialName |> AddNonUserCompilerGeneratedAttribs g + //let ilMethodDef = + // let ilCode = + // CodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) + + // let ilMethodBody = MethodBody.IL(lazy ilCode) + + // (mkILStaticMethod ([], ilGetterMethSpec.Name, access, [], mkILReturn ilTy, ilMethodBody)) + // .WithSpecialName + // |> AddNonUserCompilerGeneratedAttribs g + CountMethodDef() cgbuf.mgbuf.AddMethodDef(ilGetterMethSpec.MethodRef.DeclaringTypeRef, ilMethodDef) @@ -9069,11 +9109,10 @@ and GenMethodForBinding | [ h ] -> Some h | _ -> None - let ilCodeLazy = - CodeGenMethodForExpr cenv mgbuf (tailCallInfo, mspec.Name, eenvForMeth, 0, selfValOpt, bodyExpr, sequel) + let ilLazyCode = DelayCodeGenMethodForExpr cenv mgbuf (tailCallInfo, mspec.Name, { eenvForMeth with delayCodeGen = false }, 0, selfValOpt, bodyExpr, sequel) // This is the main code generation for most methods - false, MethodBody.IL(notlazy ilCodeLazy), false + false, MethodBody.IL(ilLazyCode), false // Do not generate DllImport attributes into the code - they are implicit from the P/Invoke let attrs = @@ -10277,7 +10316,9 @@ and GenImplFile cenv (mgbuf: AssemblyBuilder) mainInfoOpt eenv (implFile: Checke AddBindingsForLocalModuleOrNamespaceType allocVal clocCcu eenv signature - eenvafter + let eenvfinal = { eenvafter with delayedFileGen = eenvafter.delayedFileGen.Add(cenv.delayedGenMethods |> Array.ofSeq) } + cenv.delayedGenMethods.Clear() + eenvfinal and GenForceWholeFileInitializationAsPartOfCCtor cenv (mgbuf: AssemblyBuilder) (lazyInitInfo: ResizeArray<_>) tref imports m = // Authoring a .cctor with effects forces the cctor for the 'initialization' module by doing a dummy store & load of a field @@ -11555,6 +11596,14 @@ let CodegenAssembly cenv eenv mgbuf implFiles = let eenv = List.fold (GenImplFile cenv mgbuf None) eenv firstImplFiles let eenv = GenImplFile cenv mgbuf cenv.options.mainMethodInfo eenv lastImplFile + let genMeths = eenv.delayedFileGen |> Array.ofSeq + + genMeths + |> ArrayParallel.iter (fun genMeths -> + genMeths + |> Array.iter (fun gen -> gen cenv) + ) + // Some constructs generate residue types and bindings. Generate these now. They don't result in any // top-level initialization code. let extraBindings = mgbuf.GrabExtraBindingsToGenerate() @@ -11584,6 +11633,7 @@ let CodegenAssembly cenv eenv mgbuf implFiles = range0) //printfn "#_emptyTopInstrs = %d" _emptyTopInstrs.Length () + mgbuf.AddInitializeScriptsInOrderToEntryPoint(eenv.imports) @@ -11612,6 +11662,8 @@ let GetEmptyIlxGenEnv (g: TcGlobals) ccu = isInLoop = false initLocals = true imports = None + delayCodeGen = true + delayedFileGen = ImmutableArray.Empty } type IlxGenResults = @@ -11871,7 +11923,8 @@ type IlxAssemblyGenerator(amap: ImportMap, tcGlobals: TcGlobals, tcVal: Constrai intraAssemblyInfo = intraAssemblyInfo optionsOpt = None optimizeDuringCodeGen = (fun _flag expr -> expr) - stackGuard = StackGuard(IlxGenStackGuardDepth, "IlxAssemblyGenerator") + stackGuard = getEmptyStackGuard() + delayedGenMethods = Queue () } /// Register a set of referenced assemblies with the ILX code generator From e136bebda5328b7f90c4f4db83a78700f1347f6f Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 22 Nov 2022 15:21:19 +0100 Subject: [PATCH 02/44] formatting applied --- src/Compiler/CodeGen/IlxGen.fs | 53 ++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 7e89bea5c53..00427d99435 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -43,7 +43,8 @@ open FSharp.Compiler.TypeRelations let IlxGenStackGuardDepth = StackGuard.GetDepthOption "IlxGen" -let getEmptyStackGuard() = StackGuard(IlxGenStackGuardDepth, "IlxAssemblyGenerator") +let getEmptyStackGuard () = + StackGuard(IlxGenStackGuardDepth, "IlxAssemblyGenerator") let IsNonErasedTypar (tp: Typar) = not tp.IsErased @@ -1242,8 +1243,8 @@ and IlxGenEnv = /// Delay code gen for files. delayCodeGen: bool - /// Collection of code-gen functions where each function represents a file. - delayedFileGen: ImmutableArray<(cenv -> unit) []> + /// Collection of code-gen functions where each function represents a file. + delayedFileGen: ImmutableArray<(cenv -> unit)[]> } override _.ToString() = "" @@ -3127,7 +3128,13 @@ and CodeGenMethodForExpr cenv mgbuf (entryPointInfo, methodName, eenv, alreadyUs and DelayCodeGenMethodForExpr cenv mgbuf (entryPointInfo, methodName, eenv, alreadyUsedArgs, selfArgOpt, expr0, sequel0) = let ilLazyCode = lazy - CodeGenMethodForExpr { cenv with stackGuard = getEmptyStackGuard(); delayedGenMethods = Queue() } mgbuf (entryPointInfo, methodName, eenv, alreadyUsedArgs, selfArgOpt, expr0, sequel0) + CodeGenMethodForExpr + { cenv with + stackGuard = getEmptyStackGuard () + delayedGenMethods = Queue() + } + mgbuf + (entryPointInfo, methodName, eenv, alreadyUsedArgs, selfArgOpt, expr0, sequel0) if (* cenv.exprRecursionDepth > 0 || *) eenv.delayCodeGen then cenv.delayedGenMethods.Enqueue(fun _ -> ilLazyCode.Force() |> ignore) @@ -8381,11 +8388,15 @@ and GenBindingAfterDebugPoint cenv cgbuf eenv bind isStateVar startMarkOpt = if eenv.delayCodeGen then DelayCodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) else - let ilCode = CodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) + let ilCode = + CodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) + lazy ilCode let ilMethodBody = MethodBody.IL(ilLazyCode) - (mkILStaticMethod ([], ilGetterMethSpec.Name, access, [], mkILReturn ilTy, ilMethodBody)).WithSpecialName + + (mkILStaticMethod ([], ilGetterMethSpec.Name, access, [], mkILReturn ilTy, ilMethodBody)) + .WithSpecialName |> AddNonUserCompilerGeneratedAttribs g //let ilMethodDef = @@ -9109,7 +9120,19 @@ and GenMethodForBinding | [ h ] -> Some h | _ -> None - let ilLazyCode = DelayCodeGenMethodForExpr cenv mgbuf (tailCallInfo, mspec.Name, { eenvForMeth with delayCodeGen = false }, 0, selfValOpt, bodyExpr, sequel) + let ilLazyCode = + DelayCodeGenMethodForExpr + cenv + mgbuf + (tailCallInfo, + mspec.Name, + { eenvForMeth with + delayCodeGen = false + }, + 0, + selfValOpt, + bodyExpr, + sequel) // This is the main code generation for most methods false, MethodBody.IL(ilLazyCode), false @@ -10316,7 +10339,11 @@ and GenImplFile cenv (mgbuf: AssemblyBuilder) mainInfoOpt eenv (implFile: Checke AddBindingsForLocalModuleOrNamespaceType allocVal clocCcu eenv signature - let eenvfinal = { eenvafter with delayedFileGen = eenvafter.delayedFileGen.Add(cenv.delayedGenMethods |> Array.ofSeq) } + let eenvfinal = + { eenvafter with + delayedFileGen = eenvafter.delayedFileGen.Add(cenv.delayedGenMethods |> Array.ofSeq) + } + cenv.delayedGenMethods.Clear() eenvfinal @@ -11599,10 +11626,7 @@ let CodegenAssembly cenv eenv mgbuf implFiles = let genMeths = eenv.delayedFileGen |> Array.ofSeq genMeths - |> ArrayParallel.iter (fun genMeths -> - genMeths - |> Array.iter (fun gen -> gen cenv) - ) + |> ArrayParallel.iter (fun genMeths -> genMeths |> Array.iter (fun gen -> gen cenv)) // Some constructs generate residue types and bindings. Generate these now. They don't result in any // top-level initialization code. @@ -11633,7 +11657,6 @@ let CodegenAssembly cenv eenv mgbuf implFiles = range0) //printfn "#_emptyTopInstrs = %d" _emptyTopInstrs.Length () - mgbuf.AddInitializeScriptsInOrderToEntryPoint(eenv.imports) @@ -11923,8 +11946,8 @@ type IlxAssemblyGenerator(amap: ImportMap, tcGlobals: TcGlobals, tcVal: Constrai intraAssemblyInfo = intraAssemblyInfo optionsOpt = None optimizeDuringCodeGen = (fun _flag expr -> expr) - stackGuard = getEmptyStackGuard() - delayedGenMethods = Queue () + stackGuard = getEmptyStackGuard () + delayedGenMethods = Queue() } /// Register a set of referenced assemblies with the ILX code generator From a4a31db9f38d5a2651682cd223fd4d34a61d53fd Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 22 Nov 2022 17:06:24 +0100 Subject: [PATCH 03/44] Safer stack handling in recursive scenarios --- src/Compiler/CodeGen/IlxGen.fs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 955138a844d..99db8957b8b 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -6,7 +6,6 @@ module internal FSharp.Compiler.IlxGen open System.IO open System.Reflection open System.Collections.Generic -open System.Collections.Immutable open FSharp.Compiler.IO open Internal.Utilities @@ -1244,7 +1243,7 @@ and IlxGenEnv = delayCodeGen: bool /// Collection of code-gen functions where each function represents a file. - delayedFileGen: ImmutableArray<(cenv -> unit)[]> + delayedFileGenReverse: list<(cenv -> unit)[]> } override _.ToString() = "" @@ -3125,16 +3124,17 @@ and CodeGenMethodForExpr cenv mgbuf (entryPointInfo, methodName, eenv, alreadyUs code -and DelayCodeGenMethodForExpr cenv mgbuf (entryPointInfo, methodName, eenv, alreadyUsedArgs, selfArgOpt, expr0, sequel0) = +and DelayCodeGenMethodForExpr cenv mgbuf ((_, _,eenv,_, _, _,_) as args) = + let ilLazyCode = lazy CodeGenMethodForExpr { cenv with - stackGuard = getEmptyStackGuard () + stackGuard = if eenv.delayCodeGen then getEmptyStackGuard() else cenv.stackGuard delayedGenMethods = Queue() } mgbuf - (entryPointInfo, methodName, eenv, alreadyUsedArgs, selfArgOpt, expr0, sequel0) + args if (* cenv.exprRecursionDepth > 0 || *) eenv.delayCodeGen then cenv.delayedGenMethods.Enqueue(fun _ -> ilLazyCode.Force() |> ignore) @@ -10341,7 +10341,7 @@ and GenImplFile cenv (mgbuf: AssemblyBuilder) mainInfoOpt eenv (implFile: Checke let eenvfinal = { eenvafter with - delayedFileGen = eenvafter.delayedFileGen.Add(cenv.delayedGenMethods |> Array.ofSeq) + delayedFileGenReverse = (cenv.delayedGenMethods |> Array.ofSeq) :: eenvafter.delayedFileGenReverse } cenv.delayedGenMethods.Clear() @@ -11630,10 +11630,12 @@ let CodegenAssembly cenv eenv mgbuf implFiles = let eenv = List.fold (GenImplFile cenv mgbuf None) eenv firstImplFiles let eenv = GenImplFile cenv mgbuf cenv.options.mainMethodInfo eenv lastImplFile - let genMeths = eenv.delayedFileGen |> Array.ofSeq - - genMeths + + eenv.delayedFileGenReverse + |> Array.ofList + |> Array.rev |> ArrayParallel.iter (fun genMeths -> genMeths |> Array.iter (fun gen -> gen cenv)) + // Some constructs generate residue types and bindings. Generate these now. They don't result in any // top-level initialization code. @@ -11693,7 +11695,7 @@ let GetEmptyIlxGenEnv (g: TcGlobals) ccu = initLocals = true imports = None delayCodeGen = true - delayedFileGen = ImmutableArray.Empty + delayedFileGenReverse = [] } type IlxGenResults = From eff765af09c87978efe0579a1e362d218c66dcd5 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 22 Nov 2022 17:17:06 +0100 Subject: [PATCH 04/44] reformatted --- src/Compiler/CodeGen/IlxGen.fs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 99db8957b8b..7a632be76cd 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -3124,13 +3124,17 @@ and CodeGenMethodForExpr cenv mgbuf (entryPointInfo, methodName, eenv, alreadyUs code -and DelayCodeGenMethodForExpr cenv mgbuf ((_, _,eenv,_, _, _,_) as args) = +and DelayCodeGenMethodForExpr cenv mgbuf ((_, _, eenv, _, _, _, _) as args) = let ilLazyCode = lazy CodeGenMethodForExpr { cenv with - stackGuard = if eenv.delayCodeGen then getEmptyStackGuard() else cenv.stackGuard + stackGuard = + if eenv.delayCodeGen then + getEmptyStackGuard () + else + cenv.stackGuard delayedGenMethods = Queue() } mgbuf @@ -11630,12 +11634,10 @@ let CodegenAssembly cenv eenv mgbuf implFiles = let eenv = List.fold (GenImplFile cenv mgbuf None) eenv firstImplFiles let eenv = GenImplFile cenv mgbuf cenv.options.mainMethodInfo eenv lastImplFile - eenv.delayedFileGenReverse |> Array.ofList |> Array.rev |> ArrayParallel.iter (fun genMeths -> genMeths |> Array.iter (fun gen -> gen cenv)) - // Some constructs generate residue types and bindings. Generate these now. They don't result in any // top-level initialization code. From bfacdf404ca3a285945ea659de714d3a50f9a54a Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 22 Nov 2022 17:41:48 +0100 Subject: [PATCH 05/44] GenerateResourcesForQuotations --- src/Compiler/CodeGen/IlxGen.fs | 81 ++++++++++++++++------------------ 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 7a632be76cd..af9f9ec28f1 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -11627,10 +11627,6 @@ let CodegenAssembly cenv eenv mgbuf implFiles = | None -> () | Some (firstImplFiles, lastImplFile) -> - // Generate the assembly sequentially, implementation file by implementation file. - // - // NOTE: In theory this could be done in parallel, except for the presence of linear - // state in the AssemblyBuilder let eenv = List.fold (GenImplFile cenv mgbuf None) eenv firstImplFiles let eenv = GenImplFile cenv mgbuf cenv.options.mainMethodInfo eenv lastImplFile @@ -11710,6 +11706,43 @@ type IlxGenResults = quotationResourceInfo: (ILTypeRef list * byte[]) list } +let private GenerateResourcesForQuotations reflectedDefinitions cenv = + match reflectedDefinitions with + | [] -> [] + | _ -> + let qscope = + QuotationTranslator.QuotationGenerationScope.Create( + cenv.g, + cenv.amap, + cenv.viewCcu, + cenv.tcVal, + QuotationTranslator.IsReflectedDefinition.Yes + ) + + let defns = + reflectedDefinitions + |> List.choose (fun ((methName, v), e) -> + try + let mbaseR, astExpr = + QuotationTranslator.ConvReflectedDefinition qscope methName v e + + Some(mbaseR, astExpr) + with QuotationTranslator.InvalidQuotedTerm e -> + warning e + None) + + let referencedTypeDefs, typeSplices, exprSplices = qscope.Close() + + for _typeSplice, m in typeSplices do + error (InternalError("A free type variable was detected in a reflected definition", m)) + + for _exprSplice, m in exprSplices do + error (Error(FSComp.SR.ilReflectedDefinitionsCannotUseSliceOperator (), m)) + + let defnsResourceBytes = defns |> QuotationPickler.PickleDefns + + [ (referencedTypeDefs, defnsResourceBytes) ] + let GenerateCode (cenv, anonTypeTable, eenv, CheckedAssemblyAfterOptimization implFiles, assemAttribs, moduleAttribs) = use _ = UseBuildPhase BuildPhase.IlxGen @@ -11743,45 +11776,7 @@ let GenerateCode (cenv, anonTypeTable, eenv, CheckedAssemblyAfterOptimization im GenAttrs cenv eenv (assemAttribs |> List.filter (fun a -> not (IsAssemblyVersionAttribute g a))) let tdefs, reflectedDefinitions = mgbuf.Close() - - // Generate the quotations - let quotationResourceInfo = - match reflectedDefinitions with - | [] -> [] - | _ -> - let qscope = - QuotationTranslator.QuotationGenerationScope.Create( - g, - cenv.amap, - cenv.viewCcu, - cenv.tcVal, - QuotationTranslator.IsReflectedDefinition.Yes - ) - - let defns = - reflectedDefinitions - |> List.choose (fun ((methName, v), e) -> - try - let mbaseR, astExpr = - QuotationTranslator.ConvReflectedDefinition qscope methName v e - - Some(mbaseR, astExpr) - with QuotationTranslator.InvalidQuotedTerm e -> - warning e - None) - - let referencedTypeDefs, typeSplices, exprSplices = qscope.Close() - - for _typeSplice, m in typeSplices do - error (InternalError("A free type variable was detected in a reflected definition", m)) - - for _exprSplice, m in exprSplices do - error (Error(FSComp.SR.ilReflectedDefinitionsCannotUseSliceOperator (), m)) - - let defnsResourceBytes = defns |> QuotationPickler.PickleDefns - - [ (referencedTypeDefs, defnsResourceBytes) ] - + let quotationResourceInfo = GenerateResourcesForQuotations reflectedDefinitions cenv let ilNetModuleAttrs = GenAttrs cenv eenv moduleAttribs let casApplied = Dictionary() From 6968abfb773fa8f7c85074f5d141c81692c93f7f Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Wed, 23 Nov 2022 10:00:25 +0100 Subject: [PATCH 06/44] cleanup --- src/Compiler/CodeGen/IlxGen.fs | 20 ++++---------------- src/Compiler/Driver/CompilerConfig.fs | 3 +++ src/Compiler/Driver/CompilerConfig.fsi | 4 ++++ src/Compiler/Driver/CompilerOptions.fs | 1 + 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index af9f9ec28f1..f7aebdacb6e 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -336,10 +336,7 @@ type cenv = /// Used to apply forced inlining optimizations to witnesses generated late during codegen mutable optimizeDuringCodeGen: bool -> Expr -> Expr - /// What depth are we at when generating an expression? - //mutable exprRecursionDepth: int - - /// Delayed Method Generation - prevents stack overflows when we need to generate methods that are split into many methods by the optimizer. + /// Delayed Method Generation - which can later be parallelized across multiple files delayedGenMethods: Queue unit> /// Guard the stack and move to a new one if necessary @@ -3140,10 +3137,9 @@ and DelayCodeGenMethodForExpr cenv mgbuf ((_, _, eenv, _, _, _, _) as args) = mgbuf args - if (* cenv.exprRecursionDepth > 0 || *) eenv.delayCodeGen then + if eenv.delayCodeGen then cenv.delayedGenMethods.Enqueue(fun _ -> ilLazyCode.Force() |> ignore) - else - // Eagerly codegen if we are not in an expression depth. + else ilLazyCode.Force() |> ignore ilLazyCode @@ -8388,15 +8384,7 @@ and GenBindingAfterDebugPoint cenv cgbuf eenv bind isStateVar startMarkOpt = cgbuf.mgbuf.AddOrMergePropertyDef(ilGetterMethSpec.MethodRef.DeclaringTypeRef, ilPropDef, m) let ilMethodDef = - let ilLazyCode = - if eenv.delayCodeGen then - DelayCodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) - else - let ilCode = - CodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) - - lazy ilCode - + let ilLazyCode = DelayCodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) let ilMethodBody = MethodBody.IL(ilLazyCode) (mkILStaticMethod ([], ilGetterMethSpec.Name, access, [], mkILReturn ilTy, ilMethodBody)) diff --git a/src/Compiler/Driver/CompilerConfig.fs b/src/Compiler/Driver/CompilerConfig.fs index 91a2e9fde3d..2d047010945 100644 --- a/src/Compiler/Driver/CompilerConfig.fs +++ b/src/Compiler/Driver/CompilerConfig.fs @@ -508,6 +508,7 @@ type TcConfigBuilder = mutable deterministic: bool mutable concurrentBuild: bool mutable parallelCheckingWithSignatureFiles: bool + mutable parallelIlxGen: bool mutable emitMetadataAssembly: MetadataAssemblyGeneration mutable preferredUiLang: string option mutable lcid: int option @@ -734,6 +735,7 @@ type TcConfigBuilder = deterministic = false concurrentBuild = true parallelCheckingWithSignatureFiles = false + parallelIlxGen = false emitMetadataAssembly = MetadataAssemblyGeneration.None preferredUiLang = None lcid = None @@ -1287,6 +1289,7 @@ type TcConfig private (data: TcConfigBuilder, validate: bool) = member _.deterministic = data.deterministic member _.concurrentBuild = data.concurrentBuild member _.parallelCheckingWithSignatureFiles = data.parallelCheckingWithSignatureFiles + member _.parallelIlxGen = data.parallelIlxGen member _.emitMetadataAssembly = data.emitMetadataAssembly member _.pathMap = data.pathMap member _.langVersion = data.langVersion diff --git a/src/Compiler/Driver/CompilerConfig.fsi b/src/Compiler/Driver/CompilerConfig.fsi index 70abf7beb63..30cb723bca4 100644 --- a/src/Compiler/Driver/CompilerConfig.fsi +++ b/src/Compiler/Driver/CompilerConfig.fsi @@ -414,6 +414,8 @@ type TcConfigBuilder = mutable parallelCheckingWithSignatureFiles: bool + mutable parallelIlxGen: bool + mutable emitMetadataAssembly: MetadataAssemblyGeneration mutable preferredUiLang: string option @@ -734,6 +736,8 @@ type TcConfig = member parallelCheckingWithSignatureFiles: bool + member parallelIlxGen: bool + member emitMetadataAssembly: MetadataAssemblyGeneration member pathMap: PathMap diff --git a/src/Compiler/Driver/CompilerOptions.fs b/src/Compiler/Driver/CompilerOptions.fs index 02c3306b2d9..bd1c1c5d9ec 100644 --- a/src/Compiler/Driver/CompilerOptions.fs +++ b/src/Compiler/Driver/CompilerOptions.fs @@ -1388,6 +1388,7 @@ let testFlag tcConfigB = | "ContinueAfterParseFailure" -> tcConfigB.continueAfterParseFailure <- true | "ParallelOff" -> tcConfigB.concurrentBuild <- false | "ParallelCheckingWithSignatureFilesOn" -> tcConfigB.parallelCheckingWithSignatureFiles <- true + | "ParallelIlxGen" -> tcConfigB.parallelIlxGen <- true #if DEBUG | "ShowParserStackOnParseError" -> showParserStackOnParseError <- true #endif From 5feaceaa161bbb25538c7683290954fd5562d487 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Wed, 23 Nov 2022 10:44:07 +0100 Subject: [PATCH 07/44] fsc.exe test switch added --- src/Compiler/CodeGen/IlxGen.fs | 14 ++++---------- src/Compiler/CodeGen/IlxGen.fsi | 3 +++ src/Compiler/Driver/OptimizeInputs.fs | 1 + 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index f7aebdacb6e..48ac5f4ae54 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -296,6 +296,9 @@ type IlxGenOptions = /// Whenever possible, use callvirt instead of call alwaysCallVirt: bool + + /// When set to true, the IlxGen will delay generation of method bodies and generated them later in parallel (parallelized across files) + parallelIlxGenEnabled: bool } /// Compilation environment for compiling a fragment of an assembly @@ -8391,16 +8394,6 @@ and GenBindingAfterDebugPoint cenv cgbuf eenv bind isStateVar startMarkOpt = .WithSpecialName |> AddNonUserCompilerGeneratedAttribs g - //let ilMethodDef = - // let ilCode = - // CodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) - - // let ilMethodBody = MethodBody.IL(lazy ilCode) - - // (mkILStaticMethod ([], ilGetterMethSpec.Name, access, [], mkILReturn ilTy, ilMethodBody)) - // .WithSpecialName - // |> AddNonUserCompilerGeneratedAttribs g - CountMethodDef() cgbuf.mgbuf.AddMethodDef(ilGetterMethSpec.MethodRef.DeclaringTypeRef, ilMethodDef) @@ -11742,6 +11735,7 @@ let GenerateCode (cenv, anonTypeTable, eenv, CheckedAssemblyAfterOptimization im let eenv = { eenv with cloc = CompLocForFragment cenv.options.fragName cenv.viewCcu + delayCodeGen = cenv.options.parallelIlxGenEnabled } // Generate the PrivateImplementationDetails type diff --git a/src/Compiler/CodeGen/IlxGen.fsi b/src/Compiler/CodeGen/IlxGen.fsi index d68463e9ca7..41988542d91 100644 --- a/src/Compiler/CodeGen/IlxGen.fsi +++ b/src/Compiler/CodeGen/IlxGen.fsi @@ -59,6 +59,9 @@ type internal IlxGenOptions = /// Indicates that, whenever possible, use callvirt instead of call alwaysCallVirt: bool + + /// When set to true, the IlxGen will delay generation of method bodies and generated them later in parallel (parallelized across files) + parallelIlxGenEnabled: bool } /// The results of the ILX compilation of one fragment of an assembly diff --git a/src/Compiler/Driver/OptimizeInputs.fs b/src/Compiler/Driver/OptimizeInputs.fs index 0696758780b..68c1452003b 100644 --- a/src/Compiler/Driver/OptimizeInputs.fs +++ b/src/Compiler/Driver/OptimizeInputs.fs @@ -255,6 +255,7 @@ let GenerateIlxCode isInteractive = tcConfig.isInteractive isInteractiveItExpr = isInteractiveItExpr alwaysCallVirt = tcConfig.alwaysCallVirt + parallelIlxGenEnabled = tcConfig.parallelIlxGen } ilxGenerator.GenerateCode(ilxGenOpts, optimizedImpls, topAttrs.assemblyAttrs, topAttrs.netModuleAttrs) From b8113562687c18901307c6e89ee27eb360c25e99 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Wed, 23 Nov 2022 10:53:57 +0100 Subject: [PATCH 08/44] formatted --- src/Compiler/CodeGen/IlxGen.fs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 48ac5f4ae54..6c3b81f5a19 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -3142,7 +3142,7 @@ and DelayCodeGenMethodForExpr cenv mgbuf ((_, _, eenv, _, _, _, _) as args) = if eenv.delayCodeGen then cenv.delayedGenMethods.Enqueue(fun _ -> ilLazyCode.Force() |> ignore) - else + else ilLazyCode.Force() |> ignore ilLazyCode @@ -8387,7 +8387,9 @@ and GenBindingAfterDebugPoint cenv cgbuf eenv bind isStateVar startMarkOpt = cgbuf.mgbuf.AddOrMergePropertyDef(ilGetterMethSpec.MethodRef.DeclaringTypeRef, ilPropDef, m) let ilMethodDef = - let ilLazyCode = DelayCodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) + let ilLazyCode = + DelayCodeGenMethodForExpr cenv cgbuf.mgbuf ([], ilGetterMethSpec.Name, eenv, 0, None, rhsExpr, Return) + let ilMethodBody = MethodBody.IL(ilLazyCode) (mkILStaticMethod ([], ilGetterMethSpec.Name, access, [], mkILReturn ilTy, ilMethodBody)) @@ -11687,7 +11689,7 @@ type IlxGenResults = quotationResourceInfo: (ILTypeRef list * byte[]) list } -let private GenerateResourcesForQuotations reflectedDefinitions cenv = +let private GenerateResourcesForQuotations reflectedDefinitions cenv = match reflectedDefinitions with | [] -> [] | _ -> @@ -11758,7 +11760,7 @@ let GenerateCode (cenv, anonTypeTable, eenv, CheckedAssemblyAfterOptimization im GenAttrs cenv eenv (assemAttribs |> List.filter (fun a -> not (IsAssemblyVersionAttribute g a))) let tdefs, reflectedDefinitions = mgbuf.Close() - let quotationResourceInfo = GenerateResourcesForQuotations reflectedDefinitions cenv + let quotationResourceInfo = GenerateResourcesForQuotations reflectedDefinitions cenv let ilNetModuleAttrs = GenAttrs cenv eenv moduleAttribs let casApplied = Dictionary() From 480a54d5f86c49a3b086b289ae408d4f0f3760ea Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Wed, 23 Nov 2022 11:09:21 +0100 Subject: [PATCH 09/44] Automatically enabled experimental features (based on env var) --- src/Compiler/Driver/CompilerConfig.fs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Compiler/Driver/CompilerConfig.fs b/src/Compiler/Driver/CompilerConfig.fs index 2d047010945..4c34aac876d 100644 --- a/src/Compiler/Driver/CompilerConfig.fs +++ b/src/Compiler/Driver/CompilerConfig.fs @@ -47,6 +47,9 @@ let FSharpScriptFileSuffixes = [ ".fsscript"; ".fsx" ] let FSharpIndentationAwareSyntaxFileSuffixes = [ ".fs"; ".fsscript"; ".fsx"; ".fsi" ] +let FsharpExperimentalFeaturesEnabledAutomatically = + Environment.GetEnvironmentVariable("FSHARP_EXPERIMENTAL_FEATURES") |> isNotNull + //-------------------------------------------------------------------------- // General file name resolver //-------------------------------------------------------------------------- @@ -735,7 +738,7 @@ type TcConfigBuilder = deterministic = false concurrentBuild = true parallelCheckingWithSignatureFiles = false - parallelIlxGen = false + parallelIlxGen = FsharpExperimentalFeaturesEnabledAutomatically emitMetadataAssembly = MetadataAssemblyGeneration.None preferredUiLang = None lcid = None From e48fc458d3f6574da7db3ae696583bc6d5121fa1 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Thu, 24 Nov 2022 13:37:16 +0100 Subject: [PATCH 10/44] Improved dict defaults to avoid collisions --- src/Compiler/AbstractIL/il.fs | 4 ++-- src/Compiler/AbstractIL/ilmorph.fs | 2 +- src/Compiler/Checking/MethodCalls.fs | 2 +- src/Compiler/CodeGen/IlxGen.fs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Compiler/AbstractIL/il.fs b/src/Compiler/AbstractIL/il.fs index e2cca7d1809..b3ca04aa586 100644 --- a/src/Compiler/AbstractIL/il.fs +++ b/src/Compiler/AbstractIL/il.fs @@ -3902,7 +3902,7 @@ let prependInstrsToCode (instrs: ILInstr list) (c2: ILCode) = // If there is a sequence point as the first instruction then keep it at the front | I_seqpoint _ as i0 -> let labels = - let dict = Dictionary.newWithSize c2.Labels.Count + let dict = Dictionary.newWithSize (c2.Labels.Count * 2 ) for kvp in c2.Labels do dict.Add(kvp.Key, (if kvp.Value = 0 then 0 else kvp.Value + n)) @@ -3915,7 +3915,7 @@ let prependInstrsToCode (instrs: ILInstr list) (c2: ILCode) = } | _ -> let labels = - let dict = Dictionary.newWithSize c2.Labels.Count + let dict = Dictionary.newWithSize (c2.Labels.Count * 2) for kvp in c2.Labels do dict.Add(kvp.Key, kvp.Value + n) diff --git a/src/Compiler/AbstractIL/ilmorph.fs b/src/Compiler/AbstractIL/ilmorph.fs index dad1e16369d..6b67348ddbc 100644 --- a/src/Compiler/AbstractIL/ilmorph.fs +++ b/src/Compiler/AbstractIL/ilmorph.fs @@ -37,7 +37,7 @@ let code_instr2instrs f (code: ILCode) = adjust[old] <- nw let labels = - let dict = Dictionary.newWithSize code.Labels.Count + let dict = Dictionary.newWithSize (code.Labels.Count*2) for kvp in code.Labels do dict.Add(kvp.Key, adjust[kvp.Value]) diff --git a/src/Compiler/Checking/MethodCalls.fs b/src/Compiler/Checking/MethodCalls.fs index c7523e7ba50..1175cc3dded 100644 --- a/src/Compiler/Checking/MethodCalls.fs +++ b/src/Compiler/Checking/MethodCalls.fs @@ -1808,7 +1808,7 @@ module ProvidedMethodCalls = let varConv = // note: using paramVars.Length as assumed initial size, but this might not // be the optimal value; this wasn't checked before obsoleting Dictionary.ofList - let dict = Dictionary.newWithSize paramVars.Length + let dict = Dictionary.newWithSize (paramVars.Length*2) for v, e in Seq.zip (paramVars |> Seq.map (fun x -> x.PUntaint(id, m))) (Option.toList thisArg @ allArgs) do dict.Add(v, (None, e)) dict diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 6c3b81f5a19..c79ffb4f46c 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -2663,7 +2663,7 @@ type CodeGenBuffer(m: range, mgbuf: AssemblyBuilder, methodName, alreadyUsedArgs i2) let codeLabels = - let dict = Dictionary.newWithSize (codeLabelToPC.Count + codeLabelToCodeLabel.Count) + let dict = Dictionary.newWithSize ((codeLabelToPC.Count + codeLabelToCodeLabel.Count) * 2) for kvp in codeLabelToPC do dict.Add(kvp.Key, lab2pc 0 kvp.Key) From b53b1e582ac3c29cef832a0bf6e4d6744ddf4c50 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Thu, 24 Nov 2022 13:38:13 +0100 Subject: [PATCH 11/44] format --- src/Compiler/AbstractIL/il.fs | 2 +- src/Compiler/AbstractIL/ilmorph.fs | 2 +- src/Compiler/CodeGen/IlxGen.fs | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Compiler/AbstractIL/il.fs b/src/Compiler/AbstractIL/il.fs index b3ca04aa586..e5c1861262c 100644 --- a/src/Compiler/AbstractIL/il.fs +++ b/src/Compiler/AbstractIL/il.fs @@ -3902,7 +3902,7 @@ let prependInstrsToCode (instrs: ILInstr list) (c2: ILCode) = // If there is a sequence point as the first instruction then keep it at the front | I_seqpoint _ as i0 -> let labels = - let dict = Dictionary.newWithSize (c2.Labels.Count * 2 ) + let dict = Dictionary.newWithSize (c2.Labels.Count * 2) for kvp in c2.Labels do dict.Add(kvp.Key, (if kvp.Value = 0 then 0 else kvp.Value + n)) diff --git a/src/Compiler/AbstractIL/ilmorph.fs b/src/Compiler/AbstractIL/ilmorph.fs index 6b67348ddbc..47bb985db36 100644 --- a/src/Compiler/AbstractIL/ilmorph.fs +++ b/src/Compiler/AbstractIL/ilmorph.fs @@ -37,7 +37,7 @@ let code_instr2instrs f (code: ILCode) = adjust[old] <- nw let labels = - let dict = Dictionary.newWithSize (code.Labels.Count*2) + let dict = Dictionary.newWithSize (code.Labels.Count * 2) for kvp in code.Labels do dict.Add(kvp.Key, adjust[kvp.Value]) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index c79ffb4f46c..fa3b3e7e8c6 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -2663,7 +2663,8 @@ type CodeGenBuffer(m: range, mgbuf: AssemblyBuilder, methodName, alreadyUsedArgs i2) let codeLabels = - let dict = Dictionary.newWithSize ((codeLabelToPC.Count + codeLabelToCodeLabel.Count) * 2) + let dict = + Dictionary.newWithSize ((codeLabelToPC.Count + codeLabelToCodeLabel.Count) * 2) for kvp in codeLabelToPC do dict.Add(kvp.Key, lab2pc 0 kvp.Key) From 0751caefc939762b09101c8fbc99330e2f54662c Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Thu, 24 Nov 2022 16:19:44 +0100 Subject: [PATCH 12/44] more aggressive ilxgen parallelization --- src/Compiler/CodeGen/IlxGen.fs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index fa3b3e7e8c6..b5a4cb54ac7 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -9114,9 +9114,10 @@ and GenMethodForBinding mgbuf (tailCallInfo, mspec.Name, - { eenvForMeth with - delayCodeGen = false - }, + eenvForMeth, + //{ eenvForMeth with + // delayCodeGen = false + //}, 0, selfValOpt, bodyExpr, From 327b4186d39e6c44e0db7b777ee6ceafe5025132 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Thu, 24 Nov 2022 20:34:55 +0100 Subject: [PATCH 13/44] Reducing locks in certain paths --- src/Compiler/Utilities/illib.fs | 37 +++++++++------------------------ 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/src/Compiler/Utilities/illib.fs b/src/Compiler/Utilities/illib.fs index d0bb8f7843b..7ba25bf9a4d 100644 --- a/src/Compiler/Utilities/illib.fs +++ b/src/Compiler/Utilities/illib.fs @@ -1040,29 +1040,21 @@ module CancellableAutoOpens = let cancellable = CancellableBuilder() /// Generates unique stamps -type UniqueStampGenerator<'T when 'T: equality>() = - let gate = obj () - let encodeTab = ConcurrentDictionary<'T, int>(HashIdentity.Structural) - let mutable nItems = 0 - - let encode str = - match encodeTab.TryGetValue str with - | true, idx -> idx - | _ -> - lock gate (fun () -> - let idx = nItems - encodeTab[str] <- idx - nItems <- nItems + 1 - idx) +type UniqueStampGenerator<'T when 'T: equality>() = + let encodeTab = ConcurrentDictionary<'T, Lazy>(HashIdentity.Structural) + let mutable nItems = -1 + + let computeFunc = Func<'T,_>(fun _ -> lazy( Interlocked.Increment(&nItems))) - member _.Encode str = encode str + member _.Encode str = encodeTab.GetOrAdd(str,computeFunc).Value member _.Table = encodeTab.Keys /// memoize tables (all entries cached, never collected) type MemoizationTable<'T, 'U>(compute: 'T -> 'U, keyComparer: IEqualityComparer<'T>, ?canMemoize) = - - let table = new ConcurrentDictionary<'T, 'U>(keyComparer) + + let table = new ConcurrentDictionary<'T, Lazy<'U>>(keyComparer) + let computeFunc = Func<_,_>(fun key -> lazy(compute key)) member t.Apply x = if @@ -1070,16 +1062,7 @@ type MemoizationTable<'T, 'U>(compute: 'T -> 'U, keyComparer: IEqualityComparer< | None -> true | Some f -> f x) then - match table.TryGetValue x with - | true, res -> res - | _ -> - lock table (fun () -> - match table.TryGetValue x with - | true, res -> res - | _ -> - let res = compute x - table[x] <- res - res) + table.GetOrAdd(x,computeFunc).Value else compute x From 013c93bb0e89c80165622ab8701ab0f2f6f4ab02 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Thu, 24 Nov 2022 20:45:16 +0100 Subject: [PATCH 14/44] thread safety comments --- src/Compiler/CodeGen/IlxGen.fs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index b5a4cb54ac7..2bdd86d65aa 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -1976,9 +1976,14 @@ type TypeDefBuilder(tdef: ILTypeDef, tdefDiscards) = gmethods.Add(mkILClassCtor body) and TypeDefsBuilder() = + (* TODO Tomas : This is not thread safe. Make it so + This builder acts for all types - can be accessed concurrently + The internal TypeDefBuilder's will be created from 1 file each, therefore those can remain for sequential usage. (??) + *) let tdefs: HashMultiMap = HashMultiMap(0, HashIdentity.Structural) + (* TODO Tomas : This is not thread safe. Make it so*) let mutable countDown = System.Int32.MaxValue member b.Close() = @@ -2001,6 +2006,7 @@ and TypeDefsBuilder() = yield tdef ] + (* TODO Tomas : This is not thread safe. Make it so*) member b.FindTypeDefBuilder nm = try tdefs[nm] |> snd |> fst @@ -2023,6 +2029,9 @@ and TypeDefsBuilder() = tdefs.Add(tdef.Name, (idx, (TypeDefBuilder(tdef, tdefDiscards), eliminateIfEmpty))) + (* TODO Tomas : Annonymous types are being collected outside of the delayed code path, so this can remain as is. + I can move the anontable generation step to be parallel separately, only makes sense in codebases making heavy use of them + *) type AnonTypeGenerationTable() = // Dictionary is safe here as it will only be used during the codegen stage - will happen on a single thread. let dict = @@ -2038,9 +2047,12 @@ type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbu // The definitions of top level values, as quotations. // Dictionary is safe here as it will only be used during the codegen stage - will happen on a single thread. + (* TODO Tomas : This is not thread safe. Make it so *) let mutable reflectedDefinitions: Dictionary = Dictionary(HashIdentity.Reference) + + (* TODO Tomas : This is not thread safe. Make it so *) let mutable extraBindingsToGenerate = [] // A memoization table for generating value types for big constant arrays @@ -2289,9 +2301,11 @@ type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbu (ilCtorRef, ilMethodRefs, ilTy) + (* TODO Tomas : This is not thread safe, but there will be only one entry point*) let mutable explicitEntryPointInfo: ILTypeRef option = None /// static init fields on script modules. + (* TODO Tomas : This is not thread safe. Make it so *) let mutable scriptInitFspecs: (ILFieldSpec * range) list = [] member _.AddScriptInitFieldSpec(fieldSpec, range) = From 6f877cf8bc43afc9cddfba008fe283a02af9ff1b Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Thu, 24 Nov 2022 20:46:14 +0100 Subject: [PATCH 15/44] formatted --- src/Compiler/CodeGen/IlxGen.fs | 3 +-- src/Compiler/Utilities/illib.fs | 13 +++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 2bdd86d65aa..07cd7c2fb69 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -2029,7 +2029,7 @@ and TypeDefsBuilder() = tdefs.Add(tdef.Name, (idx, (TypeDefBuilder(tdef, tdefDiscards), eliminateIfEmpty))) - (* TODO Tomas : Annonymous types are being collected outside of the delayed code path, so this can remain as is. +(* TODO Tomas : Annonymous types are being collected outside of the delayed code path, so this can remain as is. I can move the anontable generation step to be parallel separately, only makes sense in codebases making heavy use of them *) type AnonTypeGenerationTable() = @@ -2051,7 +2051,6 @@ type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbu let mutable reflectedDefinitions: Dictionary = Dictionary(HashIdentity.Reference) - (* TODO Tomas : This is not thread safe. Make it so *) let mutable extraBindingsToGenerate = [] diff --git a/src/Compiler/Utilities/illib.fs b/src/Compiler/Utilities/illib.fs index 7ba25bf9a4d..e5df5449135 100644 --- a/src/Compiler/Utilities/illib.fs +++ b/src/Compiler/Utilities/illib.fs @@ -1040,21 +1040,22 @@ module CancellableAutoOpens = let cancellable = CancellableBuilder() /// Generates unique stamps -type UniqueStampGenerator<'T when 'T: equality>() = +type UniqueStampGenerator<'T when 'T: equality>() = let encodeTab = ConcurrentDictionary<'T, Lazy>(HashIdentity.Structural) let mutable nItems = -1 - let computeFunc = Func<'T,_>(fun _ -> lazy( Interlocked.Increment(&nItems))) + let computeFunc = Func<'T, _>(fun _ -> lazy (Interlocked.Increment(&nItems))) - member _.Encode str = encodeTab.GetOrAdd(str,computeFunc).Value + member _.Encode str = + encodeTab.GetOrAdd(str, computeFunc).Value member _.Table = encodeTab.Keys /// memoize tables (all entries cached, never collected) type MemoizationTable<'T, 'U>(compute: 'T -> 'U, keyComparer: IEqualityComparer<'T>, ?canMemoize) = - + let table = new ConcurrentDictionary<'T, Lazy<'U>>(keyComparer) - let computeFunc = Func<_,_>(fun key -> lazy(compute key)) + let computeFunc = Func<_, _>(fun key -> lazy (compute key)) member t.Apply x = if @@ -1062,7 +1063,7 @@ type MemoizationTable<'T, 'U>(compute: 'T -> 'U, keyComparer: IEqualityComparer< | None -> true | Some f -> f x) then - table.GetOrAdd(x,computeFunc).Value + table.GetOrAdd(x, computeFunc).Value else compute x From 422c168fbef2a91f02f5848ccbeb79f6300fdb81 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 25 Nov 2022 10:17:58 +0100 Subject: [PATCH 16/44] A few more concurrency related changes --- src/Compiler/CodeGen/IlxGen.fs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 07cd7c2fb69..6589f722198 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -1982,8 +1982,7 @@ and TypeDefsBuilder() = *) let tdefs: HashMultiMap = HashMultiMap(0, HashIdentity.Structural) - - (* TODO Tomas : This is not thread safe. Make it so*) + let mutable countDown = System.Int32.MaxValue member b.Close() = @@ -2021,9 +2020,8 @@ and TypeDefsBuilder() = member b.AddTypeDef(tdef: ILTypeDef, eliminateIfEmpty, addAtEnd, tdefDiscards) = let idx = - if addAtEnd then - (countDown <- countDown - 1 - countDown) + if addAtEnd then + System.Threading.Interlocked.Decrement(&countDown) else tdefs.Count @@ -2052,7 +2050,7 @@ type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbu Dictionary(HashIdentity.Reference) (* TODO Tomas : This is not thread safe. Make it so *) - let mutable extraBindingsToGenerate = [] + let extraBindingsToGenerate = System.Collections.Concurrent.ConcurrentStack() // A memoization table for generating value types for big constant arrays let rawDataValueTypeGenerator = @@ -2283,20 +2281,20 @@ type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbu mgbuf.AddTypeDef(ilTypeRef, ilTypeDef, false, true, None) let extraBindings = - [ + [| yield! AugmentWithHashCompare.MakeBindingsForCompareAugmentation g tycon yield! AugmentWithHashCompare.MakeBindingsForCompareWithComparerAugmentation g tycon yield! AugmentWithHashCompare.MakeBindingsForEqualityWithComparerAugmentation g tycon yield! AugmentWithHashCompare.MakeBindingsForEqualsAugmentation g tycon - ] + |] let optimizedExtraBindings = extraBindings - |> List.map (fun (TBind (a, b, c)) -> + |> Array.map (fun (TBind (a, b, c)) -> // Disable method splitting for bindings related to anonymous records TBind(a, cenv.optimizeDuringCodeGen true b, c)) - - extraBindingsToGenerate <- optimizedExtraBindings @ extraBindingsToGenerate + + extraBindingsToGenerate.PushRange(optimizedExtraBindings) (ilCtorRef, ilMethodRefs, ilTy) @@ -2360,8 +2358,8 @@ type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbu anonTypeTable.Table[anonInfo.Stamp] member _.GrabExtraBindingsToGenerate() = - let result = extraBindingsToGenerate - extraBindingsToGenerate <- [] + let result = extraBindingsToGenerate.ToArray() + extraBindingsToGenerate.Clear() result member _.AddTypeDef(tref: ILTypeRef, tdef, eliminateIfEmpty, addAtEnd, tdefDiscards) = @@ -11637,7 +11635,7 @@ let CodegenAssembly cenv eenv mgbuf implFiles = // top-level initialization code. let extraBindings = mgbuf.GrabExtraBindingsToGenerate() //printfn "#extraBindings = %d" extraBindings.Length - if not (isNil extraBindings) then + if extraBindings.Length > 0 then let mexpr = TMDefs [ for b in extraBindings -> TMDefLet(b, range0) ] let _emptyTopInstrs, _emptyTopCode = From 1c63f9e5dc6956a22986449ee74b06bc9578abcb Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 25 Nov 2022 10:19:49 +0100 Subject: [PATCH 17/44] formatting --- src/Compiler/CodeGen/IlxGen.fs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 6589f722198..9edd7c88163 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -1982,7 +1982,7 @@ and TypeDefsBuilder() = *) let tdefs: HashMultiMap = HashMultiMap(0, HashIdentity.Structural) - + let mutable countDown = System.Int32.MaxValue member b.Close() = @@ -2020,8 +2020,8 @@ and TypeDefsBuilder() = member b.AddTypeDef(tdef: ILTypeDef, eliminateIfEmpty, addAtEnd, tdefDiscards) = let idx = - if addAtEnd then - System.Threading.Interlocked.Decrement(&countDown) + if addAtEnd then + System.Threading.Interlocked.Decrement(&countDown) else tdefs.Count @@ -2293,7 +2293,7 @@ type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbu |> Array.map (fun (TBind (a, b, c)) -> // Disable method splitting for bindings related to anonymous records TBind(a, cenv.optimizeDuringCodeGen true b, c)) - + extraBindingsToGenerate.PushRange(optimizedExtraBindings) (ilCtorRef, ilMethodRefs, ilTy) From 5810320a04382c2af4d06e3d3658e2a3c6673d17 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 25 Nov 2022 12:04:01 +0100 Subject: [PATCH 18/44] Thread safety in AnonTypeGenerationTable and AssemblyBuilder --- src/Compiler/CodeGen/IlxGen.fs | 136 +++++++++++++++---------------- src/Compiler/Utilities/illib.fs | 21 +++++ src/Compiler/Utilities/illib.fsi | 10 +++ 3 files changed, 97 insertions(+), 70 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 9edd7c88163..a15b62df563 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -6,6 +6,8 @@ module internal FSharp.Compiler.IlxGen open System.IO open System.Reflection open System.Collections.Generic +open System.Collections.Concurrent +open System.Threading open FSharp.Compiler.IO open Internal.Utilities @@ -2021,55 +2023,21 @@ and TypeDefsBuilder() = member b.AddTypeDef(tdef: ILTypeDef, eliminateIfEmpty, addAtEnd, tdefDiscards) = let idx = if addAtEnd then - System.Threading.Interlocked.Decrement(&countDown) + Interlocked.Decrement(&countDown) else tdefs.Count tdefs.Add(tdef.Name, (idx, (TypeDefBuilder(tdef, tdefDiscards), eliminateIfEmpty))) -(* TODO Tomas : Annonymous types are being collected outside of the delayed code path, so this can remain as is. - I can move the anontable generation step to be parallel separately, only makes sense in codebases making heavy use of them - *) type AnonTypeGenerationTable() = - // Dictionary is safe here as it will only be used during the codegen stage - will happen on a single thread. let dict = - Dictionary(HashIdentity.Structural) - - member _.Table = dict - -/// Assembly generation buffers -type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbuf = - let g = cenv.g - // The Abstract IL table of types - let gtdefs = TypeDefsBuilder() + ConcurrentDictionary>(HashIdentity.Structural) - // The definitions of top level values, as quotations. - // Dictionary is safe here as it will only be used during the codegen stage - will happen on a single thread. - (* TODO Tomas : This is not thread safe. Make it so *) - let mutable reflectedDefinitions: Dictionary = - Dictionary(HashIdentity.Reference) + let extraBindingsToGenerate = ConcurrentStack() - (* TODO Tomas : This is not thread safe. Make it so *) - let extraBindingsToGenerate = System.Collections.Concurrent.ConcurrentStack() - - // A memoization table for generating value types for big constant arrays - let rawDataValueTypeGenerator = - MemoizationTable( - (fun (cloc, size) -> - let name = - CompilerGeneratedName("T" + string (newUnique ()) + "_" + string size + "Bytes") // Type names ending ...$T_37Bytes - - let vtdef = mkRawDataValueTypeDef g.iltyp_ValueType (name, size, 0us) - let vtref = NestedTypeRefForCompLoc cloc vtdef.Name - let vtspec = mkILTySpec (vtref, []) - let vtdef = vtdef.WithAccess(ComputeTypeAccess vtref true) - mgbuf.AddTypeDef(vtref, vtdef, false, true, None) - vtspec), - keyComparer = HashIdentity.Structural - ) - - let generateAnonType genToStringMethod (isStruct, ilTypeRef, nms) = + let generateAnonType cenv (mgbuf: AssemblyBuilder) genToStringMethod (isStruct, ilTypeRef, nms) = + let g = cenv.g let propTys = [ for i, nm in Array.indexed nms -> nm, ILType.TypeVar(uint16 i) ] // Note that this alternative below would give the same names as C#, but the generated @@ -2298,15 +2266,63 @@ type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbu (ilCtorRef, ilMethodRefs, ilTy) - (* TODO Tomas : This is not thread safe, but there will be only one entry point*) + member this.GenerateAnonType(cenv, mgbuf, genToStringMethod, anonInfo: AnonRecdTypeInfo) = + let isStruct = evalAnonInfoIsStruct anonInfo + let key = anonInfo.Stamp + + let at = + dict.GetOrAdd(key, lazy (generateAnonType cenv mgbuf genToStringMethod (isStruct, anonInfo.ILTypeRef, anonInfo.SortedNames))) + + at.Force() |> ignore + + member this.LookupAnonType(cenv, mgbuf, genToStringMethod, anonInfo: AnonRecdTypeInfo) = + match dict.TryGetValue anonInfo.Stamp with + | true, res -> res.Value + | _ -> + if anonInfo.ILTypeRef.Scope.IsLocalRef then + failwithf "the anonymous record %A has not been generated in the pre-phase of generating this module" anonInfo.ILTypeRef + + this.GenerateAnonType(cenv, mgbuf, genToStringMethod, anonInfo) + dict[anonInfo.Stamp].Value + + member _.GrabExtraBindingsToGenerate() = + let result = extraBindingsToGenerate.ToArray() + extraBindingsToGenerate.Clear() + result + +/// Assembly generation buffers +and AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbuf = + let g = cenv.g + // The Abstract IL table of types + let gtdefs = TypeDefsBuilder() + + // The definitions of top level values, as quotations + let reflectedDefinitions = + new StampedDictionairy(HashIdentity.Reference) + + // A memoization table for generating value types for big constant arrays + let rawDataValueTypeGenerator = + MemoizationTable( + (fun (cloc, size) -> + let name = + CompilerGeneratedName("T" + string (newUnique ()) + "_" + string size + "Bytes") // Type names ending ...$T_37Bytes + + let vtdef = mkRawDataValueTypeDef g.iltyp_ValueType (name, size, 0us) + let vtref = NestedTypeRefForCompLoc cloc vtdef.Name + let vtspec = mkILTySpec (vtref, []) + let vtdef = vtdef.WithAccess(ComputeTypeAccess vtref true) + mgbuf.AddTypeDef(vtref, vtdef, false, true, None) + vtspec), + keyComparer = HashIdentity.Structural + ) + let mutable explicitEntryPointInfo: ILTypeRef option = None /// static init fields on script modules. - (* TODO Tomas : This is not thread safe. Make it so *) - let mutable scriptInitFspecs: (ILFieldSpec * range) list = [] + let scriptInitFspecs = new ConcurrentStack() member _.AddScriptInitFieldSpec(fieldSpec, range) = - scriptInitFspecs <- (fieldSpec, range) :: scriptInitFspecs + scriptInitFspecs.Push((fieldSpec, range)) /// This initializes the script in #load and fsc command-line order causing their /// side effects to be executed. @@ -2327,7 +2343,7 @@ type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbu [] ) - scriptInitFspecs |> List.iter InitializeCompiledScript + scriptInitFspecs.ToArray() |> Array.iter InitializeCompiledScript | None -> () member _.GenerateRawDataValueType(cloc, size) = @@ -2338,29 +2354,13 @@ type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbu rawDataValueTypeGenerator.Apply((cloc, size)) member _.GenerateAnonType(genToStringMethod, anonInfo: AnonRecdTypeInfo) = - let isStruct = evalAnonInfoIsStruct anonInfo - let key = anonInfo.Stamp - - if not (anonTypeTable.Table.ContainsKey key) then - let info = - generateAnonType genToStringMethod (isStruct, anonInfo.ILTypeRef, anonInfo.SortedNames) - - anonTypeTable.Table[ key ] <- info + anonTypeTable.GenerateAnonType(cenv, mgbuf, genToStringMethod, anonInfo) member this.LookupAnonType(genToStringMethod, anonInfo: AnonRecdTypeInfo) = - match anonTypeTable.Table.TryGetValue anonInfo.Stamp with - | true, res -> res - | _ -> - if anonInfo.ILTypeRef.Scope.IsLocalRef then - failwithf "the anonymous record %A has not been generated in the pre-phase of generating this module" anonInfo.ILTypeRef - - this.GenerateAnonType(genToStringMethod, anonInfo) - anonTypeTable.Table[anonInfo.Stamp] + anonTypeTable.LookupAnonType(cenv, mgbuf, genToStringMethod, anonInfo) member _.GrabExtraBindingsToGenerate() = - let result = extraBindingsToGenerate.ToArray() - extraBindingsToGenerate.Clear() - result + anonTypeTable.GrabExtraBindingsToGenerate() member _.AddTypeDef(tref: ILTypeRef, tdef, eliminateIfEmpty, addAtEnd, tdefDiscards) = gtdefs @@ -2371,14 +2371,10 @@ type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbu gtdefs.FindNestedTypeDefBuilder(tref).GetCurrentFields() member _.AddReflectedDefinition(vspec: Val, expr) = - // preserve order by storing index of item - let n = reflectedDefinitions.Count - reflectedDefinitions.Add(vspec, (vspec.CompiledName cenv.g.CompilerGlobalState, n, expr)) + reflectedDefinitions.Add(vspec, (vspec.CompiledName cenv.g.CompilerGlobalState, expr)) member _.ReplaceNameOfReflectedDefinition(vspec, newName) = - match reflectedDefinitions.TryGetValue vspec with - | true, (name, n, expr) when name <> newName -> reflectedDefinitions[vspec] <- (newName, n, expr) - | _ -> () + reflectedDefinitions.Update(vspec, (fun (oldName, expr) -> if newName = oldName then None else Some(newName, expr))) member _.AddMethodDef(tref: ILTypeRef, ilMethodDef) = gtdefs.FindNestedTypeDefBuilder(tref).AddMethodDef(ilMethodDef) @@ -2418,7 +2414,7 @@ type AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbu // old implementation adds new element to the head of list so result was accumulated in reversed order let orderedReflectedDefinitions = [ - for KeyValue (vspec, (name, n, expr)) in reflectedDefinitions -> n, ((name, vspec), expr) + for (vspec, (n, (name, expr))) in reflectedDefinitions.GetAll() -> n, ((name, vspec), expr) ] |> List.sortBy (fst >> (~-)) // invert the result to get 'order-by-descending' behavior (items in list are 0..* so we don't need to worry about int.MinValue) |> List.map snd diff --git a/src/Compiler/Utilities/illib.fs b/src/Compiler/Utilities/illib.fs index e5df5449135..bd09f963c69 100644 --- a/src/Compiler/Utilities/illib.fs +++ b/src/Compiler/Utilities/illib.fs @@ -1067,6 +1067,27 @@ type MemoizationTable<'T, 'U>(compute: 'T -> 'U, keyComparer: IEqualityComparer< else compute x +type internal StampedDictionairy<'T, 'U>(keyComparer: IEqualityComparer<'T>) = + let table = new ConcurrentDictionary<'T, Lazy>(keyComparer) + let mutable count = -1 + + member _.Add(key, value) = + let entry = table.GetOrAdd(key, lazy (Interlocked.Increment(&count), value)) + entry.Force() |> ignore + + member _.Update(key, valueReplaceFunc) = + match table.TryGetValue key with + | true, v -> + let (stamp, oldVal) = v.Value + + match valueReplaceFunc oldVal with + | None -> () + | Some newVal -> table.TryUpdate(key, lazy (stamp, newVal), v) |> ignore + | _ -> () + + member _.GetAll() = + table |> Seq.map (fun kvp -> kvp.Key, kvp.Value.Value) + exception UndefinedException type LazyWithContextFailure(exn: exn) = diff --git a/src/Compiler/Utilities/illib.fsi b/src/Compiler/Utilities/illib.fsi index 40f8c8f8162..3ba295eed76 100644 --- a/src/Compiler/Utilities/illib.fsi +++ b/src/Compiler/Utilities/illib.fsi @@ -438,6 +438,16 @@ type internal MemoizationTable<'T, 'U> = member Apply: x: 'T -> 'U +type internal StampedDictionairy<'T, 'U> = + + new: keyComparer: IEqualityComparer<'T> -> StampedDictionairy<'T, 'U> + + member Add: key: 'T * value: 'U -> unit + + member Update: key: 'T * valueReplaceFunc: ('U -> 'U option) -> unit + + member GetAll: unit -> seq<'T * (int * 'U)> + exception internal UndefinedException type internal LazyWithContextFailure = From 41facc441372ed8137327540e2c0cff09fdffac1 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 25 Nov 2022 12:36:44 +0100 Subject: [PATCH 19/44] TypeDefsBuilder made concurrent --- src/Compiler/CodeGen/IlxGen.fs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index a15b62df563..e252a906044 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -238,6 +238,7 @@ type IlxGenIntraAssemblyInfo = /// only accessible intra-assembly. Across assemblies, taking the address of static mutable module-bound values is not permitted. /// The key to the table is the method ref for the property getter for the value, which is a stable name for the Val's /// that come from both the signature and the implementation. + (*TODO Tomas Make thread safe*) StaticFieldInfo: Dictionary } @@ -336,6 +337,7 @@ type cenv = intraAssemblyInfo: IlxGenIntraAssemblyInfo /// Cache methods with SecurityAttribute applied to them, to prevent unnecessary calls to ExistsInEntireHierarchyOfType + (*TODO Tomas Make thread safe*) casApplied: Dictionary /// Used to apply forced inlining optimizations to witnesses generated late during codegen @@ -1978,14 +1980,12 @@ type TypeDefBuilder(tdef: ILTypeDef, tdefDiscards) = gmethods.Add(mkILClassCtor body) and TypeDefsBuilder() = - (* TODO Tomas : This is not thread safe. Make it so - This builder acts for all types - can be accessed concurrently - The internal TypeDefBuilder's will be created from 1 file each, therefore those can remain for sequential usage. (??) - *) - let tdefs: HashMultiMap = - HashMultiMap(0, HashIdentity.Structural) + + let tdefs = + ConcurrentDictionary>(HashIdentity.Structural) let mutable countDown = System.Int32.MaxValue + let mutable countUp = -1 member b.Close() = //The order we emit type definitions is not deterministic since it is using the reverse of a range from a hash table. We should use an approximation of source order. @@ -1993,7 +1993,7 @@ and TypeDefsBuilder() = // However, for some tests FSI generated code appears sensitive to the order, especially for nested types. [ - for b, eliminateIfEmpty in HashRangeSorted tdefs do + for _, (b, eliminateIfEmpty) in tdefs.Values |> Seq.collect id |> Seq.sortBy fst do let tdef = b.Close() // Skip the type if it is empty if @@ -2010,7 +2010,7 @@ and TypeDefsBuilder() = (* TODO Tomas : This is not thread safe. Make it so*) member b.FindTypeDefBuilder nm = try - tdefs[nm] |> snd |> fst + tdefs[nm] |> List.head |> snd |> fst with :? KeyNotFoundException -> failwith ("FindTypeDefBuilder: " + nm + " not found") @@ -2025,9 +2025,12 @@ and TypeDefsBuilder() = if addAtEnd then Interlocked.Decrement(&countDown) else - tdefs.Count + Interlocked.Increment(&countUp) + + let newVal = idx, (TypeDefBuilder(tdef, tdefDiscards), eliminateIfEmpty) - tdefs.Add(tdef.Name, (idx, (TypeDefBuilder(tdef, tdefDiscards), eliminateIfEmpty))) + tdefs.AddOrUpdate(tdef.Name, [ newVal ], (fun key oldList -> newVal :: oldList)) + |> ignore type AnonTypeGenerationTable() = let dict = From 9425fa1257edc9f2b11f5f154d0176f6702ada32 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 25 Nov 2022 12:51:47 +0100 Subject: [PATCH 20/44] Making sure we emit IL methods for AnnonTypes in same order as before --- src/Compiler/CodeGen/IlxGen.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index e252a906044..35b920e1632 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -2006,8 +2006,7 @@ and TypeDefsBuilder() = then yield tdef ] - - (* TODO Tomas : This is not thread safe. Make it so*) + member b.FindTypeDefBuilder nm = try tdefs[nm] |> List.head |> snd |> fst @@ -2264,6 +2263,7 @@ type AnonTypeGenerationTable() = |> Array.map (fun (TBind (a, b, c)) -> // Disable method splitting for bindings related to anonymous records TBind(a, cenv.optimizeDuringCodeGen true b, c)) + |> Array.rev extraBindingsToGenerate.PushRange(optimizedExtraBindings) From de85e92a920a0dcd26655077c146af28c618efed Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 25 Nov 2022 13:30:07 +0100 Subject: [PATCH 21/44] format --- src/Compiler/Checking/AttributeChecking.fs | 1 + src/Compiler/CodeGen/IlxGen.fs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Compiler/Checking/AttributeChecking.fs b/src/Compiler/Checking/AttributeChecking.fs index de1aadfc803..60fe9b6d524 100644 --- a/src/Compiler/Checking/AttributeChecking.fs +++ b/src/Compiler/Checking/AttributeChecking.fs @@ -534,6 +534,7 @@ let IsSecurityAttribute (g: TcGlobals) amap (casmap : Dictionary) ( | true, c -> c | _ -> let exists = ExistsInEntireHierarchyOfType (fun t -> typeEquiv g t (mkAppTy attr.TyconRef [])) g amap m AllowMultiIntfInstantiations.Yes (mkAppTy tcref []) + // TODO TOMAS Concurrency casmap[tcs] <- exists exists | ValueNone -> false diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 35b920e1632..9c34c01b90a 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -2006,7 +2006,7 @@ and TypeDefsBuilder() = then yield tdef ] - + member b.FindTypeDefBuilder nm = try tdefs[nm] |> List.head |> snd |> fst From fa3bc30cb7d3d78c225af00b80388d6fdf21ab59 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 25 Nov 2022 15:23:20 +0100 Subject: [PATCH 22/44] Thread safe security attribute check caching and removing unused AsyncUtil --- src/Compiler/Checking/AttributeChecking.fs | 5 +- src/Compiler/Checking/AttributeChecking.fsi | 2 +- src/Compiler/CodeGen/IlxGen.fs | 22 ++--- src/Compiler/TypedTree/CompilerGlobalState.fs | 14 ++- src/Compiler/Utilities/lib.fs | 86 ------------------- src/Compiler/Utilities/lib.fsi | 19 ---- 6 files changed, 15 insertions(+), 133 deletions(-) diff --git a/src/Compiler/Checking/AttributeChecking.fs b/src/Compiler/Checking/AttributeChecking.fs index 60fe9b6d524..087ebc1e0b1 100644 --- a/src/Compiler/Checking/AttributeChecking.fs +++ b/src/Compiler/Checking/AttributeChecking.fs @@ -522,7 +522,7 @@ let CheckRecdFieldInfoAttributes g (x:RecdFieldInfo) m = CheckRecdFieldAttributes g x.RecdFieldRef m // Identify any security attributes -let IsSecurityAttribute (g: TcGlobals) amap (casmap : Dictionary) (Attrib(tcref, _, _, _, _, _, _)) m = +let IsSecurityAttribute (g: TcGlobals) amap (casmap : IDictionary) (Attrib(tcref, _, _, _, _, _, _)) m = // There's no CAS on Silverlight, so we have to be careful here match g.attrib_SecurityAttribute with | None -> false @@ -533,8 +533,7 @@ let IsSecurityAttribute (g: TcGlobals) amap (casmap : Dictionary) ( match casmap.TryGetValue tcs with | true, c -> c | _ -> - let exists = ExistsInEntireHierarchyOfType (fun t -> typeEquiv g t (mkAppTy attr.TyconRef [])) g amap m AllowMultiIntfInstantiations.Yes (mkAppTy tcref []) - // TODO TOMAS Concurrency + let exists = ExistsInEntireHierarchyOfType (fun t -> typeEquiv g t (mkAppTy attr.TyconRef [])) g amap m AllowMultiIntfInstantiations.Yes (mkAppTy tcref []) casmap[tcs] <- exists exists | ValueNone -> false diff --git a/src/Compiler/Checking/AttributeChecking.fsi b/src/Compiler/Checking/AttributeChecking.fsi index 622864eff4e..b4a608ef1d1 100644 --- a/src/Compiler/Checking/AttributeChecking.fsi +++ b/src/Compiler/Checking/AttributeChecking.fsi @@ -96,7 +96,7 @@ val CheckValAttributes: g: TcGlobals -> x: ValRef -> m: range -> OperationResult val CheckRecdFieldInfoAttributes: g: TcGlobals -> x: RecdFieldInfo -> m: range -> OperationResult val IsSecurityAttribute: - g: TcGlobals -> amap: Import.ImportMap -> casmap: Dictionary -> Attrib -> m: range -> bool + g: TcGlobals -> amap: Import.ImportMap -> casmap: IDictionary -> Attrib -> m: range -> bool val IsSecurityCriticalAttribute: g: TcGlobals -> Attrib -> bool diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 9c34c01b90a..d66014cb41b 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -238,8 +238,7 @@ type IlxGenIntraAssemblyInfo = /// only accessible intra-assembly. Across assemblies, taking the address of static mutable module-bound values is not permitted. /// The key to the table is the method ref for the property getter for the value, which is a stable name for the Val's /// that come from both the signature and the implementation. - (*TODO Tomas Make thread safe*) - StaticFieldInfo: Dictionary + StaticFieldInfo: ConcurrentDictionary } /// Helper to make sure we take tailcalls in some situations @@ -337,8 +336,7 @@ type cenv = intraAssemblyInfo: IlxGenIntraAssemblyInfo /// Cache methods with SecurityAttribute applied to them, to prevent unnecessary calls to ExistsInEntireHierarchyOfType - (*TODO Tomas Make thread safe*) - casApplied: Dictionary + casApplied: IDictionary /// Used to apply forced inlining optimizations to witnesses generated late during codegen mutable optimizeDuringCodeGen: bool -> Expr -> Expr @@ -1512,13 +1510,7 @@ let ComputeFieldSpecForVal match optIntraAssemblyInfo with | None -> generate () - | Some intraAssemblyInfo -> - match intraAssemblyInfo.StaticFieldInfo.TryGetValue ilGetterMethRef with - | true, res -> res - | _ -> - let res = generate () - intraAssemblyInfo.StaticFieldInfo[ ilGetterMethRef ] <- res - res + | Some iai -> iai.StaticFieldInfo.GetOrAdd(ilGetterMethRef, fun _ -> generate()) /// Compute the representation information for an F#-declared value (not a member nor a function). /// Mutable and literal static fields must have stable names and live in the "public" location @@ -11921,14 +11913,12 @@ type IlxAssemblyGenerator(amap: ImportMap, tcGlobals: TcGlobals, tcVal: Constrai // The incremental state held by the ILX code generator let mutable ilxGenEnv = GetEmptyIlxGenEnv tcGlobals ccu let anonTypeTable = AnonTypeGenerationTable() - // Dictionaries are safe here as they will only be used during the codegen stage - will happen on a single thread. + let intraAssemblyInfo = { - StaticFieldInfo = Dictionary<_, _>(HashIdentity.Structural) + StaticFieldInfo = ConcurrentDictionary<_, _>(HashIdentity.Structural) } - let casApplied = Dictionary() - let cenv = { g = tcGlobals @@ -11944,7 +11934,7 @@ type IlxAssemblyGenerator(amap: ImportMap, tcGlobals: TcGlobals, tcVal: Constrai ilUnitTy = None namedDebugPointsForInlinedCode = Map.empty amap = amap - casApplied = casApplied + casApplied = ConcurrentDictionary() intraAssemblyInfo = intraAssemblyInfo optionsOpt = None optimizeDuringCodeGen = (fun _flag expr -> expr) diff --git a/src/Compiler/TypedTree/CompilerGlobalState.fs b/src/Compiler/TypedTree/CompilerGlobalState.fs index b7eea4fb718..7a17363f08e 100644 --- a/src/Compiler/TypedTree/CompilerGlobalState.fs +++ b/src/Compiler/TypedTree/CompilerGlobalState.fs @@ -16,7 +16,7 @@ open FSharp.Compiler.Text /// policy to make all globally-allocated objects concurrency safe in case future versions of the compiler /// are used to host multiple concurrent instances of compilation. type NiceNameGenerator() = - + (* TODO Tomas lockfree *) let lockObj = obj() let basicNameCounts = Dictionary(100) @@ -43,7 +43,7 @@ type NiceNameGenerator() = /// This type may be accessed concurrently, though in practice it is only used from the compilation thread. /// It is made concurrency-safe since a global instance of the type is allocated in tast.fs. type StableNiceNameGenerator() = - + (* TODO Tomas lockfree *) let lockObj = obj() let names = Dictionary(100) @@ -92,12 +92,10 @@ type internal CompilerGlobalState () = type Unique = int64 //++GLOBAL MUTABLE STATE (concurrency-safe) -let newUnique = - let i = ref 0L - fun () -> System.Threading.Interlocked.Increment i +let mutable private uniqueCount = 0L +let newUnique() = System.Threading.Interlocked.Increment &uniqueCount /// Unique name generator for stamps attached to to val_specs, tycon_specs etc. //++GLOBAL MUTABLE STATE (concurrency-safe) -let newStamp = - let i = ref 0L - fun () -> System.Threading.Interlocked.Increment i +let mutable private stampCount = 0L +let newStamp() = System.Threading.Interlocked.Increment &stampCount diff --git a/src/Compiler/Utilities/lib.fs b/src/Compiler/Utilities/lib.fs index 95a20226a2d..108237fe8e9 100755 --- a/src/Compiler/Utilities/lib.fs +++ b/src/Compiler/Utilities/lib.fs @@ -420,92 +420,6 @@ type Dumper(x:obj) = [] member self.Dump = sprintf "%A" x #endif - -//--------------------------------------------------------------------------- -// AsyncUtil -//--------------------------------------------------------------------------- - -module internal AsyncUtil = - open Microsoft.FSharp.Control - - /// Represents the reified result of an asynchronous computation. - [] - type AsyncResult<'T> = - | AsyncOk of 'T - | AsyncException of exn - | AsyncCanceled of OperationCanceledException - - static member Commit(res:AsyncResult<'T>) = - Async.FromContinuations (fun (cont, eCont, cCont) -> - match res with - | AsyncOk v -> cont v - | AsyncException exn -> eCont exn - | AsyncCanceled exn -> cCont exn) - - /// When using .NET 4.0 you can replace this type by - [] - type AsyncResultCell<'T>() = - let mutable result = None - // The continuation for the result, if any - let mutable savedConts = [] - - let syncRoot = obj() - - - // Record the result in the AsyncResultCell. - // Ignore subsequent sets of the result. This can happen, e.g. for a race between - // a cancellation and a success. - member x.RegisterResult (res:AsyncResult<'T>) = - let grabbedConts = - lock syncRoot (fun () -> - if result.IsSome then - [] - else - result <- Some res - // Invoke continuations in FIFO order - // Continuations that Async.FromContinuations provide do QUWI/SyncContext.Post, - // so the order is not overly relevant but still. - List.rev savedConts) - let postOrQueue (sc:SynchronizationContext, cont) = - match sc with - | null -> ThreadPool.QueueUserWorkItem(fun _ -> cont res) |> ignore - | sc -> sc.Post((fun _ -> cont res), state=null) - - // Run continuations outside the lock - match grabbedConts with - | [] -> () - | [sc, cont as c] -> - if SynchronizationContext.Current = sc then - cont res - else - postOrQueue c - | _ -> - grabbedConts |> List.iter postOrQueue - - /// Get the reified result. - member private x.AsyncPrimitiveResult = - Async.FromContinuations(fun (cont, _, _) -> - let grabbedResult = - lock syncRoot (fun () -> - match result with - | Some _ -> - result - | None -> - // Otherwise save the continuation and call it in RegisterResult - let sc = SynchronizationContext.Current - savedConts <- (sc, cont) :: savedConts - None) - // Run the action outside the lock - match grabbedResult with - | None -> () - | Some res -> cont res) - - - /// Get the result and Commit(...). - member x.AsyncResult = - async { let! res = x.AsyncPrimitiveResult - return! AsyncResult.Commit(res) } - //--------------------------------------------------------------------------- // EnableHeapTerminationOnCorruption() //--------------------------------------------------------------------------- diff --git a/src/Compiler/Utilities/lib.fsi b/src/Compiler/Utilities/lib.fsi index bab85ccd414..afb813d6b4f 100644 --- a/src/Compiler/Utilities/lib.fsi +++ b/src/Compiler/Utilities/lib.fsi @@ -267,25 +267,6 @@ val inline cacheOptRef: cache: 'a option ref -> f: (unit -> 'a) -> 'a val inline tryGetCacheValue: cache: cache<'a> -> NonNullSlot<'a> voption -module AsyncUtil = - - /// Represents the reified result of an asynchronous computation. - [] - type AsyncResult<'T> = - | AsyncOk of 'T - | AsyncException of exn - | AsyncCanceled of System.OperationCanceledException - - static member Commit: res: AsyncResult<'T> -> Async<'T> - - /// When using .NET 4.0 you can replace this type by - [] - type AsyncResultCell<'T> = - - new: unit -> AsyncResultCell<'T> - member RegisterResult: res: AsyncResult<'T> -> unit - member AsyncResult: Async<'T> - module UnmanagedProcessExecutionOptions = val EnableHeapTerminationOnCorruption: unit -> unit From 3fda4703cb3e835ec0a860e3f6f821e6f972987b Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 25 Nov 2022 16:14:53 +0100 Subject: [PATCH 23/44] cleanups and unused deletions --- src/Compiler/Checking/QuotationTranslator.fs | 2 +- src/Compiler/CodeGen/IlxGen.fs | 6 ++-- src/Compiler/Driver/StaticLinking.fs | 2 +- src/Compiler/Interactive/fsi.fs | 2 +- src/Compiler/Utilities/lib.fs | 37 +------------------- src/Compiler/Utilities/lib.fsi | 22 +----------- 6 files changed, 8 insertions(+), 63 deletions(-) diff --git a/src/Compiler/Checking/QuotationTranslator.fs b/src/Compiler/Checking/QuotationTranslator.fs index 80c7e52e7bf..063609d0232 100644 --- a/src/Compiler/Checking/QuotationTranslator.fs +++ b/src/Compiler/Checking/QuotationTranslator.fs @@ -22,7 +22,7 @@ open System.Collections.Generic module QP = QuotationPickler -let verboseCReflect = condition "VERBOSE_CREFLECT" +let verboseCReflect = isEnvVarSet "VERBOSE_CREFLECT" [] type IsReflectedDefinition = diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index d66014cb41b..b997fe69440 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -1510,7 +1510,7 @@ let ComputeFieldSpecForVal match optIntraAssemblyInfo with | None -> generate () - | Some iai -> iai.StaticFieldInfo.GetOrAdd(ilGetterMethRef, fun _ -> generate()) + | Some iai -> iai.StaticFieldInfo.GetOrAdd(ilGetterMethRef, (fun _ -> generate ())) /// Compute the representation information for an F#-declared value (not a member nor a function). /// Mutable and literal static fields must have stable names and live in the "public" location @@ -2383,8 +2383,8 @@ and AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbuf let instrs = [ yield! - (if condition "NO_ADD_FEEFEE_TO_CCTORS" then [] - elif condition "ADD_SEQPT_TO_CCTORS" then seqpt + (if isEnvVarSet "NO_ADD_FEEFEE_TO_CCTORS" then [] + elif isEnvVarSet "ADD_SEQPT_TO_CCTORS" then seqpt else feefee) // mark start of hidden code yield mkLdcInt32 0 yield mkNormalStsfld fspec diff --git a/src/Compiler/Driver/StaticLinking.fs b/src/Compiler/Driver/StaticLinking.fs index 5ad9ff15f40..0c4016b31e3 100644 --- a/src/Compiler/Driver/StaticLinking.fs +++ b/src/Compiler/Driver/StaticLinking.fs @@ -98,7 +98,7 @@ type TypeForwarding(tcImports: TcImports) = member _.TypeForwardILTypeRef tref = typeForwardILTypeRef tref -let debugStaticLinking = condition "FSHARP_DEBUG_STATIC_LINKING" +let debugStaticLinking = isEnvVarSet "FSHARP_DEBUG_STATIC_LINKING" let StaticLinkILModules ( diff --git a/src/Compiler/Interactive/fsi.fs b/src/Compiler/Interactive/fsi.fs index 6a1c4b77c04..078a3d3a8ba 100644 --- a/src/Compiler/Interactive/fsi.fs +++ b/src/Compiler/Interactive/fsi.fs @@ -3722,7 +3722,7 @@ type FsiEvaluationSession (fsi: FsiEvaluationSessionHostConfig, argv:string[], i /// A background thread is started by this thread to read from the inReader and/or console reader. member x.Run() = - progress <- condition "FSHARP_INTERACTIVE_PROGRESS" + progress <- isEnvVarSet "FSHARP_INTERACTIVE_PROGRESS" // Explanation: When Run is called we do a bunch of processing. For fsi.exe // and fsiAnyCpu.exe there are no other active threads at this point, so we can assume this is the diff --git a/src/Compiler/Utilities/lib.fs b/src/Compiler/Utilities/lib.fs index 108237fe8e9..bc03a8de2ab 100755 --- a/src/Compiler/Utilities/lib.fs +++ b/src/Compiler/Utilities/lib.fs @@ -21,7 +21,7 @@ let mutable progress = false // Intended to be a general hook to control diagnostic output when tracking down bugs let mutable tracking = false -let condition s = +let isEnvVarSet s = try (Environment.GetEnvironmentVariable(s) <> null) with _ -> false let GetEnvInteger e dflt = match Environment.GetEnvironmentVariable(e) with null -> dflt | t -> try int t with _ -> dflt @@ -73,37 +73,6 @@ module NameMap = let domain m = Map.foldBack (fun x _ acc -> Zset.add x acc) m (Zset.empty String.order) let domainL m = Zset.elements (domain m) -// Library: Pre\Post checks -//------------------------------------------------------------------------- -module Check = - - /// Throw System.InvalidOperationException if argument is None. - /// If there is a value (e.g. Some(value)) then value is returned. - let NotNone argName (arg:'T option) : 'T = - match arg with - | None -> raise (InvalidOperationException(argName)) - | Some x -> x - - /// Throw System.ArgumentNullException if argument is null. - let ArgumentNotNull arg argName = - match box(arg) with - | null -> raise (ArgumentNullException(argName)) - | _ -> () - - /// Throw System.ArgumentNullException if array argument is null. - /// Throw System.ArgumentOutOfRangeException is array argument is empty. - let ArrayArgumentNotNullOrEmpty (arr:'T[]) argName = - ArgumentNotNull arr argName - if (0 = arr.Length) then - raise (ArgumentOutOfRangeException(argName)) - - /// Throw System.ArgumentNullException if string argument is null. - /// Throw System.ArgumentOutOfRangeException is string argument is empty. - let StringArgumentNotNullOrEmpty (s:string) argName = - ArgumentNotNull s argName - if s.Length = 0 then - raise (ArgumentNullException(argName)) - //------------------------------------------------------------------------- // Library //------------------------------------------------------------------------ @@ -285,8 +254,6 @@ let mapTriple (f1, f2, f3) (a1, a2, a3) = (f1 a1, f2 a2, f3 a3) let mapQuadruple (f1, f2, f3, f4) (a1, a2, a3, a4) = (f1 a1, f2 a2, f3 a3, f4 a4) -let fmap2Of2 f z (a1, a2) = let z, a2 = f z a2 in z, (a1, a2) - //--------------------------------------------------------------------------- // Zmap rebinds //------------------------------------------------------------------------- @@ -312,8 +279,6 @@ module Zset = if Zset.equal s s0 then s0 (* fixed *) else fixpoint f s (* iterate *) -let equalOn f x y = (f x) = (f y) - /// Buffer printing utility let buildString f = let buf = StringBuilder 100 diff --git a/src/Compiler/Utilities/lib.fsi b/src/Compiler/Utilities/lib.fsi index afb813d6b4f..2e6b0318f7a 100644 --- a/src/Compiler/Utilities/lib.fsi +++ b/src/Compiler/Utilities/lib.fsi @@ -15,7 +15,7 @@ val mutable progress: bool val mutable tracking: bool -val condition: s: string -> bool +val isEnvVarSet: s: string -> bool val GetEnvInteger: e: string -> dflt: int -> int @@ -64,22 +64,6 @@ module NameMap = val domainL: m: Map -> string list -module Check = - /// Throw System.InvalidOperationException if argument is None. - /// If there is a value (e.g. Some(value)) then value is returned. - val NotNone: argName: string -> arg: 'T option -> 'T - - /// Throw System.ArgumentNullException if argument is null. - val ArgumentNotNull: arg: 'a -> argName: string -> unit - - /// Throw System.ArgumentNullException if array argument is null. - /// Throw System.ArgumentOutOfRangeException is array argument is empty. - val ArrayArgumentNotNullOrEmpty: arr: 'T[] -> argName: string -> unit - - /// Throw System.ArgumentNullException if string argument is null. - /// Throw System.ArgumentOutOfRangeException is string argument is empty. - val StringArgumentNotNullOrEmpty: s: string -> argName: string -> unit - type IntMap<'T> = Zmap module IntMap = @@ -214,8 +198,6 @@ val mapQuadruple: a1: 'a * a2: 'c * a3: 'e * a4: 'g -> 'b * 'd * 'f * 'h -val fmap2Of2: f: ('a -> 'b -> 'c * 'd) -> z: 'a -> a1: 'e * a2: 'b -> 'c * ('e * 'd) - module Zmap = val force: k: 'a -> mp: Zmap<'a, 'b> -> 'b val mapKey: key: 'a -> f: ('b option -> 'b option) -> mp: Zmap<'a, 'b> -> Zmap<'a, 'b> @@ -224,8 +206,6 @@ module Zset = val ofList: order: IComparer<'a> -> xs: 'a list -> Zset<'a> val fixpoint: f: (Zset<'a> -> Zset<'a>) -> Zset<'a> -> Zset<'a> -val equalOn: f: ('a -> 'b) -> x: 'a -> y: 'a -> bool when 'b: equality - /// Buffer printing utility val buildString: f: (StringBuilder -> unit) -> string From ea32830eb14df0a8265af71666c64d57405cdbf4 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 25 Nov 2022 16:37:41 +0100 Subject: [PATCH 24/44] NiceNameGenerator without explicit locks --- src/Compiler/TypedTree/CompilerGlobalState.fs | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/Compiler/TypedTree/CompilerGlobalState.fs b/src/Compiler/TypedTree/CompilerGlobalState.fs index 7a17363f08e..35fc769ff99 100644 --- a/src/Compiler/TypedTree/CompilerGlobalState.fs +++ b/src/Compiler/TypedTree/CompilerGlobalState.fs @@ -4,7 +4,9 @@ module FSharp.Compiler.CompilerGlobalState +open System open System.Collections.Generic +open System.Collections.Concurrent open FSharp.Compiler.Syntax.PrettyNaming open FSharp.Compiler.Text @@ -16,25 +18,17 @@ open FSharp.Compiler.Text /// policy to make all globally-allocated objects concurrency safe in case future versions of the compiler /// are used to host multiple concurrent instances of compilation. type NiceNameGenerator() = - (* TODO Tomas lockfree *) - let lockObj = obj() - let basicNameCounts = Dictionary(100) + + let basicNameCounts = ConcurrentDictionary(max Environment.ProcessorCount 1, 127) + let incrementCounter = Func(fun _ oldVal -> oldVal + 1) member _.FreshCompilerGeneratedName (name, m: range) = - lock lockObj (fun () -> let basicName = GetBasicNameOfPossibleCompilerGeneratedName name - let n = - match basicNameCounts.TryGetValue basicName with - | true, count -> count - | _ -> 0 - let nm = CompilerGeneratedNameSuffix basicName (string m.StartLine + (match n with 0 -> "" | n -> "-" + string n)) - basicNameCounts[basicName] <- n + 1 - nm) + let count = basicNameCounts.AddOrUpdate(basicName, 1, incrementCounter) + CompilerGeneratedNameSuffix basicName (string m.StartLine + (match (count-1) with 0 -> "" | n -> "-" + string n)) member _.Reset () = - lock lockObj (fun () -> - basicNameCounts.Clear() - ) + basicNameCounts.Clear() /// Generates compiler-generated names marked up with a source code location, but if given the same unique value then /// return precisely the same name. Each name generated also includes the StartLine number of the range passed in From 6493ede3e4664ed79efcb95b3ba1e7e379781e16 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Mon, 28 Nov 2022 12:14:33 +0100 Subject: [PATCH 25/44] Lock prevention in name generation --- src/Compiler/TypedTree/CompilerGlobalState.fs | 46 +++++-------------- .../TypedTree/CompilerGlobalState.fsi | 2 - 2 files changed, 12 insertions(+), 36 deletions(-) diff --git a/src/Compiler/TypedTree/CompilerGlobalState.fs b/src/Compiler/TypedTree/CompilerGlobalState.fs index 35fc769ff99..7047fc3cf35 100644 --- a/src/Compiler/TypedTree/CompilerGlobalState.fs +++ b/src/Compiler/TypedTree/CompilerGlobalState.fs @@ -7,6 +7,7 @@ module FSharp.Compiler.CompilerGlobalState open System open System.Collections.Generic open System.Collections.Concurrent +open System.Threading open FSharp.Compiler.Syntax.PrettyNaming open FSharp.Compiler.Text @@ -17,54 +18,31 @@ open FSharp.Compiler.Text /// It is made concurrency-safe since a global instance of the type is allocated in tast.fs, and it is good /// policy to make all globally-allocated objects concurrency safe in case future versions of the compiler /// are used to host multiple concurrent instances of compilation. -type NiceNameGenerator() = +type NiceNameGenerator() = + let basicNameCounts = ConcurrentDictionary>(max Environment.ProcessorCount 1, 127) - let basicNameCounts = ConcurrentDictionary(max Environment.ProcessorCount 1, 127) - let incrementCounter = Func(fun _ oldVal -> oldVal + 1) - member _.FreshCompilerGeneratedName (name, m: range) = let basicName = GetBasicNameOfPossibleCompilerGeneratedName name - let count = basicNameCounts.AddOrUpdate(basicName, 1, incrementCounter) + let countCell = basicNameCounts.GetOrAdd(basicName,fun k -> ref 0) + let count = Interlocked.Increment(countCell) + CompilerGeneratedNameSuffix basicName (string m.StartLine + (match (count-1) with 0 -> "" | n -> "-" + string n)) - member _.Reset () = - basicNameCounts.Clear() - /// Generates compiler-generated names marked up with a source code location, but if given the same unique value then /// return precisely the same name. Each name generated also includes the StartLine number of the range passed in /// at the point of first generation. /// /// This type may be accessed concurrently, though in practice it is only used from the compilation thread. /// It is made concurrency-safe since a global instance of the type is allocated in tast.fs. -type StableNiceNameGenerator() = - (* TODO Tomas lockfree *) - let lockObj = obj() +type StableNiceNameGenerator() = - let names = Dictionary(100) - let basicNameCounts = Dictionary(100) + let niceNames = ConcurrentDictionary(max Environment.ProcessorCount 1, 127) + let innerGenerator = new NiceNameGenerator() member x.GetUniqueCompilerGeneratedName (name, m: range, uniq) = - lock lockObj (fun () -> - let basicName = GetBasicNameOfPossibleCompilerGeneratedName name - let key = basicName, uniq - match names.TryGetValue key with - | true, nm -> nm - | _ -> - let n = - match basicNameCounts.TryGetValue basicName with - | true, c -> c - | _ -> 0 - let nm = CompilerGeneratedNameSuffix basicName (string m.StartLine + (match n with 0 -> "" | n -> "-" + string n)) - names[key] <- nm - basicNameCounts[basicName] <- n + 1 - nm - ) - - member x.Reset () = - lock lockObj (fun () -> - basicNameCounts.Clear() - names.Clear() - ) + let basicName = GetBasicNameOfPossibleCompilerGeneratedName name + let key = basicName, uniq + niceNames.GetOrAdd(key, fun _ -> innerGenerator.FreshCompilerGeneratedName(name, m)) type internal CompilerGlobalState () = /// A global generator of compiler generated names diff --git a/src/Compiler/TypedTree/CompilerGlobalState.fsi b/src/Compiler/TypedTree/CompilerGlobalState.fsi index 105ab236f9e..6f0dba79ddf 100644 --- a/src/Compiler/TypedTree/CompilerGlobalState.fsi +++ b/src/Compiler/TypedTree/CompilerGlobalState.fsi @@ -17,7 +17,6 @@ type NiceNameGenerator = new: unit -> NiceNameGenerator member FreshCompilerGeneratedName: name: string * m: range -> string - member Reset: unit -> unit /// Generates compiler-generated names marked up with a source code location, but if given the same unique value then /// return precisely the same name. Each name generated also includes the StartLine number of the range passed in @@ -29,7 +28,6 @@ type StableNiceNameGenerator = new: unit -> StableNiceNameGenerator member GetUniqueCompilerGeneratedName: name: string * m: range * uniq: int64 -> string - member Reset: unit -> unit type internal CompilerGlobalState = From 57c149328782eb4e9e17dc8e96c355381abe5c0f Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Mon, 28 Nov 2022 13:02:52 +0100 Subject: [PATCH 26/44] Simplify queue of delayed code --- src/Compiler/CodeGen/IlxGen.fs | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index b997fe69440..cc62e6af83a 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -342,7 +342,7 @@ type cenv = mutable optimizeDuringCodeGen: bool -> Expr -> Expr /// Delayed Method Generation - which can later be parallelized across multiple files - delayedGenMethods: Queue unit> + delayedGenMethods: Queue unit> /// Guard the stack and move to a new one if necessary mutable stackGuard: StackGuard @@ -1245,7 +1245,7 @@ and IlxGenEnv = delayCodeGen: bool /// Collection of code-gen functions where each function represents a file. - delayedFileGenReverse: list<(cenv -> unit)[]> + delayedFileGenReverse: list<(unit -> unit)[]> } override _.ToString() = "" @@ -3128,27 +3128,17 @@ and CodeGenMethodForExpr cenv mgbuf (entryPointInfo, methodName, eenv, alreadyUs code and DelayCodeGenMethodForExpr cenv mgbuf ((_, _, eenv, _, _, _, _) as args) = - - let ilLazyCode = - lazy - CodeGenMethodForExpr - { cenv with - stackGuard = - if eenv.delayCodeGen then - getEmptyStackGuard () - else - cenv.stackGuard - delayedGenMethods = Queue() - } - mgbuf - args - if eenv.delayCodeGen then - cenv.delayedGenMethods.Enqueue(fun _ -> ilLazyCode.Force() |> ignore) - else - ilLazyCode.Force() |> ignore + let cenv = + { cenv with + stackGuard = getEmptyStackGuard () + } - ilLazyCode + let lazyMethodBody = lazy (CodeGenMethodForExpr cenv mgbuf args) + cenv.delayedGenMethods.Enqueue(fun () -> lazyMethodBody.Force() |> ignore) + lazyMethodBody + else + notlazy (CodeGenMethodForExpr cenv mgbuf args) //-------------------------------------------------------------------------- // Generate sequels @@ -11620,7 +11610,7 @@ let CodegenAssembly cenv eenv mgbuf implFiles = eenv.delayedFileGenReverse |> Array.ofList |> Array.rev - |> ArrayParallel.iter (fun genMeths -> genMeths |> Array.iter (fun gen -> gen cenv)) + |> ArrayParallel.iter (fun genMeths -> genMeths |> Array.iter (fun gen -> gen ())) // Some constructs generate residue types and bindings. Generate these now. They don't result in any // top-level initialization code. From fe4cbb06a4b39f280011318c6b820fbe9a609969 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 29 Nov 2022 15:53:08 +0100 Subject: [PATCH 27/44] Code review - comments added --- src/Compiler/AbstractIL/il.fs | 4 ++-- src/Compiler/AbstractIL/ilmorph.fs | 2 +- src/Compiler/Checking/MethodCalls.fs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Compiler/AbstractIL/il.fs b/src/Compiler/AbstractIL/il.fs index e5c1861262c..19896896475 100644 --- a/src/Compiler/AbstractIL/il.fs +++ b/src/Compiler/AbstractIL/il.fs @@ -3902,7 +3902,7 @@ let prependInstrsToCode (instrs: ILInstr list) (c2: ILCode) = // If there is a sequence point as the first instruction then keep it at the front | I_seqpoint _ as i0 -> let labels = - let dict = Dictionary.newWithSize (c2.Labels.Count * 2) + let dict = Dictionary.newWithSize (c2.Labels.Count * 2) // Decrease chance of collisions by oversizing the hashtable for kvp in c2.Labels do dict.Add(kvp.Key, (if kvp.Value = 0 then 0 else kvp.Value + n)) @@ -3915,7 +3915,7 @@ let prependInstrsToCode (instrs: ILInstr list) (c2: ILCode) = } | _ -> let labels = - let dict = Dictionary.newWithSize (c2.Labels.Count * 2) + let dict = Dictionary.newWithSize (c2.Labels.Count * 2) // Decrease chance of collisions by oversizing the hashtable for kvp in c2.Labels do dict.Add(kvp.Key, kvp.Value + n) diff --git a/src/Compiler/AbstractIL/ilmorph.fs b/src/Compiler/AbstractIL/ilmorph.fs index 47bb985db36..2ab4171781a 100644 --- a/src/Compiler/AbstractIL/ilmorph.fs +++ b/src/Compiler/AbstractIL/ilmorph.fs @@ -37,7 +37,7 @@ let code_instr2instrs f (code: ILCode) = adjust[old] <- nw let labels = - let dict = Dictionary.newWithSize (code.Labels.Count * 2) + let dict = Dictionary.newWithSize (code.Labels.Count * 2) // Decrease chance of collisions by oversizing the hashtable for kvp in code.Labels do dict.Add(kvp.Key, adjust[kvp.Value]) diff --git a/src/Compiler/Checking/MethodCalls.fs b/src/Compiler/Checking/MethodCalls.fs index 1175cc3dded..870443d32b9 100644 --- a/src/Compiler/Checking/MethodCalls.fs +++ b/src/Compiler/Checking/MethodCalls.fs @@ -1806,8 +1806,8 @@ module ProvidedMethodCalls = expr: Tainted) = let varConv = - // note: using paramVars.Length as assumed initial size, but this might not - // be the optimal value; this wasn't checked before obsoleting Dictionary.ofList + // note: Assuming the size based on paramVars + // Doubling to decrease chance of collisions let dict = Dictionary.newWithSize (paramVars.Length*2) for v, e in Seq.zip (paramVars |> Seq.map (fun x -> x.PUntaint(id, m))) (Option.toList thisArg @ allArgs) do dict.Add(v, (None, e)) From 827055b1c90ffec00cdc0a1b78406ffb46510739 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 29 Nov 2022 15:58:14 +0100 Subject: [PATCH 28/44] Code review comments added Co-authored-by: Petr --- src/Compiler/CodeGen/IlxGen.fsi | 2 +- src/Compiler/Utilities/illib.fs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Compiler/CodeGen/IlxGen.fsi b/src/Compiler/CodeGen/IlxGen.fsi index 41988542d91..4658dd0693b 100644 --- a/src/Compiler/CodeGen/IlxGen.fsi +++ b/src/Compiler/CodeGen/IlxGen.fsi @@ -60,7 +60,7 @@ type internal IlxGenOptions = /// Indicates that, whenever possible, use callvirt instead of call alwaysCallVirt: bool - /// When set to true, the IlxGen will delay generation of method bodies and generated them later in parallel (parallelized across files) + /// When set to true, the IlxGen will delay generation of method bodies and generate them later in parallel (parallelized across files) parallelIlxGenEnabled: bool } diff --git a/src/Compiler/Utilities/illib.fs b/src/Compiler/Utilities/illib.fs index bd09f963c69..37f03b85813 100644 --- a/src/Compiler/Utilities/illib.fs +++ b/src/Compiler/Utilities/illib.fs @@ -1067,6 +1067,7 @@ type MemoizationTable<'T, 'U>(compute: 'T -> 'U, keyComparer: IEqualityComparer< else compute x +/// A thread-safe lookup table which is assigning an auto-increment stamp with each insert type internal StampedDictionairy<'T, 'U>(keyComparer: IEqualityComparer<'T>) = let table = new ConcurrentDictionary<'T, Lazy>(keyComparer) let mutable count = -1 From f538ad72c3eec0a63f8fef81a2e6eb39324c936e Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 29 Nov 2022 16:05:19 +0100 Subject: [PATCH 29/44] StampedDictionary.Update -> UpdateIfExists --- src/Compiler/CodeGen/IlxGen.fs | 2 +- src/Compiler/Utilities/illib.fs | 2 +- src/Compiler/Utilities/illib.fsi | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index cc62e6af83a..029b6ac93ba 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -2369,7 +2369,7 @@ and AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbuf reflectedDefinitions.Add(vspec, (vspec.CompiledName cenv.g.CompilerGlobalState, expr)) member _.ReplaceNameOfReflectedDefinition(vspec, newName) = - reflectedDefinitions.Update(vspec, (fun (oldName, expr) -> if newName = oldName then None else Some(newName, expr))) + reflectedDefinitions.UpdateIfExists(vspec, (fun (oldName, expr) -> if newName = oldName then None else Some(newName, expr))) member _.AddMethodDef(tref: ILTypeRef, ilMethodDef) = gtdefs.FindNestedTypeDefBuilder(tref).AddMethodDef(ilMethodDef) diff --git a/src/Compiler/Utilities/illib.fs b/src/Compiler/Utilities/illib.fs index 37f03b85813..36b452fd731 100644 --- a/src/Compiler/Utilities/illib.fs +++ b/src/Compiler/Utilities/illib.fs @@ -1076,7 +1076,7 @@ type internal StampedDictionairy<'T, 'U>(keyComparer: IEqualityComparer<'T>) = let entry = table.GetOrAdd(key, lazy (Interlocked.Increment(&count), value)) entry.Force() |> ignore - member _.Update(key, valueReplaceFunc) = + member _.UpdateIfExists(key, valueReplaceFunc) = match table.TryGetValue key with | true, v -> let (stamp, oldVal) = v.Value diff --git a/src/Compiler/Utilities/illib.fsi b/src/Compiler/Utilities/illib.fsi index 3ba295eed76..306ccd80572 100644 --- a/src/Compiler/Utilities/illib.fsi +++ b/src/Compiler/Utilities/illib.fsi @@ -444,7 +444,7 @@ type internal StampedDictionairy<'T, 'U> = member Add: key: 'T * value: 'U -> unit - member Update: key: 'T * valueReplaceFunc: ('U -> 'U option) -> unit + member UpdateIfExists: key: 'T * valueReplaceFunc: ('U -> 'U option) -> unit member GetAll: unit -> seq<'T * (int * 'U)> From 2053441698fc946db7921bd655f003ed5d85ca6b Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 29 Nov 2022 16:10:53 +0100 Subject: [PATCH 30/44] Naming improvements Co-authored-by: Petr --- src/Compiler/Utilities/illib.fs | 6 +++--- src/Compiler/Utilities/illib.fsi | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Compiler/Utilities/illib.fs b/src/Compiler/Utilities/illib.fs index 36b452fd731..a8c092578c5 100644 --- a/src/Compiler/Utilities/illib.fs +++ b/src/Compiler/Utilities/illib.fs @@ -1041,15 +1041,15 @@ module CancellableAutoOpens = /// Generates unique stamps type UniqueStampGenerator<'T when 'T: equality>() = - let encodeTab = ConcurrentDictionary<'T, Lazy>(HashIdentity.Structural) + let encodeTable = ConcurrentDictionary<'T, Lazy>(HashIdentity.Structural) let mutable nItems = -1 let computeFunc = Func<'T, _>(fun _ -> lazy (Interlocked.Increment(&nItems))) member _.Encode str = - encodeTab.GetOrAdd(str, computeFunc).Value + encodeTable .GetOrAdd(str, computeFunc).Value - member _.Table = encodeTab.Keys + member _.Table = encodeTable .Keys /// memoize tables (all entries cached, never collected) type MemoizationTable<'T, 'U>(compute: 'T -> 'U, keyComparer: IEqualityComparer<'T>, ?canMemoize) = diff --git a/src/Compiler/Utilities/illib.fsi b/src/Compiler/Utilities/illib.fsi index 306ccd80572..8ddaf3cd45d 100644 --- a/src/Compiler/Utilities/illib.fsi +++ b/src/Compiler/Utilities/illib.fsi @@ -438,6 +438,7 @@ type internal MemoizationTable<'T, 'U> = member Apply: x: 'T -> 'U +/// A thread-safe lookup table which is assigning an auto-increment stamp with each insert type internal StampedDictionairy<'T, 'U> = new: keyComparer: IEqualityComparer<'T> -> StampedDictionairy<'T, 'U> From e8ef31713a02aaf820be39787dc1ea3d2fa36f70 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 29 Nov 2022 16:14:04 +0100 Subject: [PATCH 31/44] Apply suggestions from code review --- src/Compiler/Utilities/illib.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Compiler/Utilities/illib.fs b/src/Compiler/Utilities/illib.fs index a8c092578c5..365595bc441 100644 --- a/src/Compiler/Utilities/illib.fs +++ b/src/Compiler/Utilities/illib.fs @@ -1047,9 +1047,9 @@ type UniqueStampGenerator<'T when 'T: equality>() = let computeFunc = Func<'T, _>(fun _ -> lazy (Interlocked.Increment(&nItems))) member _.Encode str = - encodeTable .GetOrAdd(str, computeFunc).Value + encodeTable.GetOrAdd(str, computeFunc).Value - member _.Table = encodeTable .Keys + member _.Table = encodeTable.Keys /// memoize tables (all entries cached, never collected) type MemoizationTable<'T, 'U>(compute: 'T -> 'U, keyComparer: IEqualityComparer<'T>, ?canMemoize) = From c50a3d3244dce6639da2c8816db0f205f1bb0656 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 29 Nov 2022 16:21:54 +0100 Subject: [PATCH 32/44] Update comments --- src/Compiler/CodeGen/IlxGen.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 029b6ac93ba..fec0919e58b 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -1244,7 +1244,7 @@ and IlxGenEnv = /// Delay code gen for files. delayCodeGen: bool - /// Collection of code-gen functions where each function represents a file. + /// Collection of code-gen functions where each inner array represents codegen (method bodies) functions for a single file delayedFileGenReverse: list<(unit -> unit)[]> } From 3922eaf2175b5c45d67759265ee286a04c1ae956 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 29 Nov 2022 16:25:04 +0100 Subject: [PATCH 33/44] Removing commented-out code --- src/Compiler/CodeGen/IlxGen.fs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index fec0919e58b..f9c053ea1a8 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -9107,9 +9107,6 @@ and GenMethodForBinding (tailCallInfo, mspec.Name, eenvForMeth, - //{ eenvForMeth with - // delayCodeGen = false - //}, 0, selfValOpt, bodyExpr, From 5c4212ea88104438ecf2c81d41ff080e9c6cad00 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 29 Nov 2022 16:28:24 +0100 Subject: [PATCH 34/44] Code review feedback Co-authored-by: Petr Pokorny --- src/Compiler/CodeGen/IlxGen.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index f9c053ea1a8..06d6a356997 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -2314,7 +2314,7 @@ and AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbuf let mutable explicitEntryPointInfo: ILTypeRef option = None /// static init fields on script modules. - let scriptInitFspecs = new ConcurrentStack() + let scriptInitFspecs = ConcurrentStack() member _.AddScriptInitFieldSpec(fieldSpec, range) = scriptInitFspecs.Push((fieldSpec, range)) From 019b00287024f109d163b531574f327dde12c4b4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 15:33:59 +0000 Subject: [PATCH 35/44] Automated command ran: fantomas Co-authored-by: T-Gro <46543583+T-Gro@users.noreply.github.com> --- src/Compiler/AbstractIL/il.fs | 2 +- src/Compiler/CodeGen/IlxGen.fs | 11 +---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/Compiler/AbstractIL/il.fs b/src/Compiler/AbstractIL/il.fs index 19896896475..a907da79373 100644 --- a/src/Compiler/AbstractIL/il.fs +++ b/src/Compiler/AbstractIL/il.fs @@ -3902,7 +3902,7 @@ let prependInstrsToCode (instrs: ILInstr list) (c2: ILCode) = // If there is a sequence point as the first instruction then keep it at the front | I_seqpoint _ as i0 -> let labels = - let dict = Dictionary.newWithSize (c2.Labels.Count * 2) // Decrease chance of collisions by oversizing the hashtable + let dict = Dictionary.newWithSize (c2.Labels.Count * 2) // Decrease chance of collisions by oversizing the hashtable for kvp in c2.Labels do dict.Add(kvp.Key, (if kvp.Value = 0 then 0 else kvp.Value + n)) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 06d6a356997..634d64e3072 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -9101,16 +9101,7 @@ and GenMethodForBinding | _ -> None let ilLazyCode = - DelayCodeGenMethodForExpr - cenv - mgbuf - (tailCallInfo, - mspec.Name, - eenvForMeth, - 0, - selfValOpt, - bodyExpr, - sequel) + DelayCodeGenMethodForExpr cenv mgbuf (tailCallInfo, mspec.Name, eenvForMeth, 0, selfValOpt, bodyExpr, sequel) // This is the main code generation for most methods false, MethodBody.IL(ilLazyCode), false From c86d030199e9e2d0b9ad4ae00e9ba7950c5e4283 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 29 Nov 2022 16:38:38 +0100 Subject: [PATCH 36/44] Spelling corrections --- src/Compiler/CodeGen/IlxGen.fs | 2 +- src/Compiler/Utilities/illib.fs | 2 +- src/Compiler/Utilities/illib.fsi | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 634d64e3072..091e2fa3328 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -2293,7 +2293,7 @@ and AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbuf // The definitions of top level values, as quotations let reflectedDefinitions = - new StampedDictionairy(HashIdentity.Reference) + new StampedDictionary(HashIdentity.Reference) // A memoization table for generating value types for big constant arrays let rawDataValueTypeGenerator = diff --git a/src/Compiler/Utilities/illib.fs b/src/Compiler/Utilities/illib.fs index 365595bc441..d51020452f2 100644 --- a/src/Compiler/Utilities/illib.fs +++ b/src/Compiler/Utilities/illib.fs @@ -1068,7 +1068,7 @@ type MemoizationTable<'T, 'U>(compute: 'T -> 'U, keyComparer: IEqualityComparer< compute x /// A thread-safe lookup table which is assigning an auto-increment stamp with each insert -type internal StampedDictionairy<'T, 'U>(keyComparer: IEqualityComparer<'T>) = +type internal StampedDictionary<'T, 'U>(keyComparer: IEqualityComparer<'T>) = let table = new ConcurrentDictionary<'T, Lazy>(keyComparer) let mutable count = -1 diff --git a/src/Compiler/Utilities/illib.fsi b/src/Compiler/Utilities/illib.fsi index 8ddaf3cd45d..ebc6614df45 100644 --- a/src/Compiler/Utilities/illib.fsi +++ b/src/Compiler/Utilities/illib.fsi @@ -439,9 +439,9 @@ type internal MemoizationTable<'T, 'U> = member Apply: x: 'T -> 'U /// A thread-safe lookup table which is assigning an auto-increment stamp with each insert -type internal StampedDictionairy<'T, 'U> = +type internal StampedDictionary<'T, 'U> = - new: keyComparer: IEqualityComparer<'T> -> StampedDictionairy<'T, 'U> + new: keyComparer: IEqualityComparer<'T> -> StampedDictionary<'T, 'U> member Add: key: 'T * value: 'U -> unit From 53e59ce247aa98dcde479c1c5c670be341d5b2ef Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Wed, 30 Nov 2022 14:54:42 +0100 Subject: [PATCH 37/44] Update src/Compiler/CodeGen/IlxGen.fs Co-authored-by: Petr Pokorny --- src/Compiler/CodeGen/IlxGen.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 091e2fa3328..2b69873c4d5 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -2338,7 +2338,7 @@ and AssemblyBuilder(cenv: cenv, anonTypeTable: AnonTypeGenerationTable) as mgbuf [] ) - scriptInitFspecs.ToArray() |> Array.iter InitializeCompiledScript + scriptInitFspecs |> Seq.iter InitializeCompiledScript | None -> () member _.GenerateRawDataValueType(cloc, size) = From 0b4dbb5c2c04f842fa174f15b0f44e1394887780 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Mon, 5 Dec 2022 07:52:37 +0100 Subject: [PATCH 38/44] Testing enablement via -p:AdditionalFscCmdFlags --- FSharpBuild.Directory.Build.props | 1 + 1 file changed, 1 insertion(+) diff --git a/FSharpBuild.Directory.Build.props b/FSharpBuild.Directory.Build.props index 0c8d9cef75c..21e531409e8 100644 --- a/FSharpBuild.Directory.Build.props +++ b/FSharpBuild.Directory.Build.props @@ -27,6 +27,7 @@ $(OtherFlags) --nowarn:3384 $(OtherFlags) --times --nowarn:75 $(OtherFlags) --test:ParallelCheckingWithSignatureFilesOn + $(OtherFlags) $(AdditionalFscCmdFlags) From 452135f08ceb446de0ef3cd99329ead95702b807 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Wed, 7 Dec 2022 15:05:42 +0100 Subject: [PATCH 39/44] Preventing "enqueue within enqueue" Without this, code in optimize+ was calling this method within its own invocation. Therefore codegen was delayed, but never picked up --- src/Compiler/CodeGen/IlxGen.fs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Compiler/CodeGen/IlxGen.fs b/src/Compiler/CodeGen/IlxGen.fs index 2b69873c4d5..efb652eb407 100644 --- a/src/Compiler/CodeGen/IlxGen.fs +++ b/src/Compiler/CodeGen/IlxGen.fs @@ -3128,13 +3128,17 @@ and CodeGenMethodForExpr cenv mgbuf (entryPointInfo, methodName, eenv, alreadyUs code and DelayCodeGenMethodForExpr cenv mgbuf ((_, _, eenv, _, _, _, _) as args) = + let change3rdOutOf7 (a1, a2, _, a4, a5, a6, a7) newA3 = (a1, a2, newA3, a4, a5, a6, a7) + if eenv.delayCodeGen then let cenv = { cenv with stackGuard = getEmptyStackGuard () } + // Once this is lazily-evaluated later, it should not put things in queue. They would not be picked up by anyone. + let newArgs = change3rdOutOf7 args { eenv with delayCodeGen = false } - let lazyMethodBody = lazy (CodeGenMethodForExpr cenv mgbuf args) + let lazyMethodBody = lazy (CodeGenMethodForExpr cenv mgbuf newArgs) cenv.delayedGenMethods.Enqueue(fun () -> lazyMethodBody.Force() |> ignore) lazyMethodBody else From b0a94c2cd4f98dce00f334fb3404a79d546c235a Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Thu, 5 Jan 2023 12:51:38 +0100 Subject: [PATCH 40/44] Drive signature file parallel checking based on new flag as well --- src/Compiler/Driver/CompilerConfig.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compiler/Driver/CompilerConfig.fs b/src/Compiler/Driver/CompilerConfig.fs index 077fb5bad2f..b7da443ccb8 100644 --- a/src/Compiler/Driver/CompilerConfig.fs +++ b/src/Compiler/Driver/CompilerConfig.fs @@ -738,7 +738,7 @@ type TcConfigBuilder = emitTailcalls = true deterministic = false concurrentBuild = true - parallelCheckingWithSignatureFiles = false + parallelCheckingWithSignatureFiles = FsharpExperimentalFeaturesEnabledAutomatically parallelIlxGen = FsharpExperimentalFeaturesEnabledAutomatically emitMetadataAssembly = MetadataAssemblyGeneration.None preferredUiLang = None From 7e242e709d0b67fefe2c361186bfbeb0f0827327 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Thu, 5 Jan 2023 16:45:20 +0100 Subject: [PATCH 41/44] Adding FSHARP_EXPERIMENTAL_FEATURES regular and deterministic builds --- azure-pipelines.yml | 48 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d2590b5ba6c..547f7f9213c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -238,6 +238,39 @@ stages: continueOnError: true condition: not(succeeded()) + # Determinism with experimental features on + - job: Determinism_Experimental_Features + condition: eq(variables['Build.Reason'], 'PullRequest') + variables: + - name: _SignType + value: Test + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 90 + steps: + - checkout: self + clean: true + - task: UseDotNet@2 + displayName: install SDK + inputs: + packageType: sdk + useGlobalJson: true + includePreviewVersions: false + workingDirectory: $(Build.SourcesDirectory) + installationPath: $(Build.SourcesDirectory)/.dotnet + - script: .\eng\test-determinism.cmd -configuration Debug + env: + FSHARP_EXPERIMENTAL_FEATURES: 1 + displayName: Determinism Experimental Features tests with Debug configuration + - task: PublishPipelineArtifact@1 + displayName: Publish Determinism Experimental Logs + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts/log/Debug' + artifactName: 'Determinism_Experimental_Features Attempt $(System.JobAttempt) Logs' + continueOnError: true + condition: not(succeeded()) + # Check code formatting - job: CheckCodeFormatting pool: @@ -512,8 +545,9 @@ stages: # filePath: eng\tests\UpToDate.ps1 # arguments: -configuration $(_BuildConfig) -ci -binaryLog - # Run Build with --test:ParallelCheckingWithSignatureFilesOn - - job: ParallelCheckingWithSignatureFiles + # Run Build with Fsharp Experimental Features + # Possible change: --times:$(Build.SourcesDirectory)/artifacts/log/Release/compiler_timing.csv + - job: FsharpExperimentalFeaturesBuild condition: eq(variables['Build.Reason'], 'PullRequest') variables: - name: _SignType @@ -533,13 +567,15 @@ stages: includePreviewVersions: false workingDirectory: $(Build.SourcesDirectory) installationPath: $(Build.SourcesDirectory)/.dotnet - - script: .\build.cmd -c Release -binaryLog /p:ParallelCheckingWithSignatureFilesOn=true - displayName: ParallelCheckingWithSignatureFiles build with Debug configuration + - script: .\build.cmd -c Release -binaryLog /p:AdditionalFscCmdFlags="--times" + env: + FSHARP_EXPERIMENTAL_FEATURES: 1 + displayName: FsharpExperimentalFeaturesBuild in release mode - task: PublishPipelineArtifact@1 - displayName: Publish ParallelCheckingWithSignatureFiles Logs + displayName: Publish FsharpExperimentalFeaturesBuild Logs inputs: targetPath: '$(Build.SourcesDirectory)/artifacts/log/Release' - artifactName: 'ParallelCheckingWithSignatureFiles Attempt $(System.JobAttempt) Logs' + artifactName: 'FsharpExperimentalFeaturesBuild Attempt $(System.JobAttempt) Logs' continueOnError: true # Plain build Windows From ebec2eb0f82a2b64482c4753e99f20ea8d34d9b1 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 6 Jan 2023 10:19:23 +0100 Subject: [PATCH 42/44] When running a deterministic build, parallel ilxgen is disabled (to have stable closure numeric suffixes) --- azure-pipelines.yml | 2 +- src/Compiler/Driver/OptimizeInputs.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 547f7f9213c..fc8e0d41744 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -238,7 +238,7 @@ stages: continueOnError: true condition: not(succeeded()) - # Determinism with experimental features on + # Determinism with experimental features - job: Determinism_Experimental_Features condition: eq(variables['Build.Reason'], 'PullRequest') variables: diff --git a/src/Compiler/Driver/OptimizeInputs.fs b/src/Compiler/Driver/OptimizeInputs.fs index 68c1452003b..cce4589c2f9 100644 --- a/src/Compiler/Driver/OptimizeInputs.fs +++ b/src/Compiler/Driver/OptimizeInputs.fs @@ -255,7 +255,7 @@ let GenerateIlxCode isInteractive = tcConfig.isInteractive isInteractiveItExpr = isInteractiveItExpr alwaysCallVirt = tcConfig.alwaysCallVirt - parallelIlxGenEnabled = tcConfig.parallelIlxGen + parallelIlxGenEnabled = tcConfig.parallelIlxGen && not(tcConfig.deterministic) } ilxGenerator.GenerateCode(ilxGenOpts, optimizedImpls, topAttrs.assemblyAttrs, topAttrs.netModuleAttrs) From f7f2ee15483553d0f7fab2d033ca0b6188d08cbb Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 6 Jan 2023 10:58:45 +0100 Subject: [PATCH 43/44] fantomas --- src/Compiler/Driver/OptimizeInputs.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compiler/Driver/OptimizeInputs.fs b/src/Compiler/Driver/OptimizeInputs.fs index cce4589c2f9..4c4dac6ac36 100644 --- a/src/Compiler/Driver/OptimizeInputs.fs +++ b/src/Compiler/Driver/OptimizeInputs.fs @@ -255,7 +255,7 @@ let GenerateIlxCode isInteractive = tcConfig.isInteractive isInteractiveItExpr = isInteractiveItExpr alwaysCallVirt = tcConfig.alwaysCallVirt - parallelIlxGenEnabled = tcConfig.parallelIlxGen && not(tcConfig.deterministic) + parallelIlxGenEnabled = tcConfig.parallelIlxGen && not (tcConfig.deterministic) } ilxGenerator.GenerateCode(ilxGenOpts, optimizedImpls, topAttrs.assemblyAttrs, topAttrs.netModuleAttrs) From 4eeb56ea85018a8a22353029c37034ab28bb8b1f Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 6 Jan 2023 11:49:32 +0100 Subject: [PATCH 44/44] Deduplicating build definitions into matrix --- azure-pipelines.yml | 83 +++++++-------------------- src/Compiler/Driver/CompilerConfig.fs | 3 +- 2 files changed, 22 insertions(+), 64 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fc8e0d41744..0e658575eab 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -217,6 +217,13 @@ stages: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals $(WindowsMachineQueueName) timeoutInMinutes: 90 + strategy: + maxParallel: 2 + matrix: + regular: + _experimental_flag: null + experimental_features: + _experimental_flag: 1 steps: - checkout: self clean: true @@ -229,6 +236,8 @@ stages: workingDirectory: $(Build.SourcesDirectory) installationPath: $(Build.SourcesDirectory)/.dotnet - script: .\eng\test-determinism.cmd -configuration Debug + env: + FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) displayName: Determinism tests with Debug configuration - task: PublishPipelineArtifact@1 displayName: Publish Determinism Logs @@ -238,39 +247,6 @@ stages: continueOnError: true condition: not(succeeded()) - # Determinism with experimental features - - job: Determinism_Experimental_Features - condition: eq(variables['Build.Reason'], 'PullRequest') - variables: - - name: _SignType - value: Test - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - timeoutInMinutes: 90 - steps: - - checkout: self - clean: true - - task: UseDotNet@2 - displayName: install SDK - inputs: - packageType: sdk - useGlobalJson: true - includePreviewVersions: false - workingDirectory: $(Build.SourcesDirectory) - installationPath: $(Build.SourcesDirectory)/.dotnet - - script: .\eng\test-determinism.cmd -configuration Debug - env: - FSHARP_EXPERIMENTAL_FEATURES: 1 - displayName: Determinism Experimental Features tests with Debug configuration - - task: PublishPipelineArtifact@1 - displayName: Publish Determinism Experimental Logs - inputs: - targetPath: '$(Build.SourcesDirectory)/artifacts/log/Debug' - artifactName: 'Determinism_Experimental_Features Attempt $(System.JobAttempt) Logs' - continueOnError: true - condition: not(succeeded()) - # Check code formatting - job: CheckCodeFormatting pool: @@ -525,11 +501,22 @@ stages: pool: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals $(WindowsMachineQueueName) + strategy: + maxParallel: 2 + matrix: + regular: + _experimental_flag: null + experimental_features: + _experimental_flag: 1 steps: - checkout: self clean: true - script: .\Build.cmd -c Release -pack + env: + FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c Release + env: + FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) displayName: End to end build tests # Up-to-date - disabled due to it being flaky @@ -547,36 +534,6 @@ stages: # Run Build with Fsharp Experimental Features # Possible change: --times:$(Build.SourcesDirectory)/artifacts/log/Release/compiler_timing.csv - - job: FsharpExperimentalFeaturesBuild - condition: eq(variables['Build.Reason'], 'PullRequest') - variables: - - name: _SignType - value: Test - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - timeoutInMinutes: 90 - steps: - - checkout: self - clean: true - - task: UseDotNet@2 - displayName: install SDK - inputs: - packageType: sdk - useGlobalJson: true - includePreviewVersions: false - workingDirectory: $(Build.SourcesDirectory) - installationPath: $(Build.SourcesDirectory)/.dotnet - - script: .\build.cmd -c Release -binaryLog /p:AdditionalFscCmdFlags="--times" - env: - FSHARP_EXPERIMENTAL_FEATURES: 1 - displayName: FsharpExperimentalFeaturesBuild in release mode - - task: PublishPipelineArtifact@1 - displayName: Publish FsharpExperimentalFeaturesBuild Logs - inputs: - targetPath: '$(Build.SourcesDirectory)/artifacts/log/Release' - artifactName: 'FsharpExperimentalFeaturesBuild Attempt $(System.JobAttempt) Logs' - continueOnError: true # Plain build Windows - job: Plain_Build_Windows diff --git a/src/Compiler/Driver/CompilerConfig.fs b/src/Compiler/Driver/CompilerConfig.fs index b7da443ccb8..c57e02eebeb 100644 --- a/src/Compiler/Driver/CompilerConfig.fs +++ b/src/Compiler/Driver/CompilerConfig.fs @@ -48,7 +48,8 @@ let FSharpIndentationAwareSyntaxFileSuffixes = [ ".fs"; ".fsscript"; ".fsx"; ".fsi" ] let FsharpExperimentalFeaturesEnabledAutomatically = - Environment.GetEnvironmentVariable("FSHARP_EXPERIMENTAL_FEATURES") |> isNotNull + String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("FSHARP_EXPERIMENTAL_FEATURES")) + |> not //-------------------------------------------------------------------------- // General file name resolver