Determinism of the F# Compiler #1042 (#2954)

* Add IL output option for deterministic output

Thread a new option through the options to the code writing the IL.
For the moment just use a constant for deterministic timestamp, and use 0 as the timestamp in the MVID

* Basic check to prevent wildcard version + deterministic at the same time

* Hash the code, data and metadata, and use for mvid + timestamp

* Fix fsc & fsci help tests by adding in new command line option for deterministic

* tests for command line, wildcard versions, normal non-deterministic behaviour

* Attempt to get CI to support timeout

* Allow PRECMD to call FSI more than once, and use this for copying files and pausing to prevent race-condition

* PortablePdbBuilder use consistent id 

If compiling deterministically, use deterministic id provider support in 
making a pdb builder. see https://github.com/dotnet/roslyn/blob/master/src/Compilers/Core/Portable/PEWriter/PeWriter.cs#L132

* Portable and Embedded pdb now deterministic

* wip: fix up timestamp in pdb, but mvids still out of sequence

* Fix breaking change from rebasing on master

* Add notes about determinism with non portable pdbs
Make the checks for finding embedded guids more
strict and unique

* Throw error if determinsitc build and non-portable pdb specified

Author: davidglassborow

src/absil/ilwrite.fs

@@ -551,6 +551,7 @@ type MetadataTable =
 type cenv = 
     { ilg: ILGlobals
       emitTailcalls: bool
+      deterministic: bool
       showTimes: bool
       desiredMetadataVersion: ILVersionInfo
       requiredDataFixups: (int32 * (int * bool)) list ref
@@ -2903,9 +2904,15 @@ and newGuid (modul: ILModuleDef) =
     let m2 = hash modul.Name
     [| b0 m; b1 m; b2 m; b3 m; b0 m2; b1 m2; b2 m2; b3 m2; 0xa7uy; 0x45uy; 0x03uy; 0x83uy; b0 n; b1 n; b2 n; b3 n |]
 
-and GetModuleAsRow cenv (modul: ILModuleDef) = 
+and deterministicGuid (modul: ILModuleDef) =
+    let n = 16909060
+    let m = hash n
+    let m2 = hash modul.Name
+    [| b0 m; b1 m; b2 m; b3 m; b0 m2; b1 m2; b2 m2; b3 m2; 0xa7uy; 0x45uy; 0x03uy; 0x83uy; b0 n; b1 n; b2 n; b3 n |]
+
+and GetModuleAsRow (cenv:cenv) (modul: ILModuleDef) = 
     // Store the generated MVID in the environment (needed for generating debug information)
-    let modulGuid = newGuid modul
+    let modulGuid = if cenv.deterministic then deterministicGuid modul else newGuid modul
     cenv.moduleGuid <- modulGuid
     UnsharedRow 
         [| UShort (uint16 0x0) 
@@ -2954,11 +2961,12 @@ let GenModule (cenv : cenv) (modul: ILModuleDef) =
     GenTypeDefsPass4 [] cenv tds
     reportTime cenv.showTimes "Module Generation Pass 4"
 
-let generateIL requiredDataFixups (desiredMetadataVersion,generatePdb, ilg : ILGlobals, emitTailcalls,showTimes)  (m : ILModuleDef) cilStartAddress =
+let generateIL requiredDataFixups (desiredMetadataVersion,generatePdb, ilg : ILGlobals, emitTailcalls, deterministic, showTimes)  (m : ILModuleDef) cilStartAddress =
     let isDll = m.IsDLL
 
     let cenv = 
         { emitTailcalls=emitTailcalls
+          deterministic = deterministic
           showTimes=showTimes
           ilg = ilg
           desiredMetadataVersion=desiredMetadataVersion
@@ -3098,7 +3106,7 @@ module FileSystemUtilites =
 #endif
         ()
 
-let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls,showTimes) modul cilStartAddress = 
+let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls,deterministic,showTimes) modul cilStartAddress =
 
     // When we know the real RVAs of the data section we fixup the references for the FieldRVA table. 
     // These references are stored as offsets into the metadata we return from this function 
@@ -3107,7 +3115,7 @@ let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls
     let next = cilStartAddress
 
     let strings,userStrings,blobs,guids,tables,entryPointToken,code,requiredStringFixups,data,resources,pdbData,mappings = 
-      generateIL requiredDataFixups (desiredMetadataVersion,generatePdb,ilg,emitTailcalls,showTimes) modul cilStartAddress
+      generateIL requiredDataFixups (desiredMetadataVersion,generatePdb,ilg,emitTailcalls,deterministic,showTimes) modul cilStartAddress
 
     reportTime showTimes "Generated Tables and Code"
     let tableSize (tab: TableName) = tables.[tab.Index].Count
@@ -3404,7 +3412,7 @@ let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls
 
     reportTime showTimes "Layout Metadata"
 
-    let metadata = 
+    let metadata, guidStart =
       let mdbuf =  ByteBuffer.Create 500000 
       mdbuf.EmitIntsAsBytes 
         [| 0x42; 0x53; 0x4a; 0x42; // Magic signature 
@@ -3462,6 +3470,7 @@ let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls
 
       reportTime showTimes "Write Metadata User Strings";
     // The GUID stream 
+      let guidStart = mdbuf.Position
       Array.iter mdbuf.EmitBytes guids;
 
     // The blob stream 
@@ -3473,7 +3482,7 @@ let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls
           mdbuf.EmitIntAsByte 0x00;
       reportTime showTimes "Write Blob Stream";
      // Done - close the buffer and return the result. 
-      mdbuf.Close()
+      mdbuf.Close(), guidStart
 
 
    // Now we know the user string tables etc. we can fixup the 
@@ -3488,7 +3497,7 @@ let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls
               applyFixup32 code locInCode token
     reportTime showTimes "Fixup Metadata";
 
-    entryPointToken,code, codePadding,metadata,data,resources,!requiredDataFixups,pdbData,mappings
+    entryPointToken,code, codePadding,metadata,data,resources,!requiredDataFixups,pdbData,mappings,guidStart
 
 //---------------------------------------------------------------------
 // PHYSICAL METADATA+BLOBS --> PHYSICAL PE FORMAT
@@ -3550,7 +3559,7 @@ let writeBytes (os: BinaryWriter) (chunk:byte[]) = os.Write(chunk,0,chunk.Length
 
 let writeBinaryAndReportMappings (outfile, 
                                   ilg: ILGlobals, pdbfile: string option, signer: ILStrongNameSigner option, portablePDB, embeddedPDB, 
-                                  embedAllSource, embedSourceList, sourceLink, emitTailcalls, showTimes, dumpDebugInfo) modul =
+                                  embedAllSource, embedSourceList, sourceLink, emitTailcalls, deterministic, showTimes, dumpDebugInfo ) modul =
     // Store the public key from the signer into the manifest.  This means it will be written 
     // to the binary and also acts as an indicator to leave space for delay sign 
 
@@ -3666,8 +3675,8 @@ let writeBinaryAndReportMappings (outfile,
                     | Some v -> v
                     | None -> failwith "Expected msorlib to have a version number"
 
-          let entryPointToken,code,codePadding,metadata,data,resources,requiredDataFixups,pdbData,mappings = 
-            writeILMetadataAndCode ((pdbfile <> None), desiredMetadataVersion, ilg,emitTailcalls,showTimes) modul next
+          let entryPointToken,code,codePadding,metadata,data,resources,requiredDataFixups,pdbData,mappings,guidStart =
+            writeILMetadataAndCode ((pdbfile <> None), desiredMetadataVersion, ilg,emitTailcalls, deterministic, showTimes) modul next
 
           reportTime showTimes "Generated IL and metadata";
           let _codeChunk,next = chunk code.Length next
@@ -3703,7 +3712,7 @@ let writeBinaryAndReportMappings (outfile,
           let pdbOpt =
             match portablePDB with
             | true  -> 
-                let (uncompressedLength, contentId, stream) as pdbStream = generatePortablePdb embedAllSource embedSourceList sourceLink showTimes pdbData 
+                let (uncompressedLength, contentId, stream) as pdbStream = generatePortablePdb embedAllSource embedSourceList sourceLink showTimes pdbData deterministic
                 if embeddedPDB then Some (compressPortablePdbStream uncompressedLength contentId stream)
                 else Some (pdbStream)
             | _ -> None
@@ -3849,7 +3858,33 @@ let writeBinaryAndReportMappings (outfile,
             writeInt32AsUInt16 os 0x014c;   // Machine - IMAGE_FILE_MACHINE_I386 
 
           writeInt32AsUInt16 os numSections;
-          writeInt32 os timestamp   // date since 1970 
+
+          let pdbData = 
+            if deterministic then
+              // Hash code, data and metadata
+              use sha = System.Security.Cryptography.SHA1.Create()    // IncrementalHash is core only
+              let hCode = sha.ComputeHash code
+              let hData = sha.ComputeHash data
+              let hMeta = sha.ComputeHash metadata
+              let final = [| hCode; hData; hMeta |] |> Array.collect id |> sha.ComputeHash
+
+              // Confirm we have found the correct data and aren't corrupting the metadata
+              if metadata.[ guidStart..guidStart+3]     <> [| 4uy; 3uy; 2uy; 1uy |] then failwith "Failed to find MVID"
+              if metadata.[ guidStart+12..guidStart+15] <> [| 4uy; 3uy; 2uy; 1uy |] then failwith "Failed to find MVID"
+
+              // Update MVID guid in metadata
+              Array.blit final 0 metadata guidStart 16
+
+              // Use last 4 bytes for timestamp - High bit set, to stop tool chains becoming confused
+              let timestamp = int final.[16] ||| (int final.[17] <<< 8) ||| (int final.[18] <<< 16) ||| (int (final.[19] ||| 128uy) <<< 24) 
+              writeInt32 os timestamp
+              // Update pdbData with new guid and timestamp.  Portable and embedded PDBs don't need the ModuleID
+              // Full and PdbOnly aren't supported under deterministic builds currently, they rely on non-determinsitic Windows native code
+              { pdbData with ModuleID = final.[0..15] ; Timestamp = timestamp }
+            else
+              writeInt32 os timestamp   // date since 1970
+              pdbData
+
           writeInt32 os 0x00; // Pointer to Symbol Table Always 0 
        // 00000090 
           writeInt32 os 0x00; // Number of Symbols Always 0 
@@ -4277,12 +4312,13 @@ type options =
      sourceLink: string
      signer: ILStrongNameSigner option
      emitTailcalls : bool
+     deterministic : bool
      showTimes: bool
      dumpDebugInfo:bool }
 
 let WriteILBinary (outfile, (args: options), modul) =
     writeBinaryAndReportMappings (outfile, 
                                   args.ilg, args.pdbfile, args.signer, args.portablePDB, args.embeddedPDB, args.embedAllSource, 
-                                  args.embedSourceList, args.sourceLink, args.emitTailcalls, args.showTimes, args.dumpDebugInfo) modul 
+                                  args.embedSourceList, args.sourceLink, args.emitTailcalls, args.deterministic, args.showTimes, args.dumpDebugInfo) modul
     |> ignore
 

src/absil/ilwrite.fsi

@@ -25,6 +25,7 @@ type options =
    sourceLink: string
    signer : ILStrongNameSigner option
    emitTailcalls: bool
+   deterministic: bool
    showTimes : bool
    dumpDebugInfo : bool }
 

src/absil/ilwritepdb.fs

@@ -219,7 +219,7 @@ let getRowCounts tableRowCounts =
     tableRowCounts |> Seq.iter(fun x -> builder.Add(x))
     builder.MoveToImmutable()
 
-let generatePortablePdb (embedAllSource:bool) (embedSourceList:string list) (sourceLink:string) showTimes (info:PdbData) = 
+let generatePortablePdb (embedAllSource:bool) (embedSourceList:string list) (sourceLink:string) showTimes (info:PdbData) isDeterministic =
     sortMethods showTimes info
     let externalRowCounts = getRowCounts info.TableRowCounts
     let docs = 
@@ -445,7 +445,19 @@ let generatePortablePdb (embedAllSource:bool) (embedSourceList:string list) (sou
         | None -> MetadataTokens.MethodDefinitionHandle(0)
         | Some x -> MetadataTokens.MethodDefinitionHandle(x)
 
-    let serializer = PortablePdbBuilder(metadata, externalRowCounts, entryPoint, null)
+    let deterministicIdProvider isDeterministic  : System.Func<IEnumerable<Blob>, BlobContentId> = 
+        match isDeterministic with
+        | false -> null
+        | true ->
+            let convert (content:IEnumerable<Blob>) = 
+                use sha = System.Security.Cryptography.SHA1.Create()    // IncrementalHash is core only
+                let hash = content 
+                           |> Seq.map ( fun c -> c.GetBytes().Array |> sha.ComputeHash )         
+                           |> Seq.collect id |> Array.ofSeq |> sha.ComputeHash
+                BlobContentId.FromHash(hash)
+            System.Func<IEnumerable<Blob>, BlobContentId>( convert )
+
+    let serializer = PortablePdbBuilder(metadata, externalRowCounts, entryPoint, deterministicIdProvider isDeterministic)
     let blobBuilder = new BlobBuilder()
     let contentId= serializer.Serialize(blobBuilder)
     let portablePdbStream = new MemoryStream()

src/absil/ilwritepdb.fsi

@@ -82,7 +82,7 @@ type idd =
       iddData: byte[];
       iddChunk: BinaryChunk }
 
-val generatePortablePdb : embedAllSource:bool -> embedSourceList:string list -> sourceLink: string -> showTimes:bool -> info:PdbData -> (int64 * BlobContentId * MemoryStream) 
+val generatePortablePdb : embedAllSource:bool -> embedSourceList:string list -> sourceLink: string -> showTimes:bool -> info:PdbData -> isDeterministic:bool -> (int64 * BlobContentId * MemoryStream)
 val compressPortablePdbStream : uncompressedLength:int64 -> contentId:BlobContentId -> stream:MemoryStream -> (int64 * BlobContentId * MemoryStream)
 val embedPortablePdbInfo : uncompressedLength:int64 -> contentId:BlobContentId -> stream:MemoryStream -> showTimes:bool -> fpdb:string -> cvChunk:BinaryChunk -> pdbChunk:BinaryChunk -> idd[]
 val writePortablePdbInfo : contentId:BlobContentId -> stream:MemoryStream -> showTimes:bool -> fpdb:string -> cvChunk:BinaryChunk -> idd[]

src/fsharp/CompileOps.fs

@@ -2133,6 +2133,7 @@ type TcConfigBuilder =
       mutable optsOn        : bool (* optimizations are turned on *)
       mutable optSettings   : Optimizer.OptimizationSettings 
       mutable emitTailcalls : bool
+      mutable deterministic : bool
 #if PREFERRED_UI_LANG
       mutable preferredUiLang: string option
 #endif
@@ -2301,6 +2302,7 @@ type TcConfigBuilder =
           optsOn        = false 
           optSettings   = Optimizer.OptimizationSettings.Defaults
           emitTailcalls = true
+          deterministic = false
 #if PREFERRED_UI_LANG
           preferredUiLang = None
 #endif
@@ -2791,6 +2793,7 @@ type TcConfig private (data : TcConfigBuilder,validate:bool) =
     member x.doFinalSimplify    = data.doFinalSimplify
     member x.optSettings        = data.optSettings
     member x.emitTailcalls      = data.emitTailcalls
+    member x.deterministic      = data.deterministic
 #if PREFERRED_UI_LANG
     member x.preferredUiLang    = data.preferredUiLang
 #endif

src/fsharp/CompileOps.fsi

@@ -342,6 +342,7 @@ type TcConfigBuilder =
       mutable optsOn        : bool 
       mutable optSettings   : Optimizer.OptimizationSettings 
       mutable emitTailcalls : bool
+      mutable deterministic : bool
 #if PREFERRED_UI_LANG
       mutable preferredUiLang: string option
 #endif
@@ -494,6 +495,7 @@ type TcConfig =
     member doFinalSimplify : bool
     member optSettings   : Optimizer.OptimizationSettings 
     member emitTailcalls : bool
+    member deterministic : bool
 #if PREFERRED_UI_LANG
     member preferredUiLang: string option
 #else

src/fsharp/CompileOptions.fs

@@ -420,7 +420,10 @@ let SetOptimizeSwitch (tcConfigB : TcConfigBuilder) switch =
 
 let SetTailcallSwitch (tcConfigB : TcConfigBuilder) switch =
     tcConfigB.emitTailcalls <- (switch = OptionSwitch.On)
-        
+
+let SetDeterministicSwitch (tcConfigB : TcConfigBuilder) switch =
+    tcConfigB.deterministic <- (switch = OptionSwitch.On)
+
 let jitoptimizeSwitch (tcConfigB : TcConfigBuilder) switch =
     tcConfigB.optSettings <- { tcConfigB.optSettings with jitOptUser = Some (switch = OptionSwitch.On) }
 
@@ -676,6 +679,8 @@ let codeGenerationFlags isFsi (tcConfigB : TcConfigBuilder) =
                             Some (FSComp.SR.optsOptimize()))
          CompilerOption("tailcalls", tagNone, OptionSwitch (SetTailcallSwitch tcConfigB), None,
                             Some (FSComp.SR.optsTailcalls()))
+         CompilerOption("deterministic", tagNone, OptionSwitch (SetDeterministicSwitch tcConfigB), None,
+                            Some (FSComp.SR.optsDeterministic()))
          CompilerOption("crossoptimize", tagNone, OptionSwitch (crossOptimizeSwitch tcConfigB), None,
                             Some (FSComp.SR.optsCrossoptimize()))
         ]

src/fsharp/FSComp.txt

@@ -859,6 +859,7 @@ optsDebugPM,"Emit debug information (Short form: -g)"
 optsDebug,"Specify debugging type: full, portable, embedded, pdbonly. ('%s' is the default if no debuggging type specified and enables attaching a debugger to a running program, 'portable' is a cross-platform format, 'embedded' is a cross-platform format embedded into the output file)."
 optsOptimize,"Enable optimizations (Short form: -O)"
 optsTailcalls,"Enable or disable tailcalls"
+optsDeterministic,"Produce a deterministic assembly (including module version GUID and timestamp)"
 optsCrossoptimize,"Enable or disable cross-module optimizations"
 optsWarnaserrorPM,"Report all warnings as errors"
 optsWarnaserror,"Report specific warnings as errors"
@@ -1137,6 +1138,8 @@ fscTooManyErrors,"Exiting - too many errors"
 2022,pathIsInvalid,"Problem with filename '%s': Illegal characters in path."
 2023,fscResxSourceFileDeprecated,"Passing a .resx file (%s) as a source file to the compiler is deprecated. Use resgen.exe to transform the .resx file into a .resources file to pass as a --resource option. If you are using MSBuild, this can be done via an <EmbeddedResource> item in the .fsproj project file."
 2024,fscStaticLinkingNoProfileMismatches,"Static linking may not be used on an assembly referencing mscorlib (e.g. a .NET Framework assembly) when generating an assembly that references System.Runtime (e.g. a .NET Core or Portable assembly)."
+2025,fscAssemblyWildcardAndDeterminism,"An %s specified version '%s', but this value is a wildcard, and you have requested a deterministic build, these are in conflict."
+2026,fscDeterministicDebugRequiresPortablePdb,"Determinstic builds only support portable PDBs (--debug:portable or --debug:embedded)"
 3000,etIllegalCharactersInNamespaceName,"Character '%s' is not allowed in provided namespace name '%s'"
 3001,etNullOrEmptyMemberName,"The provided type '%s' returned a member with a null or empty member name"
 3002,etNullMember,"The provided type '%s' returned a null member"

src/fsharp/fsc.fs

@@ -249,6 +249,9 @@ let ProcessCommandLineFlags (tcConfigB: TcConfigBuilder, setProcessThreadLocals,
         if not (String.IsNullOrEmpty(tcConfigB.sourceLink)) then
             error(Error(FSComp.SR.optsSourceLinkRequirePortablePDBs(), rangeCmdArgs))
 
+    if tcConfigB.deterministic && tcConfigB.debuginfo && (tcConfigB.portablePDB = false) then
+      error(Error(FSComp.SR.fscDeterministicDebugRequiresPortablePdb(), rangeCmdArgs))
+
     let inputFiles = List.rev !inputFilesRef
 
     // Check if we have a codepage from the console
@@ -704,9 +707,11 @@ module AttributeHelpers =
             None
 
     // Try to find an AssemblyVersion attribute 
-    let TryFindVersionAttribute g attrib attribName attribs =
+    let TryFindVersionAttribute g attrib attribName attribs deterministic =
         match TryFindStringAttribute g attrib attribs with
         | Some versionString ->
+             if deterministic && versionString.Contains("*") then
+                 errorR(Error(FSComp.SR.fscAssemblyWildcardAndDeterminism(attribName, versionString), Range.rangeStartup))
              try Some (IL.parseILVersion versionString)
              with e -> 
                  warning(Error(FSComp.SR.fscBadAssemblyVersion(attribName, versionString), Range.rangeStartup))
@@ -1810,7 +1815,7 @@ let main1(Args (ctok, tcGlobals, tcImports: TcImports, frameworkTcImports, gener
 
     // Try to find an AssemblyVersion attribute 
     let assemVerFromAttrib = 
-        match AttributeHelpers.TryFindVersionAttribute tcGlobals "System.Reflection.AssemblyVersionAttribute" "AssemblyVersionAttribute" topAttrs.assemblyAttrs with
+        match AttributeHelpers.TryFindVersionAttribute tcGlobals "System.Reflection.AssemblyVersionAttribute" "AssemblyVersionAttribute" topAttrs.assemblyAttrs tcConfig.deterministic with
         | Some v -> 
            match tcConfig.version with 
            | VersionNone -> Some v
@@ -1896,7 +1901,7 @@ let main1OfAst (ctok, legacyReferenceResolver, openBinariesInMemory, assemblyNam
 
     // Try to find an AssemblyVersion attribute 
     let assemVerFromAttrib = 
-        match AttributeHelpers.TryFindVersionAttribute tcGlobals "System.Reflection.AssemblyVersionAttribute" "AssemblyVersionAttribute" topAttrs.assemblyAttrs with
+        match AttributeHelpers.TryFindVersionAttribute tcGlobals "System.Reflection.AssemblyVersionAttribute" "AssemblyVersionAttribute" topAttrs.assemblyAttrs tcConfig.deterministic with
         | Some v -> 
             match tcConfig.version with 
             | VersionNone -> Some v
@@ -2016,6 +2021,7 @@ let main4 dynamicAssemblyCreator (Args (ctok, tcConfig, errorLogger: ErrorLogger
                   { ilg = tcGlobals.ilg
                     pdbfile=pdbfile
                     emitTailcalls = tcConfig.emitTailcalls
+                    deterministic = tcConfig.deterministic
                     showTimes = tcConfig.showTimes
                     portablePDB = tcConfig.portablePDB
                     embeddedPDB = tcConfig.embeddedPDB