From cdbfbdc7f95fe12addda9a82900dc66e663b565c Mon Sep 17 00:00:00 2001 From: Kevin Ransom Date: Sat, 20 Jul 2019 03:14:30 -0700 Subject: [PATCH 01/10] fixes issue #6832 --- FSharpBuild.Directory.Build.props | 3 +- eng/Build.ps1 | 11 ++-- eng/build-utils.ps1 | 3 +- src/absil/ilwritepdb.fs | 2 +- .../AssemblyCheck/AssemblyCheck.fsproj | 17 +++++ .../buildtools/AssemblyCheck/Program.fs | 64 +++++++++++++++---- src/buildtools/buildtools.proj | 1 + .../FSharp.Core.nuget/FSharp.Core.nuspec | 2 - 8 files changed, 82 insertions(+), 21 deletions(-) create mode 100644 src/buildtools/AssemblyCheck/AssemblyCheck.fsproj rename scripts/AssemblyVersionCheck.fsx => src/buildtools/AssemblyCheck/Program.fs (51%) diff --git a/FSharpBuild.Directory.Build.props b/FSharpBuild.Directory.Build.props index 515de9bbdc7..6687fb40888 100644 --- a/FSharpBuild.Directory.Build.props +++ b/FSharpBuild.Directory.Build.props @@ -75,6 +75,7 @@ https://github.com/Microsoft/visualfsharp git + <_DotGitDir>$(RepoRoot).git <_HeadFileContent Condition="Exists('$(_DotGitDir)/HEAD')">$([System.IO.File]::ReadAllText('$(_DotGitDir)/HEAD').Trim()) @@ -87,7 +88,7 @@ $(NoWarn);FS2003 true - portable + embedded fs false true diff --git a/eng/Build.ps1 b/eng/Build.ps1 index f329cba02ac..8011827a643 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -224,13 +224,14 @@ function UpdatePath() { TestAndAddToPath "$ArtifactsDir\bin\fsiAnyCpu\$configuration\net472" } -function VerifyAssemblyVersions() { - $fsiPath = Join-Path $ArtifactsDir "bin\fsi\Proto\net472\publish\fsi.exe" +function VerifyAssemblyVersionsAndSymbols() { + $assemblyVerCheckPath = Join-Path $ArtifactsDir "Bootstrap\AssemblyCheck\AssemblyCheck.dll" # Only verify versions on CI or official build if ($ci -or $official) { - $asmVerCheckPath = "$RepoRoot\scripts" - Exec-Console $fsiPath """$asmVerCheckPath\AssemblyVersionCheck.fsx"" -- ""$ArtifactsDir""" + $dotnetPath = InitializeDotNetCli + $dotnetExe = Join-Path $dotnetPath "dotnet.exe" + Exec-Console $dotnetExe """$assemblyVerCheckPath"" ""$ArtifactsDir""" } } @@ -307,7 +308,7 @@ try { } if ($build) { - VerifyAssemblyVersions + VerifyAssemblyVersionsAndSymbols } $desktopTargetFramework = "net472" diff --git a/eng/build-utils.ps1 b/eng/build-utils.ps1 index fcda1a58e26..772de110ca2 100644 --- a/eng/build-utils.ps1 +++ b/eng/build-utils.ps1 @@ -236,10 +236,11 @@ function Make-BootstrapBuild() { Remove-Item -re $dir -ErrorAction SilentlyContinue Create-Directory $dir - # prepare FsLex and Fsyacc + # prepare FsLex and Fsyacc and AssemblyCheck Run-MSBuild "$RepoRoot\src\buildtools\buildtools.proj" "/restore /t:Publish" -logFileName "BuildTools" -configuration $bootstrapConfiguration Copy-Item "$ArtifactsDir\bin\fslex\$bootstrapConfiguration\netcoreapp2.1\publish" -Destination "$dir\fslex" -Force -Recurse Copy-Item "$ArtifactsDir\bin\fsyacc\$bootstrapConfiguration\netcoreapp2.1\publish" -Destination "$dir\fsyacc" -Force -Recurse + Copy-Item "$ArtifactsDir\bin\AssemblyCheck\$bootstrapConfiguration\netcoreapp2.1\publish" -Destination "$dir\AssemblyCheck" -Force -Recurse # prepare compiler $projectPath = "$RepoRoot\proto.proj" diff --git a/src/absil/ilwritepdb.fs b/src/absil/ilwritepdb.fs index c1ffd692249..c542f59eeaf 100644 --- a/src/absil/ilwritepdb.fs +++ b/src/absil/ilwritepdb.fs @@ -186,7 +186,7 @@ let pdbGetEmbeddedPdbDebugInfo (embeddedPdbChunk: BinaryChunk) (uncompressedLeng Buffer.BlockCopy(stream.ToArray(), 0, buffer, offset, size) buffer { iddCharacteristics = 0 // Reserved - iddMajorVersion = 0 // VersionMajor should be 0 + iddMajorVersion = 0x0100 // VersionMajor should be 0 iddMinorVersion = 0x0100 // VersionMinor should be 0x0100 iddType = 17 // IMAGE_DEBUG_TYPE_EMBEDDEDPDB iddTimestamp = 0 diff --git a/src/buildtools/AssemblyCheck/AssemblyCheck.fsproj b/src/buildtools/AssemblyCheck/AssemblyCheck.fsproj new file mode 100644 index 00000000000..f3f3d61e2cd --- /dev/null +++ b/src/buildtools/AssemblyCheck/AssemblyCheck.fsproj @@ -0,0 +1,17 @@ + + + + Exe + netcoreapp2.1 + true + + + + + + + + + + + diff --git a/scripts/AssemblyVersionCheck.fsx b/src/buildtools/AssemblyCheck/Program.fs similarity index 51% rename from scripts/AssemblyVersionCheck.fsx rename to src/buildtools/AssemblyCheck/Program.fs index 0f3816a2e6c..7b90cb11cab 100644 --- a/scripts/AssemblyVersionCheck.fsx +++ b/src/buildtools/AssemblyCheck/Program.fs @@ -1,9 +1,10 @@ -// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. open System open System.Diagnostics open System.IO open System.Reflection +open System.Reflection.PortableExecutable open System.Text.RegularExpressions module AssemblyVersionCheck = @@ -13,10 +14,37 @@ module AssemblyVersionCheck = let private commitHashPattern = new Regex(@"Commit Hash: ()|([0-9a-fA-F]{40})", RegexOptions.Compiled) let private devVersionPattern = new Regex(@"-(ci|dev)", RegexOptions.Compiled) - let verifyAssemblyVersions (binariesPath:string) = + let verifyEmbeddedPdb (filename:string) = + use fileStream = File.OpenRead(filename) + let reader = new PEReader(fileStream) + let mutable hasEmbeddedPdb = false + + try + for entry in reader.ReadDebugDirectory() do + match entry.Type with + | DebugDirectoryEntryType.CodeView -> + let _ = reader.ReadCodeViewDebugDirectoryData(entry) + () + + | DebugDirectoryEntryType.EmbeddedPortablePdb -> + let _ = reader.ReadEmbeddedPortablePdbDebugDirectoryData(entry) + hasEmbeddedPdb <- true + () + + | DebugDirectoryEntryType.PdbChecksum -> + let _ = reader.ReadPdbChecksumDebugDirectoryData(entry) + () + + | _ -> () + with | e -> printfn "Error validating assembly %s\nMessage: %s" filename (e.ToString()) + hasEmbeddedPdb + + let verifyAssemblies (binariesPath:string) = + let excludedAssemblies = [ "FSharp.Data.TypeProviders.dll" ] |> Set.ofList + let fsharpAssemblies = [ "FSharp*.dll" "fsc.exe" @@ -27,6 +55,11 @@ module AssemblyVersionCheck = |> Seq.concat |> List.ofSeq |> List.filter (fun p -> (Set.contains (Path.GetFileName(p)) excludedAssemblies) |> not) + |> List.filter (fun p -> not (p.Contains(@"\Proto\") || p.Contains(@"\Bootstrap\") || p.Contains(@".resources."))) + + let fsharpExecutingWithEmbeddedPdbs = + fsharpAssemblies + |> List.filter (fun p -> not (p.Contains(@"\Proto\") || p.Contains(@"\Bootstrap\") || p.Contains(@".resources."))) // verify that all assemblies have a version number other than 0.0.0.0 or 1.0.0.0 let failedVersionCheck = @@ -34,6 +67,7 @@ module AssemblyVersionCheck = |> List.filter (fun a -> let assemblyVersion = AssemblyName.GetAssemblyName(a).Version assemblyVersion = versionZero || assemblyVersion = versionOne) + if failedVersionCheck.Length > 0 then printfn "The following assemblies had a version of %A or %A" versionZero versionOne printfn "%s\r\n" <| String.Join("\r\n", failedVersionCheck) @@ -46,24 +80,32 @@ module AssemblyVersionCheck = |> List.filter (fun a -> let fileProductVersion = FileVersionInfo.GetVersionInfo(a).ProductVersion not (commitHashPattern.IsMatch(fileProductVersion) || devVersionPattern.IsMatch(fileProductVersion))) + if failedCommitHash.Length > 0 then printfn "The following assemblies don't have a commit hash set" printfn "%s\r\n" <| String.Join("\r\n", failedCommitHash) else printfn "All shipping assemblies had an appropriate commit hash." + // verify that all assemblies have a commit hash + let failedVerifyEmbeddedPdb = + fsharpExecutingWithEmbeddedPdbs + |> List.filter (fun a -> not (verifyEmbeddedPdb a)) + + if failedVerifyEmbeddedPdb.Length > 0 then + printfn "The following assemblies don't have an embedded pdb" + printfn "%s\r\n" <| String.Join("\r\n", failedVerifyEmbeddedPdb) + else + printfn "All shipping assemblies had an embedded PDB." + // return code is the number of failures - failedVersionCheck.Length + failedCommitHash.Length + failedVersionCheck.Length + failedCommitHash.Length + failedVerifyEmbeddedPdb.Length + +[] let main (argv:string array) = if argv.Length <> 1 then - printfn "Usage: fsi.exe AssemblyVersionCheck.fsx -- path/to/binaries" + printfn "Usage: dotnet AssemblyVersionCheck.dll -- path/to/binaries" 1 else - AssemblyVersionCheck.verifyAssemblyVersions argv.[0] - -Environment.GetCommandLineArgs() -|> Seq.skipWhile ((<>) "--") -|> Seq.skip 1 -|> Array.ofSeq -|> main + AssemblyVersionCheck.verifyAssemblies argv.[0] diff --git a/src/buildtools/buildtools.proj b/src/buildtools/buildtools.proj index 630bb678561..7ac48ba2a37 100644 --- a/src/buildtools/buildtools.proj +++ b/src/buildtools/buildtools.proj @@ -8,6 +8,7 @@ + diff --git a/src/fsharp/FSharp.Core.nuget/FSharp.Core.nuspec b/src/fsharp/FSharp.Core.nuget/FSharp.Core.nuspec index 750052c45db..ee7a88b29d6 100644 --- a/src/fsharp/FSharp.Core.nuget/FSharp.Core.nuspec +++ b/src/fsharp/FSharp.Core.nuget/FSharp.Core.nuspec @@ -38,13 +38,11 @@ - - From 8ad1412e02ba037d3a2585aabb3907836604752f Mon Sep 17 00:00:00 2001 From: Kevin Ransom Date: Sat, 20 Jul 2019 03:22:14 -0700 Subject: [PATCH 02/10] Fix comment --- src/buildtools/AssemblyCheck/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/buildtools/AssemblyCheck/Program.fs b/src/buildtools/AssemblyCheck/Program.fs index 7b90cb11cab..9f1bd80688f 100644 --- a/src/buildtools/AssemblyCheck/Program.fs +++ b/src/buildtools/AssemblyCheck/Program.fs @@ -87,7 +87,7 @@ module AssemblyVersionCheck = else printfn "All shipping assemblies had an appropriate commit hash." - // verify that all assemblies have a commit hash + // verify that all assemblies have an embedded pdb let failedVerifyEmbeddedPdb = fsharpExecutingWithEmbeddedPdbs |> List.filter (fun a -> not (verifyEmbeddedPdb a)) From ea3ef0b34d482c82955367234377058f3e5dd7f9 Mon Sep 17 00:00:00 2001 From: Kevin Ransom Date: Sat, 20 Jul 2019 09:49:34 -0700 Subject: [PATCH 03/10] Forgot to remove pdbs from fsharp.compiler.nuget --- .../Microsoft.FSharp.Compiler.nuspec | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/fsharp/FSharp.Compiler.nuget/Microsoft.FSharp.Compiler.nuspec b/src/fsharp/FSharp.Compiler.nuget/Microsoft.FSharp.Compiler.nuspec index d3756ebc392..a82fbb1ac3e 100644 --- a/src/fsharp/FSharp.Compiler.nuget/Microsoft.FSharp.Compiler.nuspec +++ b/src/fsharp/FSharp.Compiler.nuget/Microsoft.FSharp.Compiler.nuspec @@ -49,15 +49,7 @@ - - - - - - - + target="lib\netcoreapp2.1" /> @@ -69,9 +61,9 @@ + target="lib\netcoreapp2.1" /> + target="lib\netcoreapp2.1" /> From 6ee93ea6baa10f9a40bb2d061c4b86909b18cda8 Mon Sep 17 00:00:00 2001 From: Kevin Ransom Date: Sat, 20 Jul 2019 11:35:11 -0700 Subject: [PATCH 04/10] Rename Program.fs and exclude FSharpSdk from checks --- .../AssemblyCheck/{Program.fs => AssemblyCheck.fs} | 9 ++++----- src/buildtools/AssemblyCheck/AssemblyCheck.fsproj | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) rename src/buildtools/AssemblyCheck/{Program.fs => AssemblyCheck.fs} (92%) diff --git a/src/buildtools/AssemblyCheck/Program.fs b/src/buildtools/AssemblyCheck/AssemblyCheck.fs similarity index 92% rename from src/buildtools/AssemblyCheck/Program.fs rename to src/buildtools/AssemblyCheck/AssemblyCheck.fs index 9f1bd80688f..5c9ef88f448 100644 --- a/src/buildtools/AssemblyCheck/Program.fs +++ b/src/buildtools/AssemblyCheck/AssemblyCheck.fs @@ -7,7 +7,7 @@ open System.Reflection open System.Reflection.PortableExecutable open System.Text.RegularExpressions -module AssemblyVersionCheck = +module AssemblyCheck = let private versionZero = Version(0, 0, 0, 0) let private versionOne = Version(1, 0, 0, 0) @@ -55,11 +55,10 @@ module AssemblyVersionCheck = |> Seq.concat |> List.ofSeq |> List.filter (fun p -> (Set.contains (Path.GetFileName(p)) excludedAssemblies) |> not) - |> List.filter (fun p -> not (p.Contains(@"\Proto\") || p.Contains(@"\Bootstrap\") || p.Contains(@".resources."))) let fsharpExecutingWithEmbeddedPdbs = fsharpAssemblies - |> List.filter (fun p -> not (p.Contains(@"\Proto\") || p.Contains(@"\Bootstrap\") || p.Contains(@".resources."))) + |> List.filter (fun p -> not (p.Contains(@"\Proto\") || p.Contains(@"\Bootstrap\") || p.Contains(@".resources.") || p.Contains(@"\FSharpSdk\"))) // verify that all assemblies have a version number other than 0.0.0.0 or 1.0.0.0 let failedVersionCheck = @@ -105,7 +104,7 @@ module AssemblyVersionCheck = [] let main (argv:string array) = if argv.Length <> 1 then - printfn "Usage: dotnet AssemblyVersionCheck.dll -- path/to/binaries" + printfn "Usage: dotnet AssemblyCheck.dll -- path/to/binaries" 1 else - AssemblyVersionCheck.verifyAssemblies argv.[0] + AssemblyCheck.verifyAssemblies argv.[0] diff --git a/src/buildtools/AssemblyCheck/AssemblyCheck.fsproj b/src/buildtools/AssemblyCheck/AssemblyCheck.fsproj index f3f3d61e2cd..72f79d02f93 100644 --- a/src/buildtools/AssemblyCheck/AssemblyCheck.fsproj +++ b/src/buildtools/AssemblyCheck/AssemblyCheck.fsproj @@ -7,7 +7,7 @@ - + From e724ecc4d46de5b25ef25b05fe74497637b5b150 Mon Sep 17 00:00:00 2001 From: Kevin Ransom Date: Sun, 21 Jul 2019 11:57:18 -0700 Subject: [PATCH 05/10] Embedded doesn't need to verify tmp or obj directories. --- src/buildtools/AssemblyCheck/AssemblyCheck.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/buildtools/AssemblyCheck/AssemblyCheck.fs b/src/buildtools/AssemblyCheck/AssemblyCheck.fs index 5c9ef88f448..8182fd53788 100644 --- a/src/buildtools/AssemblyCheck/AssemblyCheck.fs +++ b/src/buildtools/AssemblyCheck/AssemblyCheck.fs @@ -58,7 +58,7 @@ module AssemblyCheck = let fsharpExecutingWithEmbeddedPdbs = fsharpAssemblies - |> List.filter (fun p -> not (p.Contains(@"\Proto\") || p.Contains(@"\Bootstrap\") || p.Contains(@".resources.") || p.Contains(@"\FSharpSdk\"))) + |> List.filter (fun p -> not (p.Contains(@"\Proto\") || p.Contains(@"\Bootstrap\") || p.Contains(@".resources.") || p.Contains(@"\FSharpSdk\" || p.Contains(@"\tmp\" || p.Contains(@"\obj\"))) // verify that all assemblies have a version number other than 0.0.0.0 or 1.0.0.0 let failedVersionCheck = From 98947d8ab13a988d28f9985fff0534e185364352 Mon Sep 17 00:00:00 2001 From: Kevin Ransom Date: Sun, 21 Jul 2019 12:10:49 -0700 Subject: [PATCH 06/10] typo --- src/buildtools/AssemblyCheck/AssemblyCheck.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/buildtools/AssemblyCheck/AssemblyCheck.fs b/src/buildtools/AssemblyCheck/AssemblyCheck.fs index 8182fd53788..8658499b9c6 100644 --- a/src/buildtools/AssemblyCheck/AssemblyCheck.fs +++ b/src/buildtools/AssemblyCheck/AssemblyCheck.fs @@ -58,7 +58,7 @@ module AssemblyCheck = let fsharpExecutingWithEmbeddedPdbs = fsharpAssemblies - |> List.filter (fun p -> not (p.Contains(@"\Proto\") || p.Contains(@"\Bootstrap\") || p.Contains(@".resources.") || p.Contains(@"\FSharpSdk\" || p.Contains(@"\tmp\" || p.Contains(@"\obj\"))) + |> List.filter (fun p -> not (p.Contains(@"\Proto\") || p.Contains(@"\Bootstrap\") || p.Contains(@".resources.") || p.Contains(@"\FSharpSdk\") || p.Contains(@"\tmp\" || p.Contains(@"\obj\"))) // verify that all assemblies have a version number other than 0.0.0.0 or 1.0.0.0 let failedVersionCheck = From 5b1a359e5583be64182661e81bdb9e560e1c1920 Mon Sep 17 00:00:00 2001 From: Kevin Ransom Date: Sun, 21 Jul 2019 13:22:43 -0700 Subject: [PATCH 07/10] typo --- src/buildtools/AssemblyCheck/AssemblyCheck.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/buildtools/AssemblyCheck/AssemblyCheck.fs b/src/buildtools/AssemblyCheck/AssemblyCheck.fs index 8658499b9c6..848b06e403c 100644 --- a/src/buildtools/AssemblyCheck/AssemblyCheck.fs +++ b/src/buildtools/AssemblyCheck/AssemblyCheck.fs @@ -58,7 +58,7 @@ module AssemblyCheck = let fsharpExecutingWithEmbeddedPdbs = fsharpAssemblies - |> List.filter (fun p -> not (p.Contains(@"\Proto\") || p.Contains(@"\Bootstrap\") || p.Contains(@".resources.") || p.Contains(@"\FSharpSdk\") || p.Contains(@"\tmp\" || p.Contains(@"\obj\"))) + |> List.filter (fun p -> not (p.Contains(@"\Proto\") || p.Contains(@"\Bootstrap\") || p.Contains(@".resources.") || p.Contains(@"\FSharpSdk\") || p.Contains(@"\tmp\") || p.Contains(@"\obj\"))) // verify that all assemblies have a version number other than 0.0.0.0 or 1.0.0.0 let failedVersionCheck = From 831c1209a76bbd5919d191276ac87a7c0eaa3dba Mon Sep 17 00:00:00 2001 From: Kevin Ransom Date: Sun, 21 Jul 2019 15:25:40 -0700 Subject: [PATCH 08/10] Don't check FSharpSdk for hash --- src/buildtools/AssemblyCheck/AssemblyCheck.fs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/buildtools/AssemblyCheck/AssemblyCheck.fs b/src/buildtools/AssemblyCheck/AssemblyCheck.fs index 848b06e403c..c6bd035a673 100644 --- a/src/buildtools/AssemblyCheck/AssemblyCheck.fs +++ b/src/buildtools/AssemblyCheck/AssemblyCheck.fs @@ -76,6 +76,7 @@ module AssemblyCheck = // verify that all assemblies have a commit hash let failedCommitHash = fsharpAssemblies + |> List.filter (fun p -> not (p.Contains(@"\FSharpSdk\"))) |> List.filter (fun a -> let fileProductVersion = FileVersionInfo.GetVersionInfo(a).ProductVersion not (commitHashPattern.IsMatch(fileProductVersion) || devVersionPattern.IsMatch(fileProductVersion))) From 829fcf242089015b8c8b3c58149935e83ea3395b Mon Sep 17 00:00:00 2001 From: Kevin Ransom Date: Mon, 22 Jul 2019 12:42:41 -0700 Subject: [PATCH 09/10] Make comment match code --- src/absil/ilwritepdb.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/absil/ilwritepdb.fs b/src/absil/ilwritepdb.fs index c542f59eeaf..86b7ee9224f 100644 --- a/src/absil/ilwritepdb.fs +++ b/src/absil/ilwritepdb.fs @@ -186,7 +186,7 @@ let pdbGetEmbeddedPdbDebugInfo (embeddedPdbChunk: BinaryChunk) (uncompressedLeng Buffer.BlockCopy(stream.ToArray(), 0, buffer, offset, size) buffer { iddCharacteristics = 0 // Reserved - iddMajorVersion = 0x0100 // VersionMajor should be 0 + iddMajorVersion = 0x0100 // VersionMajor should be 0x0100 iddMinorVersion = 0x0100 // VersionMinor should be 0x0100 iddType = 17 // IMAGE_DEBUG_TYPE_EMBEDDEDPDB iddTimestamp = 0 From 41f3640f312aa64e8a44c8f0f742d557a9ea86ac Mon Sep 17 00:00:00 2001 From: Kevin Ransom Date: Wed, 24 Jul 2019 11:59:05 -0700 Subject: [PATCH 10/10] Empty commit to force rebuild