Merge fsharp47 + dev16.3 into Dim interop

.vsts-signed.yaml

@@ -105,6 +105,14 @@ jobs:
     continueOnError: true
     condition: succeeded()
 
+  # Publish native PDBs for archiving
+  - task: PublishBuildArtifacts@1
+    displayName: Publish Artifact Symbols
+    inputs:
+      PathtoPublish: '$(Build.SourcesDirectory)/artifacts/SymStore/$(BuildConfiguration)'
+      ArtifactName: NativeSymbols
+    condition: succeeded()
+
   # Execute cleanup tasks
   - task: ms-vseng.MicroBuildTasks.521a94ea-9e68-468a-8167-6dcf361ea776.MicroBuildCleanup@1
     displayName: Execute cleanup tasks

FSharpBuild.Directory.Build.props

@@ -75,6 +75,7 @@
     <RepositoryUrl Condition="'$(RepositoryUrl)' == ''">https://github.com/Microsoft/visualfsharp</RepositoryUrl>
     <RepositoryType Condition="'$(RepositoryType)' == ''">git</RepositoryType>
   </PropertyGroup>
+
   <PropertyGroup Condition="'$(FSharpSourceBuild)' == 'true' AND '$(RepositoryCommit)' == ''">
     <_DotGitDir>$(RepoRoot).git</_DotGitDir>
     <_HeadFileContent Condition="Exists('$(_DotGitDir)/HEAD')">$([System.IO.File]::ReadAllText('$(_DotGitDir)/HEAD').Trim())</_HeadFileContent>
@@ -87,7 +88,7 @@
   <PropertyGroup>
     <NoWarn Condition="'$(Language)' == 'F#'">$(NoWarn);FS2003</NoWarn><!-- warning when AssemblyInformationalVersion looks like '1.2.3-dev' -->
     <NoCompilerStandardLib>true</NoCompilerStandardLib><!-- necessary for resource generation using csc.exe -->
-    <DebugType>portable</DebugType>
+    <DebugType>embedded</DebugType>
     <MicroBuildAssemblyFileLanguage>fs</MicroBuildAssemblyFileLanguage>
     <UseStandardResourceNames>false</UseStandardResourceNames>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>

NuGet.config

@@ -8,6 +8,28 @@
     <clear />
     <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+    <add key="fsharp-daily" value="https://www.myget.org/F/fsharp-daily/api/v3/index.json" />
+    <add key="roslyn-master-nightly" value="https://dotnet.myget.org/F/roslyn-master-nightly/api/v3/index.json" />
+    <add key="myget-dotnet-core" value="https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" />
+    <add key="dotnet-buildtools" value="https://dotnet.myget.org/F/dotnet-buildtools/api/v3/index.json" />
+    <add key="roslyn-tools" value="https://dotnet.myget.org/F/roslyn-tools/api/v3/index.json" />
+    <add key="roslyn" value="https://dotnet.myget.org/F/roslyn/api/v3/index.json" />
+    <add key="symreader-converter" value="https://dotnet.myget.org/F/symreader-converter/api/v3/index.json" />
+    <add key="interactive-window" value="https://dotnet.myget.org/F/interactive-window/api/v3/index.json" />
+    <add key="vs-devcore" value="https://myget.org/F/vs-devcore/api/v3/index.json" />
+    <add key="vs-editor" value="https://myget.org/F/vs-editor/api/v3/index.json" />
+    <add key="vssdk" value="https://vside.myget.org/F/vssdk/api/v3/index.json" />
+    <add key="vs-impl" value="https://vside.myget.org/F/vs-impl/api/v3/index.json" />
+    <add key="roslyn_concord" value="https://myget.org/F/roslyn_concord/api/v3/index.json" />
+    <add key="devcore" value="https://vside.myget.org/F/devcore/api/v3/index.json" />
+
+    <add key="dotnet-windowsdesktop" value="https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json" />
+    <add key="aspnet-aspnetcore" value="https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json" />
+    <add key="aspnet-aspnetcore-tooling" value="https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json" />
+    <add key="aspnet-entityframeworkcore" value="https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json" />
+    <add key="aspnet-extensions" value="https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json" />
+    <add key="gRPC repository" value="https://grpc.jfrog.io/grpc/api/nuget/v3/grpc-nuget-dev" />
+
   </packageSources>
   <disabledPackageSources>
     <clear />

azure-pipelines.yml

@@ -113,6 +113,12 @@ jobs:
             PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpFull.vsix'
             ArtifactName: 'Nightly'
           condition: succeeded()
+        - task: PublishBuildArtifacts@1
+          displayName: Publish Artifact Symbols
+          inputs:
+            PathtoPublish: '$(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig)'
+            ArtifactName: 'NativeSymbols'
+          condition: succeeded()
 
 #---------------------------------------------------------------------------------------------------------------------#
 #                                                     PR builds                                                       #

eng/Build.ps1

@@ -225,13 +225,14 @@ function UpdatePath() {
     TestAndAddToPath "$ArtifactsDir\bin\fsiAnyCpu\$configuration\net472"
 }
 
-function VerifyAssemblyVersions() {
-    $fsiPath = Join-Path $ArtifactsDir "bin\fsi\Proto\net472\publish\fsi.exe"
+function VerifyAssemblyVersionsAndSymbols() {
+    $assemblyVerCheckPath = Join-Path $ArtifactsDir "Bootstrap\AssemblyCheck\AssemblyCheck.dll"
 
     # Only verify versions on CI or official build
     if ($ci -or $official) {
-        $asmVerCheckPath = "$RepoRoot\scripts"
-        Exec-Console $fsiPath """$asmVerCheckPath\AssemblyVersionCheck.fsx"" -- ""$ArtifactsDir"""
+        $dotnetPath = InitializeDotNetCli
+        $dotnetExe = Join-Path $dotnetPath "dotnet.exe"
+        Exec-Console $dotnetExe """$assemblyVerCheckPath"" ""$ArtifactsDir"""
     }
 }
 
@@ -329,7 +330,7 @@ try {
     }
 
     if ($build) {
-        VerifyAssemblyVersions
+        VerifyAssemblyVersionsAndSymbols
     }
 
     $desktopTargetFramework = "net472"

eng/Version.Details.xml

@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19359.6">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19378.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>0f5dd7680174620f31c9a00cdb2ac0b0e70e631f</Sha>
+      <Sha>a8e982d3bac01d8f4f91a4c57191147570079448</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/build-utils.ps1

@@ -236,10 +236,11 @@ function Make-BootstrapBuild() {
     Remove-Item -re $dir -ErrorAction SilentlyContinue
     Create-Directory $dir
 
-    # prepare FsLex and Fsyacc
+    # prepare FsLex and Fsyacc and AssemblyCheck
     Run-MSBuild "$RepoRoot\src\buildtools\buildtools.proj" "/restore /t:Publish" -logFileName "BuildTools" -configuration $bootstrapConfiguration
     Copy-Item "$ArtifactsDir\bin\fslex\$bootstrapConfiguration\netcoreapp2.1\publish" -Destination "$dir\fslex" -Force -Recurse
     Copy-Item "$ArtifactsDir\bin\fsyacc\$bootstrapConfiguration\netcoreapp2.1\publish" -Destination "$dir\fsyacc" -Force  -Recurse
+    Copy-Item "$ArtifactsDir\bin\AssemblyCheck\$bootstrapConfiguration\netcoreapp2.1\publish" -Destination "$dir\AssemblyCheck" -Force  -Recurse
 
     # prepare compiler
     $projectPath = "$RepoRoot\proto.proj"

eng/common/SigningValidation.proj

@@ -3,7 +3,7 @@
   <!--
     This MSBuild file is intended to be used as the body of the default 
     publishing release pipeline. The release pipeline will use this file
-    to invoke the the SignCheck tool to validate that packages about to
+    to invoke the SignCheck tool to validate that packages about to
     be published are correctly signed.
 
     Parameters:

eng/common/init-tools-native.ps1

@@ -98,12 +98,18 @@ try {
       }
 
       Write-Verbose "Installing $ToolName version $ToolVersion"
-      Write-Verbose "Executing '$InstallerPath $LocalInstallerArguments'"
+      Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'"
       & $InstallerPath @LocalInstallerArguments
       if ($LASTEXITCODE -Ne "0") {
         $errMsg = "$ToolName installation failed"
         if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) {
-            Write-Warning $errMsg
+            $showNativeToolsWarning = $true
+            if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) {
+                $showNativeToolsWarning = $false
+            }
+            if ($showNativeToolsWarning) {
+                Write-Warning $errMsg
+            }
             $toolInstallationFailure = $true
         } else {
             Write-Error $errMsg

eng/common/init-tools-native.sh

@@ -71,7 +71,7 @@ function ReadGlobalJsonNativeTools {
   local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
   native_tools_list=${native_tools_list//[\" ]/}
   native_tools_list=${native_tools_list//,/$'\n'}
-  native_tools_list="$(echo -e "${native_tools_list}" | tr -d '[:space:]')"
+  native_tools_list="$(echo -e "${native_tools_list}" | tr -d '[[:space:]]')"
 
   local old_IFS=$IFS
   while read -r line; do

eng/common/native/CommonLibrary.psm1

@@ -59,9 +59,38 @@ function DownloadAndExtract {
                                           -Verbose:$Verbose
 
   if ($UnzipStatus -Eq $False) {
-    Write-Error "Unzip failed"
-    return $False
+    # Retry Download one more time with Force=true
+    $DownloadRetryStatus = CommonLibrary\Get-File -Uri $Uri `
+                                             -Path $TempToolPath `
+                                             -DownloadRetries 1 `
+                                             -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
+                                             -Force:$True `
+                                             -Verbose:$Verbose
+
+    if ($DownloadRetryStatus -Eq $False) {
+      Write-Error "Last attempt of download failed as well"
+      return $False
+    }
+
+    # Retry unzip again one more time with Force=true
+    $UnzipRetryStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath `
+                                            -OutputDirectory $InstallDirectory `
+                                            -Force:$True `
+                                            -Verbose:$Verbose
+    if ($UnzipRetryStatus -Eq $False)
+    {
+      Write-Error "Last attempt of unzip failed as well"
+      # Clean up partial zips and extracts
+      if (Test-Path $TempToolPath) {
+        Remove-Item $TempToolPath -Force
+      }
+      if (Test-Path $InstallDirectory) {
+        Remove-Item $InstallDirectory -Force -Recurse
+      }
+      return $False
+    }
   }
+
   return $True
 }
 

eng/common/performance/perfhelixpublish.proj

@@ -0,0 +1,77 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+
+  <PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
+    <WorkItemCommand>%HELIX_CORRELATION_PAYLOAD%\performance\scripts\benchmarks_ci.py --csproj %HELIX_CORRELATION_PAYLOAD%\performance\$(TargetCsproj)</WorkItemCommand>
+    <CliArguments>--dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP%</CliArguments>
+    <Python>py -3</Python>
+    <CoreRun>%HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe</CoreRun>
+    <HelixPreCommands>$(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd</HelixPreCommands>
+    <ArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts</ArtifactsDirectory>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT' and '$(RunFromPerfRepo)' == 'false'">
+    <BaseDirectory>$HELIX_CORRELATION_PAYLOAD</BaseDirectory>
+    <PerformanceDirectory>$(BaseDirectory)/performance</PerformanceDirectory>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT' and '$(RunFromPerfRepo)' == 'true'">
+    <BaseDirectory>$HELIX_WORKITEM_PAYLOAD</BaseDirectory>
+    <PerformanceDirectory>$(BaseDirectory)</PerformanceDirectory>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
+    <WorkItemCommand>$(PerformanceDirectory)/scripts/benchmarks_ci.py --csproj $(PerformanceDirectory)/$(TargetCsproj)</WorkItemCommand>
+    <CliArguments>--dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP</CliArguments>
+    <Python>python3</Python>
+    <CoreRun>$(BaseDirectory)/Core_Root/corerun</CoreRun>
+    <HelixPreCommands>$(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh</HelixPreCommands>
+    <ArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts</ArtifactsDirectory>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(UseCoreRun)' == 'true'">
+    <CoreRunArgument>--corerun $(CoreRun)</CoreRunArgument>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(WorkItemCommand)' != ''">
+    <WorkItemCommand>$(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments)</WorkItemCommand>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(_Framework)' != 'net461'">
+    <WorkItemCommand>$(WorkItemCommand) $(CliArguments)</WorkItemCommand>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
+      <PayloadDirectory>%(Identity)</PayloadDirectory>
+    </HelixCorrelationPayload>
+  </ItemGroup>
+
+  <PropertyGroup>
+    <PartitionCount>5</PartitionCount>
+  </PropertyGroup>
+  <ItemGroup>
+    <Partition Include="$(BuildConfig).Partition0" Index="0" />
+    <Partition Include="$(BuildConfig).Partition1" Index="1" />
+    <Partition Include="$(BuildConfig).Partition2" Index="2" />
+    <Partition Include="$(BuildConfig).Partition3" Index="3" />
+    <Partition Include="$(BuildConfig).Partition4" Index="4" />
+  </ItemGroup>
+
+  <!-- 
+    Partition the Microbenchmarks project, but nothing else
+  -->
+  <ItemGroup Condition="$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
+    <HelixWorkItem Include="@(Partition)">
+      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <Command>$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
+      <Timeout>4:00</Timeout>
+    </HelixWorkItem>
+  </ItemGroup>
+  <ItemGroup Condition="!$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
+    <HelixWorkItem Include="$(BuildConfig).WorkItem">
+      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <Command>$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)"</Command>
+      <Timeout>4:00</Timeout>
+    </HelixWorkItem>
+  </ItemGroup>
+</Project>

eng/common/performance/performance-setup.ps1

@@ -0,0 +1,91 @@
+Param(
+    [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,
+    [string] $CoreRootDirectory,
+    [string] $Architecture="x64",
+    [string] $Framework="netcoreapp3.0",
+    [string] $CompilationMode="Tiered",
+    [string] $Repository=$env:BUILD_REPOSITORY_NAME,
+    [string] $Branch=$env:BUILD_SOURCEBRANCH,
+    [string] $CommitSha=$env:BUILD_SOURCEVERSION,
+    [string] $BuildNumber=$env:BUILD_BUILDNUMBER,
+    [string] $RunCategories="coreclr corefx",
+    [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj",
+    [string] $Kind="micro",
+    [switch] $Internal,
+    [string] $Configurations="CompilationMode=$CompilationMode"
+)
+
+$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance")
+$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty)
+
+$PayloadDirectory = (Join-Path $SourceDirectory "Payload")
+$PerformanceDirectory = (Join-Path $PayloadDirectory "performance")
+$WorkItemDirectory = (Join-Path $SourceDirectory "workitem")
+$ExtraBenchmarkDotNetArguments = "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true"
+$Creator = $env:BUILD_DEFINITIONNAME
+$PerfLabArguments = ""
+$HelixSourcePrefix = "pr"
+
+$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
+
+if ($Framework.StartsWith("netcoreapp")) {
+    $Queue = "Windows.10.Amd64.ClientRS4.Open"
+}
+
+if ($Internal) {
+    $Queue = "Windows.10.Amd64.ClientRS5.Perf"
+    $PerfLabArguments = "--upload-to-perflab-container"
+    $ExtraBenchmarkDotNetArguments = ""
+    $Creator = ""
+    $HelixSourcePrefix = "official"
+}
+
+$CommonSetupArguments="--frameworks $Framework --queue $Queue --build-number $BuildNumber --build-configs $Configurations"
+$SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments"
+
+if ($RunFromPerformanceRepo) {
+    $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments"
+
+    robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git
+}
+else {
+    git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory
+}
+
+if ($UseCoreRun) {
+    $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root")
+    Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot
+}
+
+$DocsDir = (Join-Path $PerformanceDirectory "docs")
+robocopy $DocsDir $WorkItemDirectory
+
+# Set variables that we will need to have in future steps
+$ci = $true
+
+. "$PSScriptRoot\..\pipeline-logging-functions.ps1"
+
+# Directories
+Write-PipelineSetVariable -Name 'PayloadDirectory' -Value "$PayloadDirectory" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'PerformanceDirectory' -Value "$PerformanceDirectory" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'WorkItemDirectory' -Value "$WorkItemDirectory" -IsMultiJobVariable $false
+
+# Script Arguments
+Write-PipelineSetVariable -Name 'Python' -Value "py -3" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'ExtraBenchmarkDotNetArguments' -Value "$ExtraBenchmarkDotNetArguments" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'SetupArguments' -Value "$SetupArguments" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'PerfLabArguments' -Value "$PerfLabArguments" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'BDNCategories' -Value "$RunCategories" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false
+
+# Helix Arguments
+Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'Queue' -Value "$Queue" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'HelixSourcePrefix' -Value "$HelixSourcePrefix" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name '_BuildConfig' -Value "$Architecture.$Kind.$Framework" -IsMultiJobVariable $false
+
+exit 0