From a61cf3285ab8925e62487982d9f60cd2d21ed979 Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Sat, 18 Apr 2020 14:37:50 -0700 Subject: [PATCH 01/11] Make `dotnet msbuild` the default on Windows too - add step using desktop `msbuild` when native builds may be involved - `-All` (without `-NoBuildNative`), `-BuildNative` or `-BuildInstallers` run this step - but `-ForceCoreMsbuild` unconditionally skips this step nits: - add binary log for RepoTasks build if `$BinaryLog` (echoes the `dotnet msbuild` command) - add blank lines between build steps --- build.ps1 | 81 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 26 deletions(-) diff --git a/build.ps1 b/build.ps1 index cf5b9e0655bc..06fea4c768cf 100644 --- a/build.ps1 +++ b/build.ps1 @@ -146,9 +146,8 @@ param( [switch]$NoBuildRepoTasks, - # By default, Windows builds will use MSBuild.exe. Passing this will force the build to run on - # dotnet.exe instead, which may cause issues if you invoke build on a project unsupported by - # MSBuild for .NET Core + # Disable pre-build of C++ code in x64 (default) and x86 builds. Affects -All and -Projects handling and causes + # -BuildInstallers and -BuildNative to be ignored. [switch]$ForceCoreMsbuild, # Diagnostics @@ -187,10 +186,6 @@ if ($DumpProcesses -or $CI) { } # Project selection -if ($All) { - $MSBuildArguments += '/p:BuildAllProjects=true' -} - if ($Projects) { if (![System.IO.Path]::IsPathRooted($Projects)) { @@ -227,20 +222,8 @@ if ($BuildManaged -or ($All -and (-not $NoBuildManaged))) { } } -if ($BuildInstallers) { $MSBuildArguments += "/p:BuildInstallers=true" } -if ($BuildManaged) { $MSBuildArguments += "/p:BuildManaged=true" } -if ($BuildNative) { $MSBuildArguments += "/p:BuildNative=true" } -if ($BuildNodeJS) { $MSBuildArguments += "/p:BuildNodeJS=true" } -if ($BuildJava) { $MSBuildArguments += "/p:BuildJava=true" } - if ($NoBuildDeps) { $MSBuildArguments += "/p:BuildProjectReferences=false" } -if ($NoBuildInstallers) { $MSBuildArguments += "/p:BuildInstallers=false" } -if ($NoBuildManaged) { $MSBuildArguments += "/p:BuildManaged=false" } -if ($NoBuildNative) { $MSBuildArguments += "/p:BuildNative=false" } -if ($NoBuildNodeJS) { $MSBuildArguments += "/p:BuildNodeJS=false" } -if ($NoBuildJava) { $MSBuildArguments += "/p:BuildJava=false" } - $RunBuild = if ($NoBuild) { $false } else { $true } # Run restore by default unless -NoRestore is set. @@ -276,6 +259,30 @@ if ($DotNetRuntimeSourceFeed -or $DotNetRuntimeSourceFeedKey) { $ToolsetBuildArguments += $runtimeFeedKeyArg } +# Split build categories between dotnet msbuild and desktop msbuild. Use desktop msbuild as little as possible. +[string[]]$dotnetBuildArguments = $MSBuildArguments +if ($All) { $dotnetBuildArguments += '/p:BuildAllProjects=true'; $BuildNative = $true } + +if ($NoBuildInstallers) { $MSBuildArguments += "/p:BuildInstallers=false"; $BuildInstallers = $false } +if ($BuildInstallers) { $MSBuildArguments += "/p:BuildInstallers=true" } +if ($NoBuildNative) { $MSBuildArguments += "/p:BuildNative=false"; $BuildNative = $false } +if ($BuildNative) { $MSBuildArguments += "/p:BuildNative=true"} + +if ($NoBuildJava) { $dotnetBuildArguments += "/p:BuildJava=false"; $BuildJava = $false } +if ($BuildJava) { $dotnetBuildArguments += "/p:BuildJava=true" } +if ($NoBuildManaged) { $dotnetBuildArguments += "/p:BuildManaged=false"; $BuildManaged = $false } +if ($BuildManaged) { $dotnetBuildArguments += "/p:BuildManaged=true" } +if ($NoBuildNodeJS) { $dotnetBuildArguments += "/p:BuildNodeJS=false"; $BuildNodeJS = $false } +if ($BuildNodeJS) { $dotnetBuildArguments += "/p:BuildNodeJS=true" } + +# Don't bother with two builds if just one will build everything. Ignore super-weird cases like +# "-Projects ... -NoBuildJava -NoBuildManaged -NoBuildNodeJS". +$ForceCoreMsbuild = $ForceCoreMsbuild -or -not ($BuildInstallers -or $BuildNative) -or ` + $Architecture.StartsWith("arm", [System.StringComparison]::OrdinalIgnoreCase) +$performDotnetBuild = $ForceCoreMsbuild -or $BuildJava -or $BuildManaged -or $BuildNodeJS -or ` + ($All -and -not ($NoBuildJava -and $NoBuildManaged -and $NoBuildNodeJS)) -or ` + ($Projects -and -not ($BuildInstallers -or $BuildNative)) + $foundJdk = $false $javac = Get-Command javac -ErrorAction Ignore -CommandType Application $localJdkPath = "$PSScriptRoot\.tools\jdk\win-x64\" @@ -343,9 +350,8 @@ $env:MSBUILDDISABLENODEREUSE=1 # Fixing this is tracked by https://github.com/dotnet/aspnetcore-internal/issues/601 $warnAsError = $false -if ($ForceCoreMsbuild) { - $msbuildEngine = 'dotnet' -} +# Use `dotnet msbuild` by default +$msbuildEngine = 'dotnet' # Ensure passing neither -bl nor -nobl on CI avoids errors in tools.ps1. This is needed because both parameters are # $false by default i.e. they always exist. (We currently avoid binary logs but that is made visible in the YAML.) @@ -367,7 +373,16 @@ Remove-Item variable:global:_MSBuildExe -ea Ignore if ($BinaryLog) { $bl = GetMSBuildBinaryLogCommandLineArgument($MSBuildArguments) if (-not $bl) { - $MSBuildArguments += "/bl:" + (Join-Path $LogDir "Build.binlog") + $dotnetBuildArguments += "/bl:" + (Join-Path $LogDir "Build.binlog") + $MSBuildArguments += "/bl:" + (Join-Path $LogDir "Build.native.binlog") + $ToolsetBuildArguments += "/bl:" + (Join-Path $LogDir "Build.repotasks.binlog") + } else { + # Use a different binary log path when running desktop msbuild if doing both builds. + if (-not $ForceCoreMsbuild -and $performDotnetBuild) { + $MSBuildArguments += [System.IO.Path]::ChangeExtension($bl, "native.binlog") + } + + $ToolsetBuildArguments += [System.IO.Path]::ChangeExtension($bl, "repotasks.binlog") } } elseif ($CI) { # Ensure the artifacts/log directory isn't empty to avoid warnings. @@ -394,6 +409,8 @@ try { } if (-not $NoBuildRepoTasks) { + Write-Host + MSBuild $toolsetBuildProj ` /p:RepoRoot=$RepoRoot ` /p:Projects=$EngRoot\tools\RepoTasks\RepoTasks.csproj ` @@ -404,9 +421,21 @@ try { @ToolsetBuildArguments } - MSBuild $toolsetBuildProj ` - /p:RepoRoot=$RepoRoot ` - @MSBuildArguments + if (-not $ForceCoreMsbuild) { + Write-Host + Remove-Item variable:global:_BuildTool -ErrorAction Ignore + $msbuildEngine = 'vs' + + MSBuild $toolsetBuildProj /p:RepoRoot=$RepoRoot @MSBuildArguments + } + + if ($performDotnetBuild) { + Write-Host + Remove-Item variable:global:_BuildTool -ErrorAction Ignore + $msbuildEngine = 'dotnet' + + MSBuild $toolsetBuildProj /p:RepoRoot=$RepoRoot @dotnetBuildArguments + } } catch { Write-Host $_.ScriptStackTrace From a5741d546d28d3a542b3a215ba296473f625e6d3 Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Wed, 20 May 2020 21:23:49 -0700 Subject: [PATCH 02/11] Adjust generation scripts to explicitly choose the MSBuild engine - ensure native assets are included in GenerateReferenceAssemblies.ps1 build - clean up the global state that tools.ps1 corrupts --- eng/scripts/GenerateProjectList.ps1 | 12 ++++++++++-- eng/scripts/GenerateReferenceAssemblies.ps1 | 20 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/eng/scripts/GenerateProjectList.ps1 b/eng/scripts/GenerateProjectList.ps1 index 47449b3d7ae8..11c5e2e296fb 100644 --- a/eng/scripts/GenerateProjectList.ps1 +++ b/eng/scripts/GenerateProjectList.ps1 @@ -1,10 +1,18 @@ param( [switch]$ci ) -$ErrorActionPreference = 'stop' +$ErrorActionPreference = 'stop' +$msbuildEngine = 'dotnet' $repoRoot = Resolve-Path "$PSScriptRoot/../.." -& "$repoRoot\eng\common\msbuild.ps1" -ci:$ci "$repoRoot/eng/CodeGen.proj" ` +try { + & "$repoRoot\eng\common\msbuild.ps1" -ci:$ci "$repoRoot/eng/CodeGen.proj" ` /t:GenerateProjectList ` /bl:artifacts/log/genprojlist.binlog +} finally { + Remove-Item variable:global:_BuildTool -ErrorAction Ignore + Remove-Item variable:global:_DotNetInstallDir -ErrorAction Ignore + Remove-Item variable:global:_ToolsetBuildProj -ErrorAction Ignore + Remove-Item variable:global:_MSBuildExe -ErrorAction Ignore +} diff --git a/eng/scripts/GenerateReferenceAssemblies.ps1 b/eng/scripts/GenerateReferenceAssemblies.ps1 index fa58025c34ed..e5a64b8bee2e 100644 --- a/eng/scripts/GenerateReferenceAssemblies.ps1 +++ b/eng/scripts/GenerateReferenceAssemblies.ps1 @@ -1,10 +1,26 @@ param( [switch]$ci ) -$ErrorActionPreference = 'stop' +$ErrorActionPreference = 'stop' $repoRoot = Resolve-Path "$PSScriptRoot/../.." -& "$repoRoot\eng\common\msbuild.ps1" -ci:$ci "$repoRoot/eng/CodeGen.proj" ` +try { + # eng/common/msbuild.ps1 builds the Debug configuration unless there's a $Configuration variable. Match that here. + & "$repoRoot\build.ps1" -ci:$ci -nobl -buildNative -configuration Debug + + Remove-Item variable:global:_BuildTool -ErrorAction Ignore + Remove-Item variable:global:_DotNetInstallDir -ErrorAction Ignore + Remove-Item variable:global:_ToolsetBuildProj -ErrorAction Ignore + Remove-Item variable:global:_MSBuildExe -ErrorAction Ignore + + $msbuildEngine = 'dotnet' + & "$repoRoot\eng\common\msbuild.ps1" -ci:$ci "$repoRoot/eng/CodeGen.proj" ` /t:GenerateReferenceSources ` /bl:artifacts/log/genrefassemblies.binlog +} finally { + Remove-Item variable:global:_BuildTool -ErrorAction Ignore + Remove-Item variable:global:_DotNetInstallDir -ErrorAction Ignore + Remove-Item variable:global:_ToolsetBuildProj -ErrorAction Ignore + Remove-Item variable:global:_MSBuildExe -ErrorAction Ignore +} From ac3fe493c427a8714b3d65bfa24065dbc14ebeea Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Mon, 18 May 2020 12:26:25 -0700 Subject: [PATCH 03/11] Enable building managed projects depending on native assets - splitting native builds out confuses these projects - use `$(BuildNative)` less, only to control actual building (not bundling) - build both native platforms in one `msbuild` invocation --- Directory.Build.props | 2 +- eng/Build.props | 12 +++++------- eng/targets/ResolveIisReferences.targets | 7 ------- .../src/Microsoft.AspNetCore.App.Runtime.csproj | 16 ++++++++++++---- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index edd4ebe1aa67..010f6b68fb77 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -180,7 +180,7 @@ $(MSBuildThisFileDirectory)src\Mvc\Mvc.Testing\src\Microsoft.AspNetCore.Mvc.Testing.targets - true + true $(ArtifactsObjDir)TargetingPack.Layout\$(Configuration)\ diff --git a/eng/Build.props b/eng/Build.props index 2af7e7dedf39..2c1098a3900b 100644 --- a/eng/Build.props +++ b/eng/Build.props @@ -84,14 +84,12 @@ Include="$(RepoRoot)src\Installers\Rpm\**\*.*proj" /> - - - - Platform=x64 - Platform=Win32 - + + + + + diff --git a/eng/targets/ResolveIisReferences.targets b/eng/targets/ResolveIisReferences.targets index a13559c7625e..5e4632a7495a 100644 --- a/eng/targets/ResolveIisReferences.targets +++ b/eng/targets/ResolveIisReferences.targets @@ -30,11 +30,4 @@ with the right MSBuild incantations to get output copied to the right place. - - - - - diff --git a/src/Framework/src/Microsoft.AspNetCore.App.Runtime.csproj b/src/Framework/src/Microsoft.AspNetCore.App.Runtime.csproj index adcdc8fbeea6..091b78b776e0 100644 --- a/src/Framework/src/Microsoft.AspNetCore.App.Runtime.csproj +++ b/src/Framework/src/Microsoft.AspNetCore.App.Runtime.csproj @@ -117,6 +117,9 @@ This package is an internal implementation of the .NET Core SDK and is not meant RuntimeList.xml $(ArtifactsObjDir)$(FrameworkListFileName) + + $(TargetArchitecture) + Win32 @@ -127,10 +130,9 @@ This package is an internal implementation of the .NET Core SDK and is not meant - - Platform=$(TargetArchitecture) - Platform=Win32 + Platform=$(NativePlatform) true @@ -138,6 +140,11 @@ This package is an internal implementation of the .NET Core SDK and is not meant Build;GetTargetPath + + <_ResolvedNativeProjectReferencePaths Condition=" '$(BuildIisNativeProjects)' == 'true' AND !$(BuildNative) " + Include="$(ArtifactsBinDir)InProcessRequestHandler\$(NativePlatform)\$(Configuration)\aspnetcorev2_inprocess.dll"> + true + @@ -216,7 +223,8 @@ This package is an internal implementation of the .NET Core SDK and is not meant + Condition=" '$(BuildIisNativeProjects)' == 'true' AND !$(BuildNative) AND + !EXISTS('$(ArtifactsBinDir)InProcessRequestHandler\$(NativePlatform)\$(Configuration)\aspnetcorev2_inprocess.dll') "> From dd70fd57c1397bb53e163c37d6b82b6fa6b26e62 Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Tue, 19 May 2020 20:33:16 -0700 Subject: [PATCH 04/11] Revert move to VS2019.Pre queues This reverts part of commit b67d161e03350053e4664af6386f5ace4991b254 - was "[release/5.0-preview5] Update dependencies from dotnet/aspnetcore-tooling (#21710)" --- .azure/pipelines/jobs/default-build.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index 58619deed73e..5c6357bd4b04 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -104,11 +104,16 @@ jobs: ${{ if eq(parameters.agentOs, 'Windows') }}: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: NetCorePublic-Pool - queue: BuildPool.Windows.10.Amd64.VS2019.Pre.Open + ${{ if ne(parameters.isTestingJob, true) }}: + # Visual Studio Build Tools + queue: BuildPool.Server.Amd64.VS2019.BT.Open + ${{ if eq(parameters.isTestingJob, true) }}: + # Visual Studio Enterprise - contains some stuff, like SQL Server and IIS Express, that we use for testing + queue: BuildPool.Server.Amd64.VS2019.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: NetCoreInternal-Pool # Visual Studio Enterprise - contains some stuff, like SQL Server and IIS Express, that we use for testing - queue: BuildPool.Windows.10.Amd64.VS2019.Pre + queue: BuildPool.Server.Amd64.VS2019 variables: - AgentOsName: ${{ parameters.agentOs }} - ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping From 089e624c3b253ea9c53963b1aa16a647399f1c21 Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Tue, 19 May 2020 21:49:04 -0700 Subject: [PATCH 05/11] Revert "!temporary! Require `msbuild` from VS2019 16.6" - this reverts commit 58cf2304a642312477dc41cd9651e6f4c2d39286 --- global.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global.json b/global.json index d16af0f0ea2d..4d4e54621db4 100644 --- a/global.json +++ b/global.json @@ -22,7 +22,7 @@ "Git": "2.22.0", "jdk": "11.0.3", "vs": { - "version": "16.6", + "version": "16.5", "components": [ "Microsoft.VisualStudio.Component.VC.ATL", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", From fdd6ccccb82bd6d95d9fb0b8b56cc0e2f077667e Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Wed, 20 May 2020 21:39:54 -0700 Subject: [PATCH 06/11] Reduce build duplication in pipelines - build native assets and repo tasks once per CI job - only cleanup framework references after packing managed projects nits: - wrap a few long lines - remove extra `-forceCoreMsbuild` options in SiteExtensions' build.cmd --- .azure/pipelines/ci.yml | 27 +++++++++++++++++--------- .azure/pipelines/helix-matrix.yml | 16 +++++++++------ .azure/pipelines/quarantined-tests.yml | 4 +++- eng/AfterSolutionBuild.targets | 3 ++- src/SiteExtensions/build.cmd | 6 +++--- 5 files changed, 36 insertions(+), 20 deletions(-) diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index 2f171d5b4c2e..5dda05432a78 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -47,7 +47,8 @@ variables: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNet-MSRC-Storage - name: _InternalRuntimeDownloadArgs - value: -DotNetRuntimeSourceFeed https://dotnetclimsrc.blob.core.windows.net/dotnet -DotNetRuntimeSourceFeedKey $(dotnetclimsrc-read-sas-token-base64) /p:DotNetAssetRootAccessTokenSuffix='$(dotnetclimsrc-read-sas-token-base64)' + value: -DotNetRuntimeSourceFeed https://dotnetclimsrc.blob.core.windows.net/dotnet -DotNetRuntimeSourceFeedKey + $(dotnetclimsrc-read-sas-token-base64) /p:DotNetAssetRootAccessTokenSuffix='$(dotnetclimsrc-read-sas-token-base64)' # The code signing doesn't use the aspnet build scripts, so the msbuild parameters have # to be passed directly. This is awkward, since we pass the same info above, but we have # to have it in two different forms @@ -140,19 +141,21 @@ stages: - script: ./build.cmd -ci -nobl + -noBuildRepoTasks -arch x86 -pack -all -noBuildJava + -noBuildNative /p:OnlyPackPlatformSpecificPackages=true $(_BuildArgs) $(_InternalRuntimeDownloadArgs) displayName: Build x86 - # This is in a separate build step with -forceCoreMsbuild to workaround MAX_PATH limitations - https://github.com/Microsoft/msbuild/issues/53 - script: .\src\SiteExtensions\build.cmd -ci -nobl + -noBuildRepoTasks -pack -noBuildDeps $(_BuildArgs) @@ -160,12 +163,13 @@ stages: condition: ne(variables['Build.Reason'], 'PullRequest') displayName: Build SiteExtension - # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If https://github.com/dotnet/arcade/issues/1957 is resolved, - # consider running code-signing inline with the other previous steps. - # Sign check is disabled because it is run in a separate step below, after installers are built. + # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If + # https://github.com/dotnet/arcade/issues/1957 is resolved, consider running code-signing inline with the other + # previous steps. Sign check is disabled because it is run in a separate step below, after installers are built. - script: ./build.cmd -ci -nobl + -noBuildRepoTasks -noBuild -noRestore -sign @@ -177,6 +181,7 @@ stages: - script: ./build.cmd -ci -nobl + -noBuildRepoTasks -sign -buildInstallers /p:DotNetSignType=$(_SignType) @@ -492,7 +497,9 @@ stages: jobDisplayName: "Test: Windows Server 2016 x64" agentOs: Windows isTestingJob: true - buildArgs: -all -pack -test "/p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false" $(_InternalRuntimeDownloadArgs) + buildArgs: -all -pack -test /p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true + /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false + $(_InternalRuntimeDownloadArgs) beforeBuild: - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1" displayName: Setup IISExpress test certificates and schema @@ -530,9 +537,9 @@ stages: steps: - script: ./build.cmd -ci -nobl -all -pack $(_InternalRuntimeDownloadArgs) displayName: Build Repo - - script: ./src/ProjectTemplates/build.cmd -ci -nobl -pack -NoRestore -NoBuilddeps "/p:RunTemplateTests=true" + - script: ./src/ProjectTemplates/build.cmd -ci -nobl -noBuildRepoTasks -pack -NoRestore -NoBuilddeps "/p:RunTemplateTests=true" displayName: Pack Templates - - script: ./src/ProjectTemplates/build.cmd -ci -nobl -test -NoRestore -NoBuild -NoBuilddeps "/p:RunTemplateTests=true" + - script: ./src/ProjectTemplates/build.cmd -ci -nobl -noBuildRepoTasks -test -NoRestore -NoBuild -NoBuilddeps "/p:RunTemplateTests=true" displayName: Test Templates artifacts: - name: Windows_Test_Templates_Dumps @@ -634,7 +641,9 @@ stages: displayName: Build shared fx - script: .\restore.cmd -ci -nobl /p:BuildInteropProjects=true displayName: Restore interop projects - - script: .\build.cmd -ci -nobl -NoRestore -test -all -projects eng\helix\helix.proj /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log + - script: ./build.cmd -ci -nobl -noBuildRepoTasks -noRestore -test -all -noBuildNative -projects eng\helix\helix.proj + /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true + /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Run build.cmd helix target env: HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues diff --git a/.azure/pipelines/helix-matrix.yml b/.azure/pipelines/helix-matrix.yml index 51065fd1ae13..a8dc39f43324 100644 --- a/.azure/pipelines/helix-matrix.yml +++ b/.azure/pipelines/helix-matrix.yml @@ -7,7 +7,7 @@ schedules: include: - master always: true - + variables: - ${{ if ne(variables['System.TeamProject'], 'internal') }}: - name: _UseHelixOpenQueues @@ -15,8 +15,8 @@ variables: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNet-HelixApi-Access - name: _UseHelixOpenQueues - value: 'false' - + value: 'false' + jobs: - template: jobs/default-build.yml parameters: @@ -30,7 +30,9 @@ jobs: displayName: Build shared fx - script: .\restore.cmd -ci /p:BuildInteropProjects=true displayName: Restore interop projects - - script: .\build.cmd -ci -nobl -NoRestore -test -all -projects eng\helix\helix.proj /p:IsHelixDaily=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log + - script: .\build.cmd -ci -nobl -NoRestore -test -all -projects eng\helix\helix.proj + /p:IsHelixDaily=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true + /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Run build.cmd helix target env: HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues @@ -39,7 +41,7 @@ jobs: - name: Helix_logs path: artifacts/log/ publishOnError: true - + # Helix ARM64 - template: jobs/default-build.yml parameters: @@ -50,7 +52,9 @@ jobs: steps: - script: ./restore.sh -ci -nobl displayName: Restore - - script: ./build.sh -ci --nobl --arch arm64 -test --no-build-nodejs --all -projects $(Build.SourcesDirectory)/eng/helix/helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log + - script: ./build.sh -ci --nobl --arch arm64 -test --no-build-nodejs --all -projects + $(Build.SourcesDirectory)/eng/helix/helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true + /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Run build.sh helix arm64 target env: HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues diff --git a/.azure/pipelines/quarantined-tests.yml b/.azure/pipelines/quarantined-tests.yml index 9b2c9da91b43..006c1d9d71cb 100644 --- a/.azure/pipelines/quarantined-tests.yml +++ b/.azure/pipelines/quarantined-tests.yml @@ -35,7 +35,9 @@ jobs: displayName: Build shared fx - script: .\restore.cmd -ci /p:BuildInteropProjects=true displayName: Restore interop projects - - script: .\build.cmd -ci -NoRestore -test -noBuildJava -all -projects eng\helix\helix.proj /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl + - script: ./build.cmd -ci -noRestore -test -all -noBuildJava -noBuildNative -projects eng\helix\helix.proj + /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true + /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Run build.cmd helix target env: HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets index d3697c09ee1d..4a3cb4ba2adf 100644 --- a/eng/AfterSolutionBuild.targets +++ b/eng/AfterSolutionBuild.targets @@ -5,7 +5,8 @@ - + <_BuildOutput Include="$(ArtifactsShippingPackagesDir)*.nupkg" Exclude="$(ArtifactsShippingPackagesDir)*.symbols.nupkg" /> diff --git a/src/SiteExtensions/build.cmd b/src/SiteExtensions/build.cmd index 76ca111525b3..8f9d90787927 100644 --- a/src/SiteExtensions/build.cmd +++ b/src/SiteExtensions/build.cmd @@ -18,21 +18,21 @@ IF %ERRORLEVEL% NEQ 0 ( ECHO Building x64 LoggingBranch REM /p:DisableTransitiveFrameworkReferences=true is needed to prevent SDK from picking up transitive references to REM Microsoft.AspNetCore.App as framework references https://github.com/dotnet/sdk/pull/3221 -CALL "%RepoRoot%\build.cmd" -forceCoreMsbuild -arch x64 -projects "%~dp0LoggingBranch\LB.csproj" ^ +CALL "%RepoRoot%\build.cmd" -arch x64 -projects "%~dp0LoggingBranch\LB.csproj" ^ /p:DisableTransitiveFrameworkReferences=true /bl:artifacts/log/SiteExtensions-LoggingBranch-x64.binlog %* IF %ERRORLEVEL% NEQ 0 ( EXIT /b %ErrorLevel% ) ECHO Building x86 LoggingBranch -CALL "%RepoRoot%\build.cmd" -forceCoreMsbuild -arch x86 -projects "%~dp0LoggingBranch\LB.csproj" ^ +CALL "%RepoRoot%\build.cmd" -arch x86 -projects "%~dp0LoggingBranch\LB.csproj" ^ /p:DisableTransitiveFrameworkReferences=true /bl:artifacts/log/SiteExtensions-LoggingBranch-x86.binlog %* IF %ERRORLEVEL% NEQ 0 ( EXIT /b %ErrorLevel% ) ECHO Building Microsoft.AspNetCore.AzureAppServices.SiteExtension -CALL "%RepoRoot%\build.cmd" -forceCoreMsbuild -projects ^ +CALL "%RepoRoot%\build.cmd" -projects ^ "%~dp0LoggingAggregate\src\Microsoft.AspNetCore.AzureAppServices.SiteExtension\Microsoft.AspNetCore.AzureAppServices.SiteExtension.csproj" ^ /bl:artifacts/log/SiteExtensions-LoggingAggregate.binlog %* IF %ERRORLEVEL% NEQ 0 ( From 9f038b5a2db459a5d75d72f0190e473ada8c6c6f Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Mon, 25 May 2020 21:30:23 -0700 Subject: [PATCH 07/11] Fix Helix jobs - restore.cmd doesn't work well with `-projects`; script unconditionally adds `-all` --- .azure/pipelines/ci.yml | 2 +- .azure/pipelines/helix-matrix.yml | 2 +- .azure/pipelines/quarantined-tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index 5dda05432a78..0bd64368b0ec 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -639,7 +639,7 @@ stages: # Build the shared framework - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Build shared fx - - script: .\restore.cmd -ci -nobl /p:BuildInteropProjects=true + - script: ./build.cmd -ci -nobl -noBuildRepoTasks -restore -noBuild -projects src/Grpc/**/*.csproj displayName: Restore interop projects - script: ./build.cmd -ci -nobl -noBuildRepoTasks -noRestore -test -all -noBuildNative -projects eng\helix\helix.proj /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true diff --git a/.azure/pipelines/helix-matrix.yml b/.azure/pipelines/helix-matrix.yml index a8dc39f43324..c7d1b339d835 100644 --- a/.azure/pipelines/helix-matrix.yml +++ b/.azure/pipelines/helix-matrix.yml @@ -28,7 +28,7 @@ jobs: # Build the shared framework - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Build shared fx - - script: .\restore.cmd -ci /p:BuildInteropProjects=true + - script: ./build.cmd -ci -nobl -restore -noBuild -projects src/Grpc/**/*.csproj displayName: Restore interop projects - script: .\build.cmd -ci -nobl -NoRestore -test -all -projects eng\helix\helix.proj /p:IsHelixDaily=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true diff --git a/.azure/pipelines/quarantined-tests.yml b/.azure/pipelines/quarantined-tests.yml index 006c1d9d71cb..b4b0ac96d857 100644 --- a/.azure/pipelines/quarantined-tests.yml +++ b/.azure/pipelines/quarantined-tests.yml @@ -33,7 +33,7 @@ jobs: # Build the shared framework - script: ./build.cmd -ci -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log /bl:artifacts/log/helix.build.x64.binlog displayName: Build shared fx - - script: .\restore.cmd -ci /p:BuildInteropProjects=true + - script: ./build.cmd -ci -nobl -restore -noBuild -projects src/Grpc/**/*.csproj displayName: Restore interop projects - script: ./build.cmd -ci -noRestore -test -all -noBuildJava -noBuildNative -projects eng\helix\helix.proj /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true From 648cc822d5bc4905e6f5df3ff64209064ed4e212 Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Fri, 29 May 2020 16:26:45 -0700 Subject: [PATCH 08/11] !fixup! Reduce duplications further - missed a couple of places `-noBuildRepoTasks` helps --- .azure/pipelines/helix-matrix.yml | 6 +++--- .azure/pipelines/quarantined-tests.yml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.azure/pipelines/helix-matrix.yml b/.azure/pipelines/helix-matrix.yml index c7d1b339d835..104481013675 100644 --- a/.azure/pipelines/helix-matrix.yml +++ b/.azure/pipelines/helix-matrix.yml @@ -28,9 +28,9 @@ jobs: # Build the shared framework - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Build shared fx - - script: ./build.cmd -ci -nobl -restore -noBuild -projects src/Grpc/**/*.csproj + - script: ./build.cmd -ci -nobl -noBuildRepoTasks -restore -noBuild -projects src/Grpc/**/*.csproj displayName: Restore interop projects - - script: .\build.cmd -ci -nobl -NoRestore -test -all -projects eng\helix\helix.proj + - script: .\build.cmd -ci -nobl -noBuildRepoTasks -NoRestore -test -all -projects eng\helix\helix.proj /p:IsHelixDaily=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Run build.cmd helix target @@ -52,7 +52,7 @@ jobs: steps: - script: ./restore.sh -ci -nobl displayName: Restore - - script: ./build.sh -ci --nobl --arch arm64 -test --no-build-nodejs --all -projects + - script: ./build.sh --ci --nobl --noBuildRepoTasks --arch arm64 -test --no-build-nodejs --all --projects $(Build.SourcesDirectory)/eng/helix/helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Run build.sh helix arm64 target diff --git a/.azure/pipelines/quarantined-tests.yml b/.azure/pipelines/quarantined-tests.yml index b4b0ac96d857..9fc87265390b 100644 --- a/.azure/pipelines/quarantined-tests.yml +++ b/.azure/pipelines/quarantined-tests.yml @@ -33,11 +33,11 @@ jobs: # Build the shared framework - script: ./build.cmd -ci -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log /bl:artifacts/log/helix.build.x64.binlog displayName: Build shared fx - - script: ./build.cmd -ci -nobl -restore -noBuild -projects src/Grpc/**/*.csproj + - script: ./build.cmd -ci -nobl -noBuildRepoTasks -restore -noBuild -projects src/Grpc/**/*.csproj displayName: Restore interop projects - - script: ./build.cmd -ci -noRestore -test -all -noBuildJava -noBuildNative -projects eng\helix\helix.proj - /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true - /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log + - script: ./build.cmd -ci -noBuildRepoTasks -noRestore -test -all -noBuildJava -noBuildNative + -projects eng\helix\helix.proj /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true + /p:BuildInteropProjects=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Run build.cmd helix target env: HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues From 7d9dad7c02dfb2c0120f0531678d7936bd4f14de Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Fri, 29 May 2020 16:38:10 -0700 Subject: [PATCH 09/11] Cleanup: Remove a few dangling binary logs --- .azure/pipelines/jobs/codesign-xplat.yml | 1 + .azure/pipelines/quarantined-tests.yml | 4 ++-- eng/scripts/GenerateProjectList.ps1 | 4 +--- eng/scripts/GenerateReferenceAssemblies.ps1 | 6 ++---- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.azure/pipelines/jobs/codesign-xplat.yml b/.azure/pipelines/jobs/codesign-xplat.yml index 2bf05f711d5c..d65c9a59826e 100644 --- a/.azure/pipelines/jobs/codesign-xplat.yml +++ b/.azure/pipelines/jobs/codesign-xplat.yml @@ -30,6 +30,7 @@ jobs: flattenFolders: true - powershell: .\eng\common\build.ps1 -ci + -nobl -restore -sign -publish diff --git a/.azure/pipelines/quarantined-tests.yml b/.azure/pipelines/quarantined-tests.yml index 9fc87265390b..355c94268fec 100644 --- a/.azure/pipelines/quarantined-tests.yml +++ b/.azure/pipelines/quarantined-tests.yml @@ -31,11 +31,11 @@ jobs: timeoutInMinutes: 240 steps: # Build the shared framework - - script: ./build.cmd -ci -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log /bl:artifacts/log/helix.build.x64.binlog + - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Build shared fx - script: ./build.cmd -ci -nobl -noBuildRepoTasks -restore -noBuild -projects src/Grpc/**/*.csproj displayName: Restore interop projects - - script: ./build.cmd -ci -noBuildRepoTasks -noRestore -test -all -noBuildJava -noBuildNative + - script: ./build.cmd -ci -nobl -noBuildRepoTasks -noRestore -test -all -noBuildJava -noBuildNative -projects eng\helix\helix.proj /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildInteropProjects=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Run build.cmd helix target diff --git a/eng/scripts/GenerateProjectList.ps1 b/eng/scripts/GenerateProjectList.ps1 index 11c5e2e296fb..c76bd981ac5d 100644 --- a/eng/scripts/GenerateProjectList.ps1 +++ b/eng/scripts/GenerateProjectList.ps1 @@ -7,9 +7,7 @@ $msbuildEngine = 'dotnet' $repoRoot = Resolve-Path "$PSScriptRoot/../.." try { - & "$repoRoot\eng\common\msbuild.ps1" -ci:$ci "$repoRoot/eng/CodeGen.proj" ` - /t:GenerateProjectList ` - /bl:artifacts/log/genprojlist.binlog + & "$repoRoot\eng\common\msbuild.ps1" -ci:$ci -nobl "$repoRoot/eng/CodeGen.proj" /t:GenerateProjectList } finally { Remove-Item variable:global:_BuildTool -ErrorAction Ignore Remove-Item variable:global:_DotNetInstallDir -ErrorAction Ignore diff --git a/eng/scripts/GenerateReferenceAssemblies.ps1 b/eng/scripts/GenerateReferenceAssemblies.ps1 index e5a64b8bee2e..96522d1c64e9 100644 --- a/eng/scripts/GenerateReferenceAssemblies.ps1 +++ b/eng/scripts/GenerateReferenceAssemblies.ps1 @@ -7,7 +7,7 @@ $repoRoot = Resolve-Path "$PSScriptRoot/../.." try { # eng/common/msbuild.ps1 builds the Debug configuration unless there's a $Configuration variable. Match that here. - & "$repoRoot\build.ps1" -ci:$ci -nobl -buildNative -configuration Debug + & "$repoRoot\build.ps1" -ci:$ci -nobl -noBuildRepoTasks -noRestore -buildNative -configuration Debug Remove-Item variable:global:_BuildTool -ErrorAction Ignore Remove-Item variable:global:_DotNetInstallDir -ErrorAction Ignore @@ -15,9 +15,7 @@ try { Remove-Item variable:global:_MSBuildExe -ErrorAction Ignore $msbuildEngine = 'dotnet' - & "$repoRoot\eng\common\msbuild.ps1" -ci:$ci "$repoRoot/eng/CodeGen.proj" ` - /t:GenerateReferenceSources ` - /bl:artifacts/log/genrefassemblies.binlog + & "$repoRoot\eng\common\msbuild.ps1" -ci:$ci -nobl "$repoRoot/eng/CodeGen.proj" /t:GenerateReferenceSources } finally { Remove-Item variable:global:_BuildTool -ErrorAction Ignore Remove-Item variable:global:_DotNetInstallDir -ErrorAction Ignore From 3b149b50fecfb479084bd110a63779108279af58 Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Fri, 29 May 2020 17:12:08 -0700 Subject: [PATCH 10/11] !fixup! Correct typos in generation scripts --- eng/scripts/GenerateProjectList.ps1 | 3 ++- eng/scripts/GenerateReferenceAssemblies.ps1 | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/scripts/GenerateProjectList.ps1 b/eng/scripts/GenerateProjectList.ps1 index c76bd981ac5d..60c6faed2535 100644 --- a/eng/scripts/GenerateProjectList.ps1 +++ b/eng/scripts/GenerateProjectList.ps1 @@ -3,11 +3,12 @@ param( ) $ErrorActionPreference = 'stop' +$excludeCIBinarylog = true $msbuildEngine = 'dotnet' $repoRoot = Resolve-Path "$PSScriptRoot/../.." try { - & "$repoRoot\eng\common\msbuild.ps1" -ci:$ci -nobl "$repoRoot/eng/CodeGen.proj" /t:GenerateProjectList + & "$repoRoot\eng\common\msbuild.ps1" -ci:$ci "$repoRoot/eng/CodeGen.proj" /t:GenerateProjectList } finally { Remove-Item variable:global:_BuildTool -ErrorAction Ignore Remove-Item variable:global:_DotNetInstallDir -ErrorAction Ignore diff --git a/eng/scripts/GenerateReferenceAssemblies.ps1 b/eng/scripts/GenerateReferenceAssemblies.ps1 index 96522d1c64e9..42118d8b5dc6 100644 --- a/eng/scripts/GenerateReferenceAssemblies.ps1 +++ b/eng/scripts/GenerateReferenceAssemblies.ps1 @@ -14,8 +14,9 @@ try { Remove-Item variable:global:_ToolsetBuildProj -ErrorAction Ignore Remove-Item variable:global:_MSBuildExe -ErrorAction Ignore + $excludeCIBinarylog = true $msbuildEngine = 'dotnet' - & "$repoRoot\eng\common\msbuild.ps1" -ci:$ci -nobl "$repoRoot/eng/CodeGen.proj" /t:GenerateReferenceSources + & "$repoRoot\eng\common\msbuild.ps1" -ci:$ci "$repoRoot/eng/CodeGen.proj" /t:GenerateReferenceSources } finally { Remove-Item variable:global:_BuildTool -ErrorAction Ignore Remove-Item variable:global:_DotNetInstallDir -ErrorAction Ignore From 5baad6bcdcad925c2ceec4409137b66e91e3c8af Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Fri, 29 May 2020 17:23:30 -0700 Subject: [PATCH 11/11] !fixup! Another typo in the generation scripts --- eng/scripts/GenerateProjectList.ps1 | 2 +- eng/scripts/GenerateReferenceAssemblies.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/scripts/GenerateProjectList.ps1 b/eng/scripts/GenerateProjectList.ps1 index 60c6faed2535..5135db533b77 100644 --- a/eng/scripts/GenerateProjectList.ps1 +++ b/eng/scripts/GenerateProjectList.ps1 @@ -3,7 +3,7 @@ param( ) $ErrorActionPreference = 'stop' -$excludeCIBinarylog = true +$excludeCIBinarylog = $true $msbuildEngine = 'dotnet' $repoRoot = Resolve-Path "$PSScriptRoot/../.." diff --git a/eng/scripts/GenerateReferenceAssemblies.ps1 b/eng/scripts/GenerateReferenceAssemblies.ps1 index 42118d8b5dc6..56c1101fb325 100644 --- a/eng/scripts/GenerateReferenceAssemblies.ps1 +++ b/eng/scripts/GenerateReferenceAssemblies.ps1 @@ -14,7 +14,7 @@ try { Remove-Item variable:global:_ToolsetBuildProj -ErrorAction Ignore Remove-Item variable:global:_MSBuildExe -ErrorAction Ignore - $excludeCIBinarylog = true + $excludeCIBinarylog = $true $msbuildEngine = 'dotnet' & "$repoRoot\eng\common\msbuild.ps1" -ci:$ci "$repoRoot/eng/CodeGen.proj" /t:GenerateReferenceSources } finally {