diff --git a/eng/pipelines/templates/stages/vmr-build.yml b/eng/pipelines/templates/stages/vmr-build.yml index 9523c854dcbc..b484e828c96e 100644 --- a/eng/pipelines/templates/stages/vmr-build.yml +++ b/eng/pipelines/templates/stages/vmr-build.yml @@ -123,6 +123,26 @@ stages: - template: ../steps/vmr-validate-asset-baseline.yml parameters: continueOnError: true + - job: ValidateInstallers_Linux_x64 + displayName: Validate Installers - Linux x64 + pool: ${{ parameters.pool_Linux }} + timeoutInMinutes: 30 + steps: + - template: ../steps/vmr-validate-installers.yml + parameters: + targetArchitecture: x64 + continueOnError: true + OS: Linux + - job: ValidateInstallers_Linux_arm64 + displayName: Validate Installers - Linux arm64 + pool: ${{ parameters.pool_LinuxArm64 }} + timeoutInMinutes: 30 + steps: + - template: ../steps/vmr-validate-installers.yml + parameters: + targetArchitecture: arm64 + continueOnError: true + OS: Linux - ${{ if eq(variables.signEnabled, 'true') }}: - job: ValidateSigning_Windows displayName: Validate Signing - Windows diff --git a/eng/pipelines/templates/steps/vmr-validate-installers.yml b/eng/pipelines/templates/steps/vmr-validate-installers.yml new file mode 100755 index 000000000000..710e9a48cd4b --- /dev/null +++ b/eng/pipelines/templates/steps/vmr-validate-installers.yml @@ -0,0 +1,67 @@ +parameters: +- name: continueOnError + type: boolean + default: false + +- name: targetArchitecture + type: string + default: '' + +- name: OS + type: string + values: + - Windows_NT + - Linux + - Darwin + +steps: +- task: DownloadBuildArtifacts@1 + inputs: + artifactName: BlobArtifacts + downloadPath: $(Build.ArtifactStagingDirectory) + checkDownloadedFiles: true + displayName: Download Blob Artifacts + +- task: DownloadBuildArtifacts@1 + inputs: + artifactName: PackageArtifacts + downloadPath: $(Build.ArtifactStagingDirectory) + checkDownloadedFiles: true + displayName: Download Package Artifacts + +# This is necessary whenever we want to publish/restore to an AzDO private feed +# Since sdk-task.ps1 tries to restore packages we need to do this authentication here +# otherwise it'll complain about accessing a private feed. +- task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + +- ${{ if eq(parameters.OS, 'Linux') }}: + - script: | + extraBuildProperties="/p:TestRpmPackages=true" + if [[ '${{ parameters.targetArchitecture }}' == 'x64' ]]; then + # At the moment Deb packages are only available for x64 + extraBuildProperties="$extraBuildProperties /p:TestDebPackages=true" + fi + + ./build.sh \ + --ci \ + -t \ + --projects test/Microsoft.DotNet.Installer.Tests/Microsoft.DotNet.Installer.Tests.csproj \ + /p:BlobArtifactsDir=$(Build.ArtifactStagingDirectory)/BlobArtifacts \ + /p:PackageArtifactsDir=$(Build.ArtifactStagingDirectory)/PackageArtifacts \ + $extraBuildProperties + displayName: Validate installer packages + workingDirectory: $(Build.SourcesDirectory) + continueOnError: ${{ parameters.continueOnError }} + + - task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + continueOnError: true + inputs: + testRunner: VSTest + testResultsFiles: 'artifacts/TestResults/Release/*.trx' + searchFolder: $(Build.SourcesDirectory) + mergeTestResults: true + publishRunAttachments: true + testRunTitle: Tests_$(Agent.JobName) diff --git a/src/SourceBuild/content/build.sh b/src/SourceBuild/content/build.sh index 6337a9ce4558..2f07c907cf8b 100755 --- a/src/SourceBuild/content/build.sh +++ b/src/SourceBuild/content/build.sh @@ -39,6 +39,7 @@ usage() echo " --build-repo-tests Build repository tests" echo "Advanced settings:" + echo " --projects Project or solution file to build" echo " --ci Set when running on CI server" echo " --clean-while-building Cleans each repo after building (reduces disk space usage, short: -cwb)" echo " --excludeCIBinarylog Don't output binary log (short: -nobl)" @@ -82,6 +83,7 @@ packagesArchiveDir="${packagesDir}archive/" packagesPreviouslySourceBuiltDir="${packagesDir}previously-source-built/" # Advanced settings +projects='' ci=false exclude_ci_binary_log=false prepare_machine=false @@ -176,6 +178,10 @@ while [[ $# > 0 ]]; do -build-repo-tests) properties+=( "/p:DotNetBuildTests=true" ) ;; + -projects) + projects=$2 + shift + ;; -ci) ci=true ;; @@ -222,6 +228,11 @@ if [[ "$test" == true ]]; then export MSBUILDDISABLENODEREUSE=1 fi +# Override project if specified on cmd-line +if [[ ! -z "$projects" ]]; then + project="$projects" +fi + function Build { if [[ "$sourceOnly" != "true" ]]; then diff --git a/src/SourceBuild/content/eng/build.ps1 b/src/SourceBuild/content/eng/build.ps1 index f5ef3c167bc8..996b923498c1 100644 --- a/src/SourceBuild/content/eng/build.ps1 +++ b/src/SourceBuild/content/eng/build.ps1 @@ -13,6 +13,7 @@ Param( # Advanced settings [switch]$buildRepoTests, + [string]$projects, [switch]$ci, [switch][Alias('cwb')]$cleanWhileBuilding, [switch][Alias('nobl')]$excludeCIBinarylog, @@ -36,6 +37,7 @@ function Get-Usage() { Write-Host "Advanced settings:" Write-Host " -buildRepoTests Build repository tests" + Write-Host " -projects Project or solution file to build" Write-Host " -ci Set when running on CI server" Write-Host " -cleanWhileBuilding Cleans each repo after building (reduces disk space usage, short: -cwb)" Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)" @@ -64,6 +66,11 @@ if ($test) { $env:MSBUILDENSURESTDOUTFORTASKPROCESSES="1" } +# Override project if specified on cmd-line +if ($projects) { + $project = $projects +} + if ($sign) { $arguments += "/p:Sign=true" } diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.Installer.Tests/Config.cs b/src/SourceBuild/content/test/Microsoft.DotNet.Installer.Tests/Config.cs index 663cd347d712..9a74ec32e58b 100755 --- a/src/SourceBuild/content/test/Microsoft.DotNet.Installer.Tests/Config.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.Installer.Tests/Config.cs @@ -9,6 +9,12 @@ namespace Microsoft.DotNet.Installer.Tests; +public enum Architecture +{ + X64, + Arm64 +} + public static class Config { public static string AssetsDirectory { get; } = GetRuntimeConfig(AssetsDirectorySwitch); @@ -20,7 +26,7 @@ public static class Config public static string ScenarioTestsNuGetConfigPath { get; } = GetRuntimeConfig(ScenarioTestsNuGetConfigSwitch); const string ScenarioTestsNuGetConfigSwitch = RuntimeConfigSwitchPrefix + nameof(ScenarioTestsNuGetConfigPath); - public static string Architecture { get; } = GetRuntimeConfig(ArchitectureSwitch); + public static Architecture Architecture { get; } = GetArchitecture(GetRuntimeConfig(ArchitectureSwitch)); const string ArchitectureSwitch = RuntimeConfigSwitchPrefix + nameof(Architecture); public static bool TestRpmPackages { get; } = TryGetRuntimeConfig(TestRpmPackagesSwitch, out bool value) ? value : false; @@ -34,6 +40,13 @@ public static class Config public const string RuntimeConfigSwitchPrefix = "Microsoft.DotNet.Installer.Tests."; + public static Architecture GetArchitecture(string architecture) => architecture switch + { + "x64" => Architecture.X64, + "arm64" => Architecture.Arm64, + _ => throw new ArgumentException($"Unknown architecture: {architecture}") + }; + public static string GetRuntimeConfig(string key) { return TryGetRuntimeConfig(key, out string? value) ? value : throw new InvalidOperationException($"Runtime config setting '{key}' must be specified"); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.Installer.Tests/LinuxInstallerTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.Installer.Tests/LinuxInstallerTests.cs index 236c6ddc8828..14e76b2a25fb 100755 --- a/src/SourceBuild/content/test/Microsoft.DotNet.Installer.Tests/LinuxInstallerTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.Installer.Tests/LinuxInstallerTests.cs @@ -25,6 +25,7 @@ public class LinuxInstallerTests : IDisposable private readonly string _tmpDir; private readonly string _contextDir; private readonly ITestOutputHelper _outputHelper; + private readonly string _excludeLinuxArch; private bool _rpmContextInitialized = false; private bool _debContextInitialized = false; @@ -32,7 +33,7 @@ public class LinuxInstallerTests : IDisposable private const string NetStandard21RpmPackage = @"https://dotnetcli.blob.core.windows.net/dotnet/Runtime/3.1.0/netstandard-targeting-pack-2.1.0-x64.rpm"; private const string NetStandard21DebPackage = @"https://dotnetcli.blob.core.windows.net/dotnet/Runtime/3.1.0/netstandard-targeting-pack-2.1.0-x64.deb"; - private const string RuntimeDepsRepo = "mcr.microsoft.com/dotnet/nightly/runtime-deps"; + private const string RuntimeDepsRepo = "mcr.microsoft.com/dotnet/runtime-deps"; private const string RuntimeDepsVersion = "10.0-preview"; public static bool IncludeRpmTests => Config.TestRpmPackages; @@ -53,13 +54,17 @@ public LinuxInstallerTests(ITestOutputHelper outputHelper) Directory.CreateDirectory(_tmpDir); _contextDir = Path.Combine(_tmpDir, Path.GetRandomFileName()); Directory.CreateDirectory(_contextDir); + + _excludeLinuxArch = Config.Architecture == Architecture.X64 ? + Architecture.Arm64.ToString().ToLower() : + Architecture.X64.ToString().ToLower(); } public void Dispose() { try { - //Directory.Delete(_tmpDir, recursive: true); + Directory.Delete(_tmpDir, recursive: true); } catch { @@ -92,11 +97,21 @@ public void DebTest(string repo, string tag) private void InitializeContext(PackageType packageType) { + string packageArchitecture = + Config.Architecture == Architecture.X64 ? + "x64" : + packageType == PackageType.Rpm ? + "aarch64" : + "arm64"; + if (packageType == PackageType.Rpm && !_rpmContextInitialized) { - // For rpm enumerate RPM packages, excluding those that contain ".cm." in the name - List rpmPackages = Directory.GetFiles(Config.AssetsDirectory, "*.rpm", SearchOption.AllDirectories) - .Where(p => !Path.GetFileName(p).Contains("-cm.") && !Path.GetFileName(p).EndsWith("azl.rpm")) + // Copy all applicable RPM packages, excluding Mariner and Azure Linux copies + List rpmPackages = + Directory.GetFiles(Config.AssetsDirectory, $"*-{packageArchitecture}*.rpm", SearchOption.AllDirectories) + .Where(p => !Path.GetFileName(p).Contains("-cm.") && + !Path.GetFileName(p).Contains("-azl-") && + !Path.GetFileName(p).EndsWith("azl.rpm")) .ToList(); foreach (string rpmPackage in rpmPackages) @@ -104,19 +119,24 @@ private void InitializeContext(PackageType packageType) File.Copy(rpmPackage, Path.Combine(_contextDir, Path.GetFileName(rpmPackage))); } - DownloadFileAsync(NetStandard21RpmPackage, Path.Combine(_contextDir, Path.GetFileName(NetStandard21RpmPackage))).Wait(); + if (Config.Architecture == Architecture.X64) + { + DownloadFileAsync(NetStandard21RpmPackage, Path.Combine(_contextDir, Path.GetFileName(NetStandard21RpmPackage))).Wait(); + } _rpmContextInitialized = true; } else if (!_debContextInitialized) { - // Copy all DEB packages as well - foreach (string debPackage in Directory.GetFiles(Config.AssetsDirectory, "*.deb", SearchOption.AllDirectories)) + // Copy all applicable DEB packages + foreach (string debPackage in Directory.GetFiles(Config.AssetsDirectory, $"*-{packageArchitecture}*.deb", SearchOption.AllDirectories)) { File.Copy(debPackage, Path.Combine(_contextDir, Path.GetFileName(debPackage))); } - // Download NetStandard 2.1 packages - DownloadFileAsync(NetStandard21DebPackage, Path.Combine(_contextDir, Path.GetFileName(NetStandard21DebPackage))).Wait(); + if (Config.Architecture == Architecture.X64) + { + DownloadFileAsync(NetStandard21DebPackage, Path.Combine(_contextDir, Path.GetFileName(NetStandard21DebPackage))).Wait(); + } _debContextInitialized = true; } @@ -127,7 +147,10 @@ private void InitializeContext(PackageType packageType) Directory.CreateDirectory(nugetPackagesDir); foreach (string package in Directory.GetFiles(Config.PackagesDirectory, "*.nupkg", SearchOption.AllDirectories)) { - File.Copy(package, Path.Combine(nugetPackagesDir, Path.GetFileName(package))); + if (ShouldCopyPackage(package.ToLower())) + { + File.Copy(package, Path.Combine(nugetPackagesDir, Path.GetFileName(package))); + } } // Copy and update NuGet.config from scenario-tests repo @@ -147,6 +170,24 @@ private void InitializeContext(PackageType packageType) } } + private bool ShouldCopyPackage(string package) + { + if (package.Contains(".osx-") || + package.Contains(".win-") || + package.Contains(".linux-musl-") || + package.Contains(".linux-bionic-") || + package.Contains(".mono.") || + package.Contains("symbols") || + package.Contains("vs.redist") || + package.Contains(".linux-arm.") || + package.Contains($".linux-{_excludeLinuxArch}.")) + { + return false; + } + + return true; + } + private void InsertLocalPackagesPathToNuGetConfig(string nuGetConfig, string localPackagesPath) { XDocument doc = XDocument.Load(nuGetConfig); @@ -247,7 +288,7 @@ private List GetPackageList(string baseImage, PackageType packageType) AddPackage(packageList, "aspnetcore-runtime-", packageType); AddPackage(packageList, "aspnetcore-targeting-pack-", packageType); AddPackage(packageList, "dotnet-apphost-pack-", packageType); - if (Config.Architecture == "x64") + if (Config.Architecture == Architecture.X64) { // netstandard package exists for x64 only AddPackage(packageList, "netstandard-targeting-pack-", packageType); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.Installer.Tests/Microsoft.DotNet.Installer.Tests.csproj b/src/SourceBuild/content/test/Microsoft.DotNet.Installer.Tests/Microsoft.DotNet.Installer.Tests.csproj index 3e3707bb6cdf..b6d450e74298 100755 --- a/src/SourceBuild/content/test/Microsoft.DotNet.Installer.Tests/Microsoft.DotNet.Installer.Tests.csproj +++ b/src/SourceBuild/content/test/Microsoft.DotNet.Installer.Tests/Microsoft.DotNet.Installer.Tests.csproj @@ -6,6 +6,11 @@ $(VSTestCLIRunSettings);RunConfiguration.DotNetHostPath=$(DotnetTool) + + $(ArtifactsAssetsDir) + $(ArtifactsPackagesDir) + + @@ -15,10 +20,10 @@ - $(ArtifactsAssetsDir) + $(BlobArtifactsDir) - $(ArtifactsPackagesDir) + $(PackageArtifactsDir) $(RepoRoot)src\scenario-tests\NuGet.config