From 7da15f5d20842ead0f8ce17fca0610ece8bf47e4 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Fri, 19 Apr 2019 12:26:53 +0000 Subject: [PATCH 1/7] Update dependencies from https://github.com/dotnet/arcade build 20190418.4 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19218.4 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index accc00e5562..728c0cfa8a8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 46718d98c0fd03690a6a8c83da692a4a85a17902 + 1d4c87ff798d7de521128e6c3d890055c2401e80 diff --git a/global.json b/global.json index cd0f7712fcc..a01e48045b5 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19218.1", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19218.4", "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2" } } From 93f77a5df4351c73ca2c588e93d3e6d739dd2889 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Sat, 20 Apr 2019 12:24:04 +0000 Subject: [PATCH 2/7] Update dependencies from https://github.com/dotnet/arcade build 20190418.7 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19218.7 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 728c0cfa8a8..5550cc453e2 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 1d4c87ff798d7de521128e6c3d890055c2401e80 + 5e7ce5b394f3477bb0a485a4b761b7742e95be37 diff --git a/global.json b/global.json index a01e48045b5..a8d15ae026c 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19218.4", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19218.7", "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2" } } From 9d63ba1c8811a18611a40889780a551152abfeb2 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Tue, 23 Apr 2019 12:29:07 +0000 Subject: [PATCH 3/7] Update dependencies from https://github.com/dotnet/arcade build 20190422.2 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19222.2 --- eng/Version.Details.xml | 4 ++-- eng/common/templates/steps/send-to-helix.yml | 2 +- global.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5550cc453e2..a47ce8cfe30 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 5e7ce5b394f3477bb0a485a4b761b7742e95be37 + 851e36df83d3361e4bd8a70a2a8a89f762469f9a diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 7c185e94147..d1ce577db5b 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -23,7 +23,7 @@ parameters: WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false diff --git a/global.json b/global.json index a8d15ae026c..754231e82e8 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19218.7", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19222.2", "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2" } } From c21c817c42d896ea6793c95a04ebb0619b33b752 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Wed, 24 Apr 2019 12:29:42 +0000 Subject: [PATCH 4/7] Update dependencies from https://github.com/dotnet/arcade build 20190423.2 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19223.2 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a47ce8cfe30..71149780705 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 851e36df83d3361e4bd8a70a2a8a89f762469f9a + e3919d0c158716ef4685c8e057cc58640af1af83 diff --git a/global.json b/global.json index 754231e82e8..0286a710f0a 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19222.2", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19223.2", "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2" } } From 2e13716a645454930bc0af107323dc86976b1f6a Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Thu, 25 Apr 2019 12:30:00 +0000 Subject: [PATCH 5/7] Update dependencies from https://github.com/dotnet/arcade build 20190424.9 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19224.9 --- eng/Version.Details.xml | 4 +- eng/common/templates/job/job.yml | 7 ++- eng/common/templates/steps/helix-publish.yml | 51 -------------------- global.json | 2 +- 4 files changed, 9 insertions(+), 55 deletions(-) delete mode 100644 eng/common/templates/steps/helix-publish.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 71149780705..643b01010a7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - e3919d0c158716ef4685c8e057cc58640af1af83 + e02c88fca482f1141a9bb310c97be20b0ebd0465 diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 74dd81fdc0a..7839b70bb70 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -43,9 +43,12 @@ parameters: # Optional: enable sending telemetry enableTelemetry: false - # Optional: define the helix repo for telemeetry (example: 'dotnet/arcade') + # Optional: define the helix repo for telemetry (example: 'dotnet/arcade') helixRepo: '' + # Optional: define the helix type for telemetry (example: 'build/product/') + helixType: '' + # Required: name of the job name: '' @@ -122,6 +125,8 @@ jobs: displayName: 'Send Helix Start Telemetry' inputs: helixRepo: ${{ parameters.helixRepo }} + ${{ if ne(parameters.helixType, '') }}: + helixType: ${{ parameters.helixType }} buildConfig: $(_BuildConfig) runAsPublic: ${{ parameters.runAsPublic }} continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/steps/helix-publish.yml b/eng/common/templates/steps/helix-publish.yml deleted file mode 100644 index 470ab65da0c..00000000000 --- a/eng/common/templates/steps/helix-publish.yml +++ /dev/null @@ -1,51 +0,0 @@ -parameters: - HelixSource: 'pr/dotnet-github-anon-kaonashi-bot' - HelixType: ̓'tests/default' - HelixBuild: $(Build.BuildNumber) - HelixTargetQueues: '' - HelixAccessToken: '' - HelixPreCommands: '' - HelixPostCommands: '' - WorkItemDirectory: '' - WorkItemCommand: '' - CorrelationPayloadDirectory: '' - XUnitProjects: '' - XUnitTargetFramework: '' - XUnitRunnerVersion: '' - IncludeDotNetCli: false - DotNetCliPackageType: '' - DotNetCliVersion: '' - EnableXUnitReporter: false - WaitForWorkItemCompletion: true - condition: succeeded() - continueOnError: false - -steps: - - task: DotNetCoreCLI@2 - inputs: - command: custom - projects: eng/common/helixpublish.proj - custom: msbuild - arguments: '/bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog' - displayName: Send job to Helix - env: - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} diff --git a/global.json b/global.json index 0286a710f0a..0a92ec345d5 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19223.2", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19224.9", "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2" } } From d23daa9bb99fa42171751a58843245deb6c2fce2 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Fri, 26 Apr 2019 12:29:57 +0000 Subject: [PATCH 6/7] Update dependencies from https://github.com/dotnet/arcade build 20190425.5 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19225.5 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 643b01010a7..8550113fae4 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - e02c88fca482f1141a9bb310c97be20b0ebd0465 + d37270268a65592cae630f1b979b70f74d4614dd diff --git a/global.json b/global.json index 0a92ec345d5..394d5b5016f 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19224.9", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19225.5", "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2" } } From e5d5c59d04942b765789ce2c3f40c7b6b3bdbfb9 Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Mon, 29 Apr 2019 12:27:06 +0000 Subject: [PATCH 7/7] Update dependencies from https://github.com/dotnet/arcade build 20190426.3 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19226.3 --- eng/Version.Details.xml | 4 +- eng/common/SourceLinkValidation.ps1 | 184 ++++++++++++++++++++++++++++ global.json | 2 +- 3 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 eng/common/SourceLinkValidation.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8550113fae4..2eb743ef9f5 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - d37270268a65592cae630f1b979b70f74d4614dd + 7bec23ce3da545d97f53f99abce457a2e252aa58 diff --git a/eng/common/SourceLinkValidation.ps1 b/eng/common/SourceLinkValidation.ps1 new file mode 100644 index 00000000000..cb2d28cb99e --- /dev/null +++ b/eng/common/SourceLinkValidation.ps1 @@ -0,0 +1,184 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $SourceLinkToolPath, # Full path to directory where dotnet SourceLink CLI was installed + [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade + [Parameter(Mandatory=$true)][string] $GHCommit # GitHub commit SHA used to build the packages +) + +# Cache/HashMap (File -> Exist flag) used to consult whether a file exist +# in the repository at a specific commit point. This is populated by inserting +# all files present in the repo at a specific commit point. +$global:RepoFiles = @{} + +$ValidatePackage = { + param( + [string] $PackagePath # Full path to a Symbols.NuGet package + ) + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + throw "Input file does not exist: $PackagePath" + } + + # Extensions for which we'll look for SourceLink information + # For now we'll only care about Portable & Embedded PDBs + $RelevantExtensions = @(".dll", ".exe", ".pdb") + + Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + $FailedFiles = 0 + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $FileName = $_.FullName + $Extension = [System.IO.Path]::GetExtension($_.Name) + $FakeName = -Join((New-Guid), $Extension) + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName + + # We ignore resource DLLs + if ($FileName.EndsWith(".resources.dll")) { + return + } + + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + + $ValidateFile = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $RealPath, + [ref] $FailedFiles + ) + + # Makes easier to reference `sourcelink cli` + Push-Location $using:SourceLinkToolPath + + $SourceLinkInfos = .\sourcelink.exe print-urls $FullPath | Out-String + + if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { + $NumFailedLinks = 0 + + # We only care about Http addresses + $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches + + if ($Matches.Count -ne 0) { + $Matches.Value | + ForEach-Object { + $Link = $_ + $CommitUrl = -Join("https://raw.githubusercontent.com/", $using:GHRepoName, "/", $using:GHCommit, "/") + $FilePath = $Link.Replace($CommitUrl, "") + $Status = 200 + $Cache = $using:RepoFiles + + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + # Only GitHub links are valid + if ($Uri.AbsoluteURI -ne $null -and $Uri.Host -match "github") { + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + $Status = 0 + } + } + catch { + $Status = 0 + } + } + + if ($Status -ne 200) { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ + } + } + } + + if ($NumFailedLinks -ne 0) { + $FailedFiles.value++ + $global:LASTEXITCODE = 1 + } + } + + Pop-Location + } + + &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) + } + + $zip.Dispose() + + if ($FailedFiles -eq 0) { + Write-Host "Passed." + } +} + +function ValidateSourceLinkLinks { + if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { + Write-Host "GHRepoName should be in the format /" + $global:LASTEXITCODE = 1 + return + } + + if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { + Write-Host "GHCommit should be a 40 chars hexadecimal string" + $global:LASTEXITCODE = 1 + return + } + + $RepoTreeURL = -Join("https://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") + $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") + + try { + # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash + $Data = Invoke-WebRequest $RepoTreeURL | ConvertFrom-Json | Select-Object -ExpandProperty tree + + foreach ($file in $Data) { + $Extension = [System.IO.Path]::GetExtension($file.path) + + if ($CodeExtensions.Contains($Extension)) { + $RepoFiles[$file.path] = 1 + } + } + } + catch { + Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" + $global:LASTEXITCODE = 1 + return + } + + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + # Process each NuGet package in parallel + $Jobs = @() + Get-ChildItem "$InputPath\*.symbols.nupkg" | + ForEach-Object { + $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName + } + + foreach ($Job in $Jobs) { + Wait-Job -Id $Job.Id | Receive-Job + } +} + +Measure-Command { ValidateSourceLinkLinks } diff --git a/global.json b/global.json index 394d5b5016f..800de67d7b4 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19225.5", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19226.3", "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2" } }