[main] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -30,9 +30,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.24311.3">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.24367.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>c214b6ad17aedca4fa48294d80f6c52ef2463081</Sha>
+      <Sha>fa3d544b066661522f1ec5d5e8cfd461a29b0f8a</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.DotNet.XliffTasks" Version="1.0.0-beta.23475.1" CoherentParentDependency="Microsoft.DotNet.Arcade.Sdk">

eng/common/post-build/publish-using-darc.ps1

@@ -2,7 +2,6 @@ param(
   [Parameter(Mandatory=$true)][int] $BuildId,
   [Parameter(Mandatory=$true)][int] $PublishingInfraVersion,
   [Parameter(Mandatory=$true)][string] $AzdoToken,
-  [Parameter(Mandatory=$true)][string] $MaestroToken,
   [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
   [Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
   [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
@@ -31,13 +30,13 @@ try {
   }
 
   & $darc add-build-to-channel `
-  --id $buildId `
-  --publishing-infra-version $PublishingInfraVersion `
-  --default-channels `
-  --source-branch main `
-  --azdev-pat $AzdoToken `
-  --bar-uri $MaestroApiEndPoint `
-  --password $MaestroToken `
+    --id $buildId `
+    --publishing-infra-version $PublishingInfraVersion `
+    --default-channels `
+    --source-branch main `
+    --azdev-pat "$AzdoToken" `
+    --bar-uri "$MaestroApiEndPoint" `
+    --ci `
 	@optionalParams
 
   if ($LastExitCode -ne 0) {

eng/common/sdl/NuGet.config

@@ -5,11 +5,11 @@
   </solution>
   <packageSources>
     <clear />
-    <add key="guardian" value="https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json" />
+    <add key="guardian" value="https://pkgs.dev.azure.com/dnceng/_packaging/Guardian1ESPTUpstreamOrgFeed/nuget/v3/index.json" />
   </packageSources>
   <packageSourceMapping>
     <packageSource key="guardian">
-      <package pattern="microsoft.guardian.cli" />
+      <package pattern="Microsoft.Guardian.Cli.win-x64" />
     </packageSource>
   </packageSourceMapping>
   <disabledPackageSources>

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -6,7 +6,6 @@ Param(
   [string] $BranchName=$env:BUILD_SOURCEBRANCH,                                                  # Optional: name of branch or version of gdn settings; defaults to master
   [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,                                         # Required: the directory where source files are located
   [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')),  # Required: the directory where build artifacts are located
-  [string] $AzureDevOpsAccessToken,                                                              # Required: access token for dnceng; should be provided via KeyVault
 
   # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list
   # format.
@@ -75,7 +74,7 @@ try {
   }
 
   Exec-BlockVerbosely {
-    & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
+    & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -GuardianLoggerLevel $GuardianLoggerLevel
   }
   $gdnFolder = Join-Path $workingDirectory '.gdn'
 
@@ -104,7 +103,6 @@ try {
           -TargetDirectory $targetDirectory `
           -GdnFolder $gdnFolder `
           -ToolsList $tools `
-          -AzureDevOpsAccessToken $AzureDevOpsAccessToken `
           -GuardianLoggerLevel $GuardianLoggerLevel `
           -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
           -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams `

eng/common/sdl/init-sdl.ps1

@@ -3,7 +3,6 @@ Param(
   [string] $Repository,
   [string] $BranchName='master',
   [string] $WorkingDirectory,
-  [string] $AzureDevOpsAccessToken,
   [string] $GuardianLoggerLevel='Standard'
 )
 
@@ -21,14 +20,7 @@ $ci = $true
 # Don't display the console progress UI - it's a huge perf hit
 $ProgressPreference = 'SilentlyContinue'
 
-# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file
-$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken"))
-$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn")
-$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0"
-$zipFile = "$WorkingDirectory/gdn.zip"
-
 Add-Type -AssemblyName System.IO.Compression.FileSystem
-$gdnFolder = (Join-Path $WorkingDirectory '.gdn')
 
 try {
   # if the folder does not exist, we'll do a guardian init and push it to the remote repository

eng/common/sdl/sdl.ps1

@@ -4,6 +4,8 @@ function Install-Gdn {
         [Parameter(Mandatory=$true)]
         [string]$Path,
 
+        [string]$Source = "https://pkgs.dev.azure.com/dnceng/_packaging/Guardian1ESPTUpstreamOrgFeed/nuget/v3/index.json",
+
         # If omitted, install the latest version of Guardian, otherwise install that specific version.
         [string]$Version
     )
@@ -19,7 +21,7 @@ function Install-Gdn {
     $ci = $true
     . $PSScriptRoot\..\tools.ps1
 
-    $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache")
+    $argumentList = @("install", "Microsoft.Guardian.Cli.win-x64", "-Source $Source", "-OutputDirectory $Path", "-NonInteractive", "-NoCache")
 
     if ($Version) {
         $argumentList += "-Version $Version"

eng/common/templates-official/job/publish-build-assets.yml

@@ -76,13 +76,16 @@ jobs:
 
     - task: NuGetAuthenticate@1
 
-    - task: PowerShell@2
+    - task: AzureCLI@2
       displayName: Publish Build Assets
       inputs:
-        filePath: eng\common\sdk-task.ps1
-        arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
+        azureSubscription: "Darc: Maestro Production"
+        scriptType: ps
+        scriptLocation: scriptPath
+        scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
+        arguments: >
+          -task PublishBuildAssets -restore -msbuildEngine dotnet
           /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
-          /p:BuildAssetRegistryToken=$(MaestroAccessToken)
           /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
           /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
           /p:OfficialBuildId=$(Build.BuildNumber)
@@ -137,14 +140,16 @@ jobs:
           BARBuildId: ${{ parameters.BARBuildId }}
           PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
 
-      - task: PowerShell@2
+      - task: AzureCLI@2
         displayName: Publish Using Darc
         inputs:
-          filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
-          arguments: -BuildId $(BARBuildId) 
+          azureSubscription: "Darc: Maestro Production"
+          scriptType: ps
+          scriptLocation: scriptPath
+          scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+          arguments: -BuildId $(BARBuildId)
             -PublishingInfraVersion 3
             -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
-            -MaestroToken '$(MaestroApiAccessToken)'
             -WaitPublishingFinish true
             -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
             -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

eng/common/templates-official/job/source-build.yml

@@ -31,6 +31,12 @@ parameters:
   #   container and pool.
   platform: {}
 
+  # If set to true and running on a non-public project,
+  # Internal blob storage locations will be enabled.
+  # This is not enabled by default because many repositories do not need internal sources
+  # and do not need to have the required service connections approved in the pipeline.
+  enableInternalSources: false
+
 jobs:
 - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
   displayName: Source-Build (${{ parameters.platform.name }})
@@ -62,6 +68,8 @@ jobs:
     clean: all
 
   steps:
+  - ${{ if eq(parameters.enableInternalSources, true) }}:
+    - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml
   - template: /eng/common/templates-official/steps/source-build.yml
     parameters:
       platform: ${{ parameters.platform }}

eng/common/templates-official/jobs/source-build.yml

@@ -21,6 +21,12 @@ parameters:
   # one job runs on 'defaultManagedPlatform'.
   platforms: []
 
+  # If set to true and running on a non-public project,
+  # Internal nuget and blob storage locations will be enabled.
+  # This is not enabled by default because many repositories do not need internal sources
+  # and do not need to have the required service connections approved in the pipeline.
+  enableInternalSources: false
+
 jobs:
 
 - ${{ if ne(parameters.allCompletedJobId, '') }}:
@@ -38,9 +44,11 @@ jobs:
     parameters:
       jobNamePrefix: ${{ parameters.jobNamePrefix }}
       platform: ${{ platform }}
+      enableInternalSources: ${{ parameters.enableInternalSources }}
 
 - ${{ if eq(length(parameters.platforms), 0) }}:
   - template: /eng/common/templates-official/job/source-build.yml
     parameters:
       jobNamePrefix: ${{ parameters.jobNamePrefix }}
       platform: ${{ parameters.defaultManagedPlatform }}
+      enableInternalSources: ${{ parameters.enableInternalSources }}

eng/common/templates-official/post-build/post-build.yml

@@ -272,14 +272,16 @@ stages:
 
         - task: NuGetAuthenticate@1
 
-        - task: PowerShell@2
+        - task: AzureCLI@2
           displayName: Publish Using Darc
           inputs:
-            filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+            azureSubscription: "Darc: Maestro Production"
+            scriptType: ps
+            scriptLocation: scriptPath
+            scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
             arguments: -BuildId $(BARBuildId) 
               -PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
               -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
-              -MaestroToken '$(MaestroApiAccessToken)'
               -WaitPublishingFinish true
               -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
               -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

eng/common/templates-official/steps/enable-internal-runtimes.yml

@@ -0,0 +1,28 @@
+# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64'
+# variable with the base64-encoded SAS token, by default
+
+parameters:
+- name: federatedServiceConnection
+  type: string
+  default: 'dotnetbuilds-internal-read'
+- name: outputVariableName
+  type: string
+  default: 'dotnetbuilds-internal-container-read-token-base64'
+- name: expiryInHours
+  type: number
+  default: 1
+- name: base64Encode
+  type: boolean
+  default: true
+
+steps:
+- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+  - template: /eng/common/templates-official/steps/get-delegation-sas.yml
+    parameters:
+      federatedServiceConnection: ${{ parameters.federatedServiceConnection }}
+      outputVariableName: ${{ parameters.outputVariableName }}
+      expiryInHours: ${{ parameters.expiryInHours }}
+      base64Encode: ${{ parameters.base64Encode }}
+      storageAccount: dotnetbuilds
+      container: internal
+      permissions: rl

eng/common/templates-official/steps/execute-sdl.yml

@@ -9,8 +9,6 @@ parameters:
 
 steps:
 - task: NuGetAuthenticate@1
-  inputs:
-    nuGetServiceConnections: GuardianConnect
 
 - task: NuGetToolInstaller@1
   displayName: 'Install NuGet.exe'

eng/common/templates-official/steps/get-delegation-sas.yml

@@ -0,0 +1,43 @@
+parameters:
+- name: federatedServiceConnection
+  type: string
+- name: outputVariableName
+  type: string
+- name: expiryInHours
+  type: number
+  default: 1
+- name: base64Encode
+  type: boolean
+  default: false
+- name: storageAccount
+  type: string
+- name: container
+  type: string
+- name: permissions
+  type: string
+  default: 'rl'
+
+steps:
+- task: AzureCLI@2
+  displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}'
+  inputs:
+    azureSubscription: ${{ parameters.federatedServiceConnection }}
+    scriptType: 'pscore'
+    scriptLocation: 'inlineScript'
+    inlineScript: |
+      # Calculate the expiration of the SAS token and convert to UTC
+      $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
+
+      $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+
+      if ($LASTEXITCODE -ne 0) {
+        Write-Error "Failed to generate SAS token."
+        exit 1
+      }
+
+      if ('${{ parameters.base64Encode }}' -eq 'true') {
+        $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas))
+      }
+
+      Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+      Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas"

eng/common/templates-official/steps/get-federated-access-token.yml

@@ -0,0 +1,40 @@
+parameters:
+- name: federatedServiceConnection
+  type: string
+- name: outputVariableName
+  type: string
+- name: stepName
+  type: string
+  default: 'getFederatedAccessToken'
+- name: condition
+  type: string
+  default: ''
+# Resource to get a token for. Common values include:
+# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps
+# - 'https://storage.azure.com/' for storage
+# Defaults to Azure DevOps
+- name: resource
+  type: string
+  default: '499b84ac-1321-427f-aa17-267ca6975798'
+- name: isStepOutputVariable
+  type: boolean
+  default: false
+
+steps:
+- task: AzureCLI@2
+  displayName: 'Getting federated access token for feeds'
+  name: ${{ parameters.stepName }}
+  ${{ if ne(parameters.condition, '') }}:
+    condition: ${{ parameters.condition }}
+  inputs:
+    azureSubscription: ${{ parameters.federatedServiceConnection }}
+    scriptType: 'pscore'
+    scriptLocation: 'inlineScript'
+    inlineScript: |
+      $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv
+      if ($LASTEXITCODE -ne 0) {
+        Write-Error "Failed to get access token for resource '${{ parameters.resource }}'"
+        exit 1
+      }
+      Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+      Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken"

eng/common/templates/job/publish-build-assets.yml

@@ -74,13 +74,16 @@ jobs:
 
     - task: NuGetAuthenticate@1
 
-    - task: PowerShell@2
+    - task: AzureCLI@2
       displayName: Publish Build Assets
       inputs:
-        filePath: eng\common\sdk-task.ps1
-        arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
+        azureSubscription: "Darc: Maestro Production"
+        scriptType: ps
+        scriptLocation: scriptPath
+        scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
+        arguments: >
+          -task PublishBuildAssets -restore -msbuildEngine dotnet
           /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
-          /p:BuildAssetRegistryToken=$(MaestroAccessToken)
           /p:MaestroApiEndpoint=https://maestro.dot.net
           /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
           /p:OfficialBuildId=$(Build.BuildNumber)
@@ -133,14 +136,16 @@ jobs:
           BARBuildId: ${{ parameters.BARBuildId }}
           PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
 
-      - task: PowerShell@2
+      - task: AzureCLI@2
         displayName: Publish Using Darc
         inputs:
-          filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
-          arguments: -BuildId $(BARBuildId)
+          azureSubscription: "Darc: Maestro Production"
+          scriptType: ps
+          scriptLocation: scriptPath
+          scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+          arguments: -BuildId $(BARBuildId) 
             -PublishingInfraVersion 3
             -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
-            -MaestroToken '$(MaestroApiAccessToken)'
             -WaitPublishingFinish true
             -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
             -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'