Skip to content

Commit 4dfc0a7

Browse files
dougbudougbu
authored
Do less copying / zipping for Helix (#40134)
- use .dotnet/ folders instead of App.Ref/App.Runtime packages for dotnet-cli layouts - no need to unzip files into another layout - fail fast or emit message when layouts don't exist - set `$(TestDependsOnAspNetPackages)` to `false` by default - no need to copy all packages into publish/ folders of most test projects - upload very few times per queue (2 today, maybe 4 in the future), not for _every_ work item - reduce the size of most work item payloads, saving bandwidth and time - reduction should not cause problems, even around major version and TFM changes (see .dotnet/ folders) - add `$(TestDependsOnAspNetAppPackages)` for the App.UnitTests case; just 2 packages needed there - get App.UnitTests working in local builds - grab RuntimeList.xml file from App.Runtime's obj/ folder - find needed packages whether running locally or on Helix agents; should always exist nits: - remove test infrastructure we don't need anymore - use one property for shared Fx and targeting pack versions; always the same - remove `$env:ASPNET_RUNTIME_PATH` - use `$(ArtifactsShippingPackagesDir)` in Helix.targets - use `[MSBuild]::NormalizeDirectory(...)` and `$([System.IO.Path]::Combine(...)` in helix.proj - rename silly ...ListListsContains... tests - remove extra `$(TestDependsOnAspNetXyz)` settings to their new default values - add more comments
1 parent cd678f8 commit 4dfc0a7

File tree

12 files changed

+111
-101
lines changed

12 files changed

+111
-101
lines changed

‎eng/helix/content/RunTests/TestRunner.cs‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,6 @@ public bool SetupEnvironment()
4343
var dotnetEFFullPath = Path.Combine(nugetRestore, helixDir, "dotnet-ef.exe");
4444
Console.WriteLine($"Set DotNetEfFullPath: {dotnetEFFullPath}");
4545
EnvironmentVariables.Add("DotNetEfFullPath", dotnetEFFullPath);
46-
var appRuntimePath = $"{Options.DotnetRoot}/shared/Microsoft.AspNetCore.App/{Options.RuntimeVersion}";
47-
Console.WriteLine($"Set ASPNET_RUNTIME_PATH: {appRuntimePath}");
48-
EnvironmentVariables.Add("ASPNET_RUNTIME_PATH", appRuntimePath);
4946
var dumpPath = Environment.GetEnvironmentVariable("HELIX_DUMP_FOLDER");
5047
Console.WriteLine($"Set VSTEST_DUMP_PATH: {dumpPath}");
5148
EnvironmentVariables.Add("VSTEST_DUMP_PATH", dumpPath);

‎eng/helix/helix.proj‎

Lines changed: 43 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@
3535
<DotNetCliChannel>Current</DotNetCliChannel>
3636

3737
<!-- Similar to ProjectLayout.props in the Arcade SDK. The Helix SDK contains nothing similar. -->
38-
<OutputPath Condition=" '$(OutputPath)' == '' ">$(RepoRoot)artifacts\bin\$(MSBuildProjectName)\</OutputPath>
38+
<OutputPath Condition=" '$(OutputPath)' == '' "
39+
>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', '$(MSBuildProjectName)'))</OutputPath>
3940
</PropertyGroup>
4041

4142
<!-- Specify the .NET runtime we need which will be included as a correlation payload. -->
@@ -52,7 +53,7 @@
5253
<When Condition=" '$(ContinuousIntegrationBuild)' == 'true' ">
5354
<PropertyGroup>
5455
<!-- Creator is not valid for internal queues -->
55-
<Creator Condition="'$(_UseHelixOpenQueues)' == 'true'">aspnetcore</Creator>
56+
<Creator Condition=" '$(_UseHelixOpenQueues)' == 'true' ">aspnetcore</Creator>
5657

5758
<HelixType>ci</HelixType>
5859
<HelixBuild>$(BUILD_BUILDNUMBER).$(SYSTEM_JOBATTEMPT)</HelixBuild>
@@ -61,8 +62,8 @@
6162
<Otherwise>
6263
<PropertyGroup>
6364
<!-- Creator is not valid for internal queues -->
64-
<Creator Condition="'$(_UseHelixOpenQueues)' == 'true'">$(USERNAME)</Creator>
65-
<Creator Condition="'$(USERNAME)' == '' AND '$(_UseHelixOpenQueues)' == 'true'">$(USER)</Creator>
65+
<Creator Condition=" '$(_UseHelixOpenQueues)' == 'true' ">$(USERNAME)</Creator>
66+
<Creator Condition=" '$(_UseHelixOpenQueues)' == 'true' AND '$(USERNAME)' == '' ">$(USER)</Creator>
6667

6768
<HelixType>dev</HelixType>
6869
<HelixBuild>$([System.DateTime]::Now.ToString('yyyyMMddHHmm'))</HelixBuild>
@@ -83,9 +84,12 @@
8384
<HelixProperties Condition="'$(RunQuarantinedTests)' != 'true'" Include="runType" Value="unquarantined" />
8485
</ItemGroup>
8586

86-
<Target Name="IncludeAspNetRuntime" BeforeTargets="Gather"
87-
Condition="'$(DoNotRequireSharedFxHelix)' != 'true' OR
88-
EXISTS('$(RepoRoot)artifacts\packages\$(Configuration)\Shipping\Microsoft.AspNetCore.App.Runtime.$(TargetRuntimeIdentifier).$(SharedFxVersion).nupkg')">
87+
<!--
88+
In inner (per-queue) builds, tell the Helix SDK to pack up our just-built App.Ref / App.Runtime layouts w/
89+
the current versions. Warning: When using RunHelix.ps1, .dotnet/ folder may contain older App.Ref and
90+
App.Runtime layouts.
91+
-->
92+
<Target Name="IncludeAspNetRuntime" BeforeTargets="Gather">
8993
<MSBuild Projects="$(RepoRoot)src\Framework\App.Runtime\src\Microsoft.AspNetCore.App.Runtime.csproj"
9094
Properties="DisableYarnCheck=true;ExcludeFromBuild=false"
9195
Targets="_GetPackageVersionInfo">
@@ -94,32 +98,40 @@
9498

9599
<PropertyGroup>
96100
<SharedFxVersion>@(_ResolvedProductVersionInfo->'%(PackageVersion)')</SharedFxVersion>
101+
<_AppRefSubPath>$([System.IO.Path]::Combine('packs', 'Microsoft.AspNetCore.App.Ref', '$(SharedFxVersion)'))</_AppRefSubPath>
102+
<_AppRuntimeSubPath>$([System.IO.Path]::Combine('shared', 'Microsoft.AspNetCore.App', '$(SharedFxVersion)'))</_AppRuntimeSubPath>
103+
<_AppRefPath>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', '.dotnet', '$(_AppRefSubPath)'))</_AppRefPath>
104+
<_AppRuntimePath>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', '.dotnet', '$(_AppRuntimeSubPath)'))</_AppRuntimePath>
105+
106+
<!--
107+
Don't bother checking for App.Runtime layout because we want both and App.Ref project depends on the
108+
App.Runtime project.
109+
-->
110+
<_AppRefPathExists Condition=" EXISTS('$(_AppRefPath)') ">true</_AppRefPathExists>
111+
<_AppRefPathExists Condition=" '$(_AppRefPathExists)' == '' ">false</_AppRefPathExists>
97112
</PropertyGroup>
98113

99-
<!-- Use package because .dotnet/ folder doesn't contain RuntimeList.xml file and SharedFxTests checks that. -->
100-
<Unzip
101-
SourceFiles="$(RepoRoot)artifacts\packages\$(Configuration)\Shipping\Microsoft.AspNetCore.App.Runtime.$(TargetRuntimeIdentifier).$(SharedFxVersion).nupkg"
102-
DestinationFolder="$(OutputPath)\SharedFx.Unzip" />
103-
<ItemGroup>
104-
<_appRuntimeFiles Include="$(OutputPath)\SharedFx.Unzip\**\*.txt" />
105-
<_appRuntimeFiles Include="$(OutputPath)\SharedFx.Unzip\**\*.json" />
106-
<_appRuntimeFiles Include="$(OutputPath)\SharedFx.Unzip\**\*.dll" />
107-
<_appRuntimeFiles Include="$(OutputPath)\SharedFx.Unzip\**\RuntimeList.xml" />
108-
</ItemGroup>
109-
<Copy SourceFiles="@(_appRuntimeFiles)"
110-
DestinationFolder="$(OutputPath)\SharedFx.Layout\shared\Microsoft.AspNetCore.App\$(SharedFxVersion)" />
111-
<Copy SourceFiles="$(OutputPath)\SharedFx.Unzip\Microsoft.AspNetCore.App.versions.txt"
112-
DestinationFiles="$(OutputPath)\SharedFx.Layout\shared\Microsoft.AspNetCore.App\$(SharedFxVersion)\.version" />
113-
114-
<Unzip Condition="Exists('$(RepoRoot)artifacts\packages\$(Configuration)\Shipping\Microsoft.AspNetCore.App.Ref.$(SharedFxVersion).nupkg')"
115-
SourceFiles="$(RepoRoot)artifacts\packages\$(Configuration)\Shipping\Microsoft.AspNetCore.App.Ref.$(SharedFxVersion).nupkg"
116-
DestinationFolder="$(OutputPath)\SharedFx.Layout\packs\Microsoft.AspNetCore.App.Ref\$(SharedFxVersion)" />
117-
<ZipDirectory Condition="Exists('$(OutputPath)\SharedFx.Layout')"
118-
SourceDirectory="$(OutputPath)\SharedFx.Layout"
119-
DestinationFile="$(OutputPath)\SharedFx.Layout.zip" Overwrite="true" />
120-
121-
<ItemGroup>
122-
<HelixCorrelationPayload Include="$(OutputPath)\SharedFx.Layout.zip" Destination="$(DotNetCliDestination)" />
114+
<!-- After complaining about missing App.Ref layout, do nothing else in this target. -->
115+
<Message Importance="High"
116+
Text="Targeting path location '$(_AppRefPath)' does not exist and tests will use the SDK's bundled runtimes and targeting pack."
117+
Condition=" !$(_AppRefPathExists) AND '$(DoNotRequireSharedFxHelix)' == true " />
118+
<Error Text="Targeting path location '$(_AppRefPath)' does not exist and some tests are guaranteed to fail."
119+
Condition=" !$(_AppRefPathExists) AND '$(DoNotRequireSharedFxHelix)' != true " />
120+
121+
<!-- Grab only the portions of the .dotnet/ tree built in this run and have Helix zip that up. -->
122+
<ItemGroup Condition=" $(_AppRefPathExists) ">
123+
<HelixCorrelationPayload Include="$(_AppRefPath)"
124+
Condition=" $(IsWindowsHelixQueue) "
125+
Destination="$(DotNetCliDestination)\$(_AppRefSubPath)" />
126+
<HelixCorrelationPayload Include="$(_AppRefPath)"
127+
Condition=" !$(IsWindowsHelixQueue) "
128+
Destination="$(DotNetCliDestination)/$(_AppRefSubPath.Replace('\','/'))" />
129+
<HelixCorrelationPayload Include="$(_AppRuntimePath)"
130+
Condition=" $(IsWindowsHelixQueue) "
131+
Destination="$(DotNetCliDestination)\$(_AppRuntimeSubPath)" />
132+
<HelixCorrelationPayload Include="$(_AppRuntimePath)"
133+
Condition=" !$(IsWindowsHelixQueue) "
134+
Destination="$(DotNetCliDestination)/$(_AppRuntimeSubPath.Replace('\','/'))" />
123135
</ItemGroup>
124136
</Target>
125137

‎eng/targets/Helix.props‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,12 @@
1414
<LoggingTestingDisableFileLogging Condition="'$(IsHelixJob)' == 'true'">false</LoggingTestingDisableFileLogging>
1515
<NodeVersion>16.11.0</NodeVersion>
1616

17-
<!-- Have all tests depend on the latest runtimes until we get a net7.0 SDK -->
18-
<TestDependsOnAspNetPackages>true</TestDependsOnAspNetPackages>
17+
<!--
18+
Many tests depend on the .dotnet/ layouts but only a few (those that restore or test packages) require
19+
packages. This should be sufficient for most test projects even as we switch to new major versions or TFMs.
20+
-->
21+
<TestDependsOnAspNetPackages>false</TestDependsOnAspNetPackages>
22+
<TestDependsOnAspNetAppPackages>false</TestDependsOnAspNetAppPackages>
1923
<TestDependsOnAspNetRuntime>true</TestDependsOnAspNetRuntime>
2024
</PropertyGroup>
2125

‎eng/targets/Helix.targets‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,13 +47,24 @@
4747
<HelixPreCommand Include="call RunPowershell.cmd InstallNode.ps1 $(NodeVersion) || exit /b 1" />
4848
</ItemGroup>
4949

50-
<!-- $(TestDependsOnAspNetRuntime) implies $(TestDependsOnAspNetPackages). Separate for the App.UnitTests case. -->
51-
<PropertyGroup Condition=" $(TestDependsOnAspNetRuntime) AND !$(TestDependsOnAspNetPackages) ">
52-
<TestDependsOnAspNetPackages>true</TestDependsOnAspNetPackages>
50+
<!-- $(TestDependsOnAspNetPackages) implies $(TestDependsOnAspNetRuntime). -->
51+
<PropertyGroup Condition=" $(TestDependsOnAspNetPackages) AND !$(TestDependsOnAspNetRuntime) ">
52+
<TestDependsOnAspNetRuntime>true</TestDependsOnAspNetRuntime>
5353
</PropertyGroup>
54-
<ItemGroup Condition=" $(TestDependsOnAspNetPackages) ">
54+
55+
<!--
56+
Note wildcards mean work items may be submitted without necessary packages when using RunHelix.ps1. Without
57+
the packages, the RunHelix.ps1 build won't fail fast and tests will fail on the Helix agents.
58+
Similarly, the artifacts/packages/ folder may contain old packages e.g. some built on
59+
another local branch when using RunHelix.ps1, invalidating tests to some extent.
60+
-->
61+
<ItemGroup>
5562
<!-- Grab all shipping packages. -->
56-
<HelixContent Include="$(RepoRoot)artifacts\packages\$(Configuration)\Shipping\*$(SharedFxVersion).nupkg" />
63+
<HelixContent Include="$(ArtifactsShippingPackagesDir)*$(SharedFxVersion).nupkg"
64+
Condition=" $(TestDependsOnAspNetPackages) "/>
65+
<!-- Grab just the App.Ref and App.Runtime packages. -->
66+
<HelixContent Include="$(ArtifactsShippingPackagesDir)Microsoft.AspNetCore.App.*$(SharedFxVersion).nupkg"
67+
Condition=" $(TestDependsOnAspNetAppPackages) "/>
5768
</ItemGroup>
5869

5970
<ItemGroup>

‎src/Framework/test/Microsoft.AspNetCore.App.UnitTests.csproj‎

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
<Project Sdk="Microsoft.NET.Sdk">
2-
32
<PropertyGroup>
43
<TargetFramework>$(DefaultNetCoreTargetFramework)</TargetFramework>
54
<RootNamespace>Microsoft.AspNetCore</RootNamespace>
65
<VerifyAncmBinary Condition="'$(TargetOsName)' == 'win' AND '$(TargetArchitecture)' != 'arm'">true</VerifyAncmBinary>
7-
<TestDependsOnAspNetPackages>true</TestDependsOnAspNetPackages>
6+
<TestDependsOnAspNetAppPackages>true</TestDependsOnAspNetAppPackages>
87
<SkipHelixArm>true</SkipHelixArm>
98
</PropertyGroup>
109

1110
<ItemGroup>
11+
<!-- Grab RuntimeList.xml because it's not placed in the .dotnet/ folder. -->
12+
<None Include="$(ArtifactsObjDir)Microsoft.AspNetCore.App.Runtime\RuntimeList.xml"
13+
CopyToOutputDirectory="PreserveNewest" />
14+
1215
<!-- Ignore aspnetcorev2_inprocess because tests expect only managed assemblies in this item group. -->
1316
<_SharedFrameworkBinariesFromRepo Include="@(AspNetCoreAppReference);@(AspNetCoreAppReferenceAndPackage)" />
1417

@@ -47,18 +50,14 @@
4750
<_Parameter1>SharedFxDependencies</_Parameter1>
4851
<_Parameter2>@(_ExpectedSharedFrameworkBinaries)</_Parameter2>
4952
</AssemblyAttribute>
50-
<AssemblyAttribute Include="Microsoft.AspNetCore.TestData">
51-
<_Parameter1>SharedFrameworkLayoutRoot</_Parameter1>
52-
<_Parameter2>$(SharedFrameworkLayoutRoot)</_Parameter2>
53-
</AssemblyAttribute>
54-
<AssemblyAttribute Include="Microsoft.AspNetCore.TestData">
55-
<_Parameter1>TargetingPackLayoutRoot</_Parameter1>
56-
<_Parameter2>$(TargetingPackLayoutRoot)</_Parameter2>
57-
</AssemblyAttribute>
5853
<AssemblyAttribute Include="Microsoft.AspNetCore.TestData">
5954
<_Parameter1>VerifyAncmBinary</_Parameter1>
6055
<_Parameter2>$(VerifyAncmBinary)</_Parameter2>
6156
</AssemblyAttribute>
57+
<AssemblyAttribute Include="Microsoft.AspNetCore.TestData">
58+
<_Parameter1>PackagesFolder</_Parameter1>
59+
<_Parameter2>$(ArtifactsShippingPackagesDir)</_Parameter2>
60+
</AssemblyAttribute>
6261
</ItemGroup>
6362

6463
<ItemGroup>
@@ -112,17 +111,6 @@
112111
<_Parameter1>RepositoryCommit</_Parameter1>
113112
<_Parameter2>$(SourceRevisionId)</_Parameter2>
114113
</AssemblyAttribute>
115-
116-
<AssemblyAttribute Include="Microsoft.AspNetCore.TestData">
117-
<_Parameter1>RuntimePackageVersion</_Parameter1>
118-
<_Parameter2>$(SharedFxVersion)</_Parameter2>
119-
</AssemblyAttribute>
120-
121-
<AssemblyAttribute Include="Microsoft.AspNetCore.TestData">
122-
<_Parameter1>TargetingPackVersion</_Parameter1>
123-
<_Parameter2>$(TargetingPackVersion)</_Parameter2>
124-
</AssemblyAttribute>
125114
</ItemGroup>
126115
</Target>
127-
128116
</Project>

‎src/Framework/test/SharedFxTests.cs‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Reflection.Metadata;
66
using System.Reflection.PortableExecutable;
77
using System.Xml.Linq;
8+
using Microsoft.AspNetCore.Testing;
89
using Newtonsoft.Json.Linq;
910
using Xunit.Abstractions;
1011

@@ -14,7 +15,6 @@ public class SharedFxTests
1415
{
1516
private readonly string _expectedTfm;
1617
private readonly string _expectedRid;
17-
private readonly string _expectedVersionFileName;
1818
private readonly string _sharedFxRoot;
1919
private readonly ITestOutputHelper _output;
2020

@@ -23,12 +23,11 @@ public SharedFxTests(ITestOutputHelper output)
2323
_output = output;
2424
_expectedTfm = TestData.GetDefaultNetCoreTargetFramework();
2525
_expectedRid = TestData.GetSharedFxRuntimeIdentifier();
26-
_sharedFxRoot = string.IsNullOrEmpty(Environment.GetEnvironmentVariable("ASPNET_RUNTIME_PATH"))
27-
? Path.Combine(TestData.GetTestDataValue("SharedFrameworkLayoutRoot"), "shared", "Microsoft.AspNetCore.App", TestData.GetTestDataValue("RuntimePackageVersion"))
28-
: Environment.GetEnvironmentVariable("ASPNET_RUNTIME_PATH");
29-
_expectedVersionFileName = string.IsNullOrEmpty(Environment.GetEnvironmentVariable("ASPNET_RUNTIME_PATH"))
30-
? ".version"
31-
: "Microsoft.AspNetCore.App.versions.txt";
26+
_sharedFxRoot = Path.Combine(
27+
Environment.GetEnvironmentVariable("DOTNET_ROOT"),
28+
"shared",
29+
"Microsoft.AspNetCore.App",
30+
TestData.GetSharedFxVersion());
3231
}
3332

3433
[Fact]
@@ -106,7 +105,7 @@ public void SharedFrameworkContainsValidDepsJson()
106105

107106
var target = $".NETCoreApp,Version=v{_expectedTfm.Substring(3)}/{_expectedRid}";
108107
var ridPackageId = $"Microsoft.AspNetCore.App.Runtime.{_expectedRid}";
109-
var libraryId = $"{ridPackageId}/{TestData.GetTestDataValue("RuntimePackageVersion")}";
108+
var libraryId = $"{ridPackageId}/{TestData.GetSharedFxVersion()}";
110109

111110
AssertEx.FileExists(depsFilePath);
112111

@@ -228,22 +227,22 @@ public void SharedFrameworkAssemblyReferencesHaveExpectedAssemblyVersions()
228227
[Fact]
229228
public void ItContainsVersionFile()
230229
{
231-
var versionFile = Path.Combine(_sharedFxRoot, _expectedVersionFileName);
230+
var versionFile = Path.Combine(_sharedFxRoot, ".version");
232231
AssertEx.FileExists(versionFile);
233232
var lines = File.ReadAllLines(versionFile);
234233
Assert.Equal(2, lines.Length);
235234
Assert.Equal(TestData.GetRepositoryCommit(), lines[0]);
236-
Assert.Equal(TestData.GetTestDataValue("RuntimePackageVersion"), lines[1]);
235+
Assert.Equal(TestData.GetSharedFxVersion(), lines[1]);
237236
}
238237

239238
[Fact]
240-
public void RuntimeListListsContainsCorrectEntries()
239+
public void RuntimeListContainsCorrectEntries()
241240
{
242241
var expectedAssemblies = TestData.GetSharedFxDependencies()
243242
.Split(';', StringSplitOptions.RemoveEmptyEntries)
244243
.ToHashSet();
245244

246-
var runtimeListPath = Path.Combine(_sharedFxRoot, "RuntimeList.xml");
245+
var runtimeListPath = "RuntimeList.xml";
247246
AssertEx.FileExists(runtimeListPath);
248247

249248
var runtimeListDoc = XDocument.Load(runtimeListPath);
@@ -294,15 +293,21 @@ public void RuntimeListListsContainsCorrectEntries()
294293
}
295294

296295
[Fact]
297-
public void RuntimeListListsContainsCorrectPaths()
296+
public void RuntimeListContainsCorrectPaths()
298297
{
299-
var runtimeListPath = Path.Combine(_sharedFxRoot, "RuntimeList.xml");
298+
var runtimeListPath = "RuntimeList.xml";
300299
AssertEx.FileExists(runtimeListPath);
301300

302301
var runtimeListDoc = XDocument.Load(runtimeListPath);
303302
var runtimeListEntries = runtimeListDoc.Root.Descendants();
304303

305-
var sharedFxPath = Path.Combine(Environment.GetEnvironmentVariable("HELIX_WORKITEM_ROOT"), ("Microsoft.AspNetCore.App.Runtime.win-x64." + TestData.GetSharedFxVersion() + ".nupkg"));
304+
var packageFolder = SkipOnHelixAttribute.OnHelix() ?
305+
Environment.GetEnvironmentVariable("HELIX_WORKITEM_ROOT") :
306+
TestData.GetPackagesFolder();
307+
var sharedFxPath = Path.Combine(
308+
packageFolder,
309+
"Microsoft.AspNetCore.App.Runtime.win-x64." + TestData.GetSharedFxVersion() + ".nupkg");
310+
AssertEx.FileExists(sharedFxPath);
306311

307312
ZipArchive archive = ZipFile.OpenRead(sharedFxPath);
308313

0 commit comments

Comments
 (0)