Capture dump of hanging test process in Helix (#21659)

BrennanConroy · web-flow · commit f56c61b812ac · 2020-05-12T13:55:09.000-07:00
diff --git a/eng/helix/content/RunTests/ProcessUtil.cs b/eng/helix/content/RunTests/ProcessUtil.cs
@@ -5,6 +5,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading;
@@ -19,10 +20,53 @@ public static class ProcessUtil
         [DllImport("libc", SetLastError = true, EntryPoint = "kill")]
         private static extern int sys_kill(int pid, int sig);
 
+        public static Task CaptureDumpAsync()
+        {
+            var dumpDirectoryPath = Environment.GetEnvironmentVariable("HELIX_DUMP_FOLDER");
+
+            if (dumpDirectoryPath == null)
+            {
+                return Task.CompletedTask;
+            }
+
+            var process = Process.GetCurrentProcess();
+            var dumpFilePath = Path.Combine(dumpDirectoryPath, $"{process.ProcessName}-{process.Id}.dmp");
+
+            return CaptureDumpAsync(process.Id, dumpFilePath);
+        }
+
+        public static Task CaptureDumpAsync(int pid)
+        {
+            var dumpDirectoryPath = Environment.GetEnvironmentVariable("HELIX_DUMP_FOLDER");
+
+            if (dumpDirectoryPath == null)
+            {
+                return Task.CompletedTask;
+            }
+
+            var process = Process.GetProcessById(pid);
+            var dumpFilePath = Path.Combine(dumpDirectoryPath, $"{process.ProcessName}.{process.Id}.dmp");
+
+            return CaptureDumpAsync(process.Id, dumpFilePath);
+        }
+
+        public static Task CaptureDumpAsync(int pid, string dumpFilePath)
+        {
+            // Skip this on OSX, we know it's unsupported right now
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+            {
+                // Can we capture stacks or do a gcdump instead?
+                return Task.CompletedTask;
+            }
+
+            return RunAsync($"{Environment.GetEnvironmentVariable("HELIX_WORKITEM_ROOT")}/dotnet-dump", $"collect -p {pid} -o \"{dumpFilePath}\"");
+        }
+
         public static async Task<ProcessResult> RunAsync(
             string filename,
             string arguments,
             string? workingDirectory = null,
+            string dumpDirectoryPath = null,
             bool throwOnError = true,
             IDictionary<string, string?>? environmentVariables = null,
             Action<string>? outputDataReceived = null,
@@ -51,6 +95,14 @@ public static async Task<ProcessResult> RunAsync(
                 process.StartInfo.WorkingDirectory = workingDirectory;
             }
 
+            dumpDirectoryPath ??= Environment.GetEnvironmentVariable("HELIX_DUMP_FOLDER");
+
+            if (dumpDirectoryPath != null)
+            {
+                process.StartInfo.EnvironmentVariables["COMPlus_DbgEnableMiniDump"] = "1";
+                process.StartInfo.EnvironmentVariables["COMPlus_DbgMiniDumpName"] = Path.Combine(dumpDirectoryPath, $"{Path.GetFileName(filename)}.%d.dmp");
+            }
+
             if (environmentVariables != null)
             {
                 foreach (var kvp in environmentVariables)
@@ -112,13 +164,20 @@ public static async Task<ProcessResult> RunAsync(
             process.BeginOutputReadLine();
             process.BeginErrorReadLine();
 
-            var cancelledTcs = new TaskCompletionSource<object?>();
-            await using var _ = cancellationToken.Register(() => cancelledTcs.TrySetResult(null));
+            var canceledTcs = new TaskCompletionSource<object?>();
+            await using var _ = cancellationToken.Register(() => canceledTcs.TrySetResult(null));
 
-            var result = await Task.WhenAny(processLifetimeTask.Task, cancelledTcs.Task);
+            var result = await Task.WhenAny(processLifetimeTask.Task, canceledTcs.Task);
 
-            if (result == cancelledTcs.Task)
+            if (result == canceledTcs.Task)
             {
+                if (dumpDirectoryPath != null)
+                {
+                    var dumpFilePath = Path.Combine(dumpDirectoryPath, $"{Path.GetFileName(filename)}.{process.Id}.dmp");
+                    // Capture a process dump if the dumpDirectory is set
+                    await CaptureDumpAsync(process.Id, dumpFilePath);
+                }
+
                 if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                 {
                     sys_kill(process.Id, sig: 2); // SIGINT
@@ -143,16 +202,5 @@ public static async Task<ProcessResult> RunAsync(
 
             return await processLifetimeTask.Task;
         }
-
-        public static void KillProcess(int pid)
-        {
-            try
-            {
-                using var process = Process.GetProcessById(pid);
-                process?.Kill();
-            }
-            catch (ArgumentException) { }
-            catch (InvalidOperationException) { }
-        }
     }
 }
diff --git a/eng/helix/content/RunTests/Program.cs b/eng/helix/content/RunTests/Program.cs
@@ -2,10 +2,6 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.Collections.Generic;
-using System.CommandLine;
-using System.IO;
-using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 
 namespace RunTests
@@ -14,7 +10,7 @@ class Program
     {
         static async Task Main(string[] args)
         {
-            try 
+            try
             {
                 var runner = new TestRunner(RunTestsOptions.Parse(args));
 
@@ -27,6 +23,10 @@ static async Task Main(string[] args)
                 {
                     keepGoing = runner.InstallAspNetRefIfNeeded();
                 }
+                if (keepGoing)
+                {
+                    keepGoing = await runner.InstallDotnetDump();
+                }
 
                 runner.DisplayContents();
 
diff --git a/eng/helix/content/RunTests/RunTestsOptions.cs b/eng/helix/content/RunTests/RunTestsOptions.cs
@@ -50,16 +50,21 @@ public static RunTestsOptions Parse(string[] args)
                     aliases: new string[] { "--ef" },
                     description: "The version of the EF tool to use")
                 { Argument = new Argument<string>(), Required = true },
-                
+
                 new Option(
                     aliases: new string[] { "--aspnetruntime" },
                     description: "The path to the aspnet runtime nupkg to install")
                 { Argument = new Argument<string>(), Required = true },
-                
+
                 new Option(
                     aliases: new string[] { "--aspnetref" },
                     description: "The path to the aspnet ref nupkg to install")
                 { Argument = new Argument<string>(), Required = true },
+
+                new Option(
+                    aliases: new string[] { "--helixTimeout" },
+                    description: "The timeout duration of the Helix job")
+                { Argument = new Argument<string>(), Required = true },
             };
 
             var parseResult = command.Parse(args);
@@ -73,6 +78,7 @@ public static RunTestsOptions Parse(string[] args)
             options.EfVersion = parseResult.ValueForOption<string>("--ef");
             options.AspNetRuntime = parseResult.ValueForOption<string>("--aspnetruntime");
             options.AspNetRef = parseResult.ValueForOption<string>("--aspnetref");
+            options.Timeout = TimeSpan.Parse(parseResult.ValueForOption<string>("--helixTimeout"));
             options.HELIX_WORKITEM_ROOT = Environment.GetEnvironmentVariable("HELIX_WORKITEM_ROOT");
             options.Path = Environment.GetEnvironmentVariable("PATH");
             options.DotnetRoot = Environment.GetEnvironmentVariable("DOTNET_ROOT");
@@ -91,5 +97,6 @@ public static RunTestsOptions Parse(string[] args)
         public string HELIX_WORKITEM_ROOT { get; set;}
         public string DotnetRoot { get; set; }
         public string Path { get; set; }
+        public TimeSpan Timeout { get; set; }
     }
 }
diff --git a/eng/helix/content/RunTests/TestRunner.cs b/eng/helix/content/RunTests/TestRunner.cs
@@ -3,10 +3,10 @@
 
 using System;
 using System.Collections.Generic;
-using System.CommandLine;
 using System.IO;
 using System.IO.Compression;
 using System.Runtime.InteropServices;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace RunTests
@@ -24,15 +24,15 @@ public TestRunner(RunTestsOptions options)
 
         public bool SetupEnvironment()
         {
-            try 
+            try
             {
                 // Rename default.NuGet.config to NuGet.config if there is not a custom one from the project
                 // We use a local NuGet.config file to avoid polluting global machine state and avoid relying on global machine state
                 if (!File.Exists("NuGet.config"))
                 {
                     File.Copy("default.NuGet.config", "NuGet.config");
                 }
-                
+
                 EnvironmentVariables.Add("PATH", Options.Path);
                 EnvironmentVariables.Add("DOTNET_ROOT", Options.DotnetRoot);
                 EnvironmentVariables.Add("helix", Options.HelixQueue);
@@ -68,7 +68,7 @@ public bool SetupEnvironment()
 
         public void DisplayContents(string path = "./")
         {
-            try 
+            try
             {
                 Console.WriteLine();
                 Console.WriteLine($"Displaying directory contents for {path}:");
@@ -88,9 +88,9 @@ public void DisplayContents(string path = "./")
             }
         }
 
-        public async Task<bool> InstallAspNetAppIfNeededAsync() 
+        public async Task<bool> InstallAspNetAppIfNeededAsync()
         {
-            try 
+            try
             {
                 if (File.Exists(Options.AspNetRuntime))
                 {
@@ -113,7 +113,7 @@ public async Task<bool> InstallAspNetAppIfNeededAsync()
                             }
                         }
                     }
-                    
+
                     DisplayContents(appRuntimePath);
 
                     Console.WriteLine($"Adding current directory to nuget sources: {Options.HELIX_WORKITEM_ROOT}");
@@ -152,7 +152,7 @@ await ProcessUtil.RunAsync($"{Options.DotnetRoot}/dotnet",
                     Options.Path += $"{Environment.GetEnvironmentVariable("DOTNET_CLI_HOME")}/.dotnet/tools";
                     EnvironmentVariables["PATH"] = Options.Path;
                 }
-                else 
+                else
                 {
                     Console.WriteLine($"No AspNetRuntime found: {Options.AspNetRuntime}, skipping...");
                 }
@@ -165,19 +165,19 @@ await ProcessUtil.RunAsync($"{Options.DotnetRoot}/dotnet",
             }
         }
 
-        public bool InstallAspNetRefIfNeeded() 
+        public bool InstallAspNetRefIfNeeded()
         {
-            try 
+            try
             {
                 if (File.Exists(Options.AspNetRef))
                 {
                     var refPath = $"Microsoft.AspNetCore.App.Ref";
                     Console.WriteLine($"Found AspNetRef: {Options.AspNetRef}, extracting to {refPath}");
                     ZipFile.ExtractToDirectory(Options.AspNetRef, "Microsoft.AspNetCore.App.Ref");
-                    
+
                     DisplayContents(refPath);
                 }
-                else 
+                else
                 {
                     Console.WriteLine($"No AspNetRef found: {Options.AspNetRef}, skipping...");
                 }
@@ -189,15 +189,37 @@ public bool InstallAspNetRefIfNeeded()
                 return false;
             }
         }
-        
+
+        public async Task<bool> InstallDotnetDump()
+        {
+            try
+            {
+                await ProcessUtil.RunAsync($"{Options.DotnetRoot}/dotnet",
+                            $"tool install dotnet-dump --tool-path {Options.HELIX_WORKITEM_ROOT} " +
+                              "--version 5.0.0-* --add-source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json",
+                            environmentVariables: EnvironmentVariables,
+                            outputDataReceived: Console.WriteLine,
+                            errorDataReceived: Console.Error.WriteLine,
+                            throwOnError: false);
+
+                return true;
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine($"Exception in InstallDotnetDump: {e}");
+                return false;
+            }
+        }
+
         public async Task<bool> CheckTestDiscoveryAsync()
         {
             try
             {
                 // Run test discovery so we know if there are tests to run
                 var discoveryResult = await ProcessUtil.RunAsync($"{Options.DotnetRoot}/dotnet",
                     $"vstest {Options.Target} -lt",
-                    environmentVariables: EnvironmentVariables);
+                    environmentVariables: EnvironmentVariables,
+                    cancellationToken: new CancellationTokenSource(TimeSpan.FromMinutes(2)).Token);
 
                 if (discoveryResult.StandardOutput.Contains("Exception thrown"))
                 {
@@ -217,8 +239,10 @@ public async Task<bool> CheckTestDiscoveryAsync()
         public async Task<int> RunTestsAsync()
         {
             var exitCode = 0;
-            try 
+            try
             {
+                // Timeout test run 5 minutes before the Helix job would timeout
+                var cts = new CancellationTokenSource(Options.Timeout.Subtract(TimeSpan.FromMinutes(5)));
                 var commonTestArgs = $"vstest {Options.Target} --logger:xunit --logger:\"console;verbosity=normal\" --blame";
                 if (Options.Quarantined)
                 {
@@ -230,7 +254,8 @@ public async Task<int> RunTestsAsync()
                         environmentVariables: EnvironmentVariables,
                         outputDataReceived: Console.WriteLine,
                         errorDataReceived: Console.Error.WriteLine,
-                        throwOnError: false);
+                        throwOnError: false,
+                        cancellationToken: cts.Token);
 
                     if (result.ExitCode != 0)
                     {
@@ -247,7 +272,8 @@ public async Task<int> RunTestsAsync()
                         environmentVariables: EnvironmentVariables,
                         outputDataReceived: Console.WriteLine,
                         errorDataReceived: Console.Error.WriteLine,
-                        throwOnError: false);
+                        throwOnError: false,
+                        cancellationToken: cts.Token);
 
                     if (result.ExitCode != 0)
                     {
diff --git a/eng/helix/content/runtests.cmd b/eng/helix/content/runtests.cmd
@@ -3,9 +3,18 @@ REM Need delayed expansion !PATH! so parens in the path don't mess up the parens
 setlocal enabledelayedexpansion
 
 REM Use '$' as a variable name prefix to avoid MSBuild variable collisions with these variables
+set $target=%1
 set $sdkVersion=%2
 set $runtimeVersion=%3
+set $queue=%4
 set $arch=%5
+set $quarantined=%6
+set $ef=%7
+set $aspnetruntime=%8
+set $aspnetref=%9
+REM Batch only supports up to 9 arguments using the %# syntax, need to shift to get more
+shift
+set $helixTimeout=%9
 
 set DOTNET_HOME=%HELIX_CORRELATION_PAYLOAD%\sdk
 set DOTNET_ROOT=%DOTNET_HOME%\%$arch%
@@ -23,10 +32,11 @@ powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePo
 set exit_code=0
 echo "Restore: dotnet restore RunTests\RunTests.csproj --source https://api.nuget.org/v3/index.json --ignore-failed-sources..."
 dotnet restore RunTests\RunTests.csproj --source https://api.nuget.org/v3/index.json --ignore-failed-sources
-echo "Running tests: dotnet run --project RunTests\RunTests.csproj -- --target %1 --sdk %2 --runtime %3 --queue %4 --arch %5 --quarantined %6 --ef %7 --aspnetruntime %8 --aspnetref %9..."
-dotnet run --project RunTests\RunTests.csproj -- --target %1 --sdk %2 --runtime %3 --queue %4 --arch %5 --quarantined %6 --ef %7 --aspnetruntime %8 --aspnetref %9
-if errorlevel 1 (
-    set exit_code=1
+
+echo "Running tests: dotnet run --project RunTests\RunTests.csproj -- --target %$target% --sdk %$sdkVersion% --runtime %$runtimeVersion% --queue %$queue% --arch %$arch% --quarantined %$quarantined% --ef %$ef% --aspnetruntime %$aspnetruntime% --aspnetref %$aspnetref% --helixTimeout %$helixTimeout%..."
+dotnet run --project RunTests\RunTests.csproj -- --target %$target% --sdk %$sdkVersion% --runtime %$runtimeVersion% --queue %$queue% --arch %$arch% --quarantined %$quarantined% --ef %$ef% --aspnetruntime %$aspnetruntime% --aspnetref %$aspnetref% --helixTimeout %$helixTimeout%
+if errorlevel neq 0 (
+    set exit_code=%errorlevel%
 )
 echo "Finished running tests: exit_code=%exit_code%"
 exit /b %exit_code%
diff --git a/eng/helix/content/runtests.sh b/eng/helix/content/runtests.sh
@@ -88,8 +88,8 @@ sync
 exit_code=0
 echo "Restore: $DOTNET_ROOT/dotnet restore RunTests/RunTests.csproj --source https://api.nuget.org/v3/index.json --ignore-failed-sources..."
 $DOTNET_ROOT/dotnet restore RunTests/RunTests.csproj --source https://api.nuget.org/v3/index.json --ignore-failed-sources
-echo "Running tests: $DOTNET_ROOT/dotnet run --project RunTests/RunTests.csproj -- --target $1 --sdk $2 --runtime $3 --queue $4 --arch $5 --quarantined $6 --ef $7 --aspnetruntime $8 --aspnetref $9..."
-$DOTNET_ROOT/dotnet run --project RunTests/RunTests.csproj -- --target $1 --sdk $2 --runtime $3 --queue $4 --arch $5 --quarantined $6 --ef $7 --aspnetruntime $8 --aspnetref $9
+echo "Running tests: $DOTNET_ROOT/dotnet run --project RunTests/RunTests.csproj -- --target $1 --sdk $2 --runtime $3 --queue $4 --arch $5 --quarantined $6 --ef $7 --aspnetruntime $8 --aspnetref $9 --helixTimeout ${10}..."
+$DOTNET_ROOT/dotnet run --project RunTests/RunTests.csproj -- --target $1 --sdk $2 --runtime $3 --queue $4 --arch $5 --quarantined $6 --ef $7 --aspnetruntime $8 --aspnetref $9 --helixTimeout ${10}
 exit_code=$?
 echo "Finished tests...exit_code=$exit_code"
 exit $exit_code
diff --git a/eng/targets/Helix.targets b/eng/targets/Helix.targets

Original file line number	Diff line number	Diff line change
`@@ -2,10 +2,6 @@`
`2`	`2`	`// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.`
`3`	`3`
`4`	`4`	`using System;`
`5`		`-using System.Collections.Generic;`
`6`		`-using System.CommandLine;`
`7`		`-using System.IO;`
`8`		`-using System.Runtime.InteropServices;`
`9`	`5`	`using System.Threading.Tasks;`
`10`	`6`
`11`	`7`	`namespace RunTests`
`@@ -14,7 +10,7 @@ class Program`
`14`	`10`	`{`
`15`	`11`	`static async Task Main(string[] args)`
`16`	`12`	`{`
`17`		`- try`
	`13`	`+ try`
`18`	`14`	`{`
`19`	`15`	`var runner = new TestRunner(RunTestsOptions.Parse(args));`
`20`	`16`
`@@ -27,6 +23,10 @@ static async Task Main(string[] args)`
`27`	`23`	`{`
`28`	`24`	`keepGoing = runner.InstallAspNetRefIfNeeded();`
`29`	`25`	`}`
	`26`	`+ if (keepGoing)`
	`27`	`+ {`
	`28`	`+ keepGoing = await runner.InstallDotnetDump();`
	`29`	`+ }`
`30`	`30`
`31`	`31`	`runner.DisplayContents();`
`32`	`32`
Original file line number	Diff line number	Diff line change
`@@ -3,10 +3,10 @@`
`3`	`3`
`4`	`4`	`using System;`
`5`	`5`	`using System.Collections.Generic;`
`6`		`-using System.CommandLine;`
`7`	`6`	`using System.IO;`
`8`	`7`	`using System.IO.Compression;`
`9`	`8`	`using System.Runtime.InteropServices;`
	`9`	`+using System.Threading;`
`10`	`10`	`using System.Threading.Tasks;`
`11`	`11`
`12`	`12`	`namespace RunTests`
`@@ -24,15 +24,15 @@ public TestRunner(RunTestsOptions options)`
`24`	`24`
`25`	`25`	`public bool SetupEnvironment()`
`26`	`26`	`{`
`27`		`- try`
	`27`	`+ try`
`28`	`28`	`{`
`29`	`29`	`// Rename default.NuGet.config to NuGet.config if there is not a custom one from the project`
`30`	`30`	`// We use a local NuGet.config file to avoid polluting global machine state and avoid relying on global machine state`
`31`	`31`	`if (!File.Exists("NuGet.config"))`
`32`	`32`	`{`
`33`	`33`	`File.Copy("default.NuGet.config", "NuGet.config");`
`34`	`34`	`}`
`35`		`-`
	`35`	`+`
`36`	`36`	`EnvironmentVariables.Add("PATH", Options.Path);`
`37`	`37`	`EnvironmentVariables.Add("DOTNET_ROOT", Options.DotnetRoot);`
`38`	`38`	`EnvironmentVariables.Add("helix", Options.HelixQueue);`
`@@ -68,7 +68,7 @@ public bool SetupEnvironment()`
`68`	`68`
`69`	`69`	`public void DisplayContents(string path = "./")`
`70`	`70`	`{`
`71`		`- try`
	`71`	`+ try`
`72`	`72`	`{`
`73`	`73`	`Console.WriteLine();`
`74`	`74`	`Console.WriteLine($"Displaying directory contents for {path}:");`
`@@ -88,9 +88,9 @@ public void DisplayContents(string path = "./")`
`88`	`88`	`}`
`89`	`89`	`}`
`90`	`90`
`91`		`- public async Task<bool> InstallAspNetAppIfNeededAsync()`
	`91`	`+ public async Task<bool> InstallAspNetAppIfNeededAsync()`
`92`	`92`	`{`
`93`		`- try`
	`93`	`+ try`
`94`	`94`	`{`
`95`	`95`	`if (File.Exists(Options.AspNetRuntime))`
`96`	`96`	`{`
`@@ -113,7 +113,7 @@ public async Task<bool> InstallAspNetAppIfNeededAsync()`
`113`	`113`	`}`
`114`	`114`	`}`
`115`	`115`	`}`
`116`		`-`
	`116`	`+`
`117`	`117`	`DisplayContents(appRuntimePath);`
`118`	`118`
`119`	`119`	`Console.WriteLine($"Adding current directory to nuget sources: {Options.HELIX_WORKITEM_ROOT}");`
`@@ -152,7 +152,7 @@ await ProcessUtil.RunAsync($"{Options.DotnetRoot}/dotnet",`
`152`	`152`	`Options.Path += $"{Environment.GetEnvironmentVariable("DOTNET_CLI_HOME")}/.dotnet/tools";`
`153`	`153`	`EnvironmentVariables["PATH"] = Options.Path;`
`154`	`154`	`}`
`155`		`- else`
	`155`	`+ else`
`156`	`156`	`{`
`157`	`157`	`Console.WriteLine($"No AspNetRuntime found: {Options.AspNetRuntime}, skipping...");`
`158`	`158`	`}`
`@@ -165,19 +165,19 @@ await ProcessUtil.RunAsync($"{Options.DotnetRoot}/dotnet",`
`165`	`165`	`}`
`166`	`166`	`}`
`167`	`167`
`168`		`- public bool InstallAspNetRefIfNeeded()`
	`168`	`+ public bool InstallAspNetRefIfNeeded()`
`169`	`169`	`{`
`170`		`- try`
	`170`	`+ try`
`171`	`171`	`{`
`172`	`172`	`if (File.Exists(Options.AspNetRef))`
`173`	`173`	`{`
`174`	`174`	`var refPath = $"Microsoft.AspNetCore.App.Ref";`
`175`	`175`	`Console.WriteLine($"Found AspNetRef: {Options.AspNetRef}, extracting to {refPath}");`
`176`	`176`	`ZipFile.ExtractToDirectory(Options.AspNetRef, "Microsoft.AspNetCore.App.Ref");`
`177`		`-`
	`177`	`+`
`178`	`178`	`DisplayContents(refPath);`
`179`	`179`	`}`
`180`		`- else`
	`180`	`+ else`
`181`	`181`	`{`
`182`	`182`	`Console.WriteLine($"No AspNetRef found: {Options.AspNetRef}, skipping...");`
`183`	`183`	`}`
`@@ -189,15 +189,37 @@ public bool InstallAspNetRefIfNeeded()`
`189`	`189`	`return false;`
`190`	`190`	`}`
`191`	`191`	`}`
`192`		`-`
	`192`	`+`
	`193`	`+ public async Task<bool> InstallDotnetDump()`
	`194`	`+ {`
	`195`	`+ try`
	`196`	`+ {`
	`197`	`+ await ProcessUtil.RunAsync($"{Options.DotnetRoot}/dotnet",`
	`198`	`+ $"tool install dotnet-dump --tool-path {Options.HELIX_WORKITEM_ROOT} " +`
	`199`	`+ "--version 5.0.0-* --add-source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json",`
	`200`	`+ environmentVariables: EnvironmentVariables,`
	`201`	`+ outputDataReceived: Console.WriteLine,`
	`202`	`+ errorDataReceived: Console.Error.WriteLine,`
	`203`	`+ throwOnError: false);`
	`204`	`+`
	`205`	`+ return true;`
	`206`	`+ }`
	`207`	`+ catch (Exception e)`
	`208`	`+ {`
	`209`	`+ Console.WriteLine($"Exception in InstallDotnetDump: {e}");`
	`210`	`+ return false;`
	`211`	`+ }`
	`212`	`+ }`
	`213`	`+`
`193`	`214`	`public async Task<bool> CheckTestDiscoveryAsync()`
`194`	`215`	`{`
`195`	`216`	`try`
`196`	`217`	`{`
`197`	`218`	`// Run test discovery so we know if there are tests to run`
`198`	`219`	`var discoveryResult = await ProcessUtil.RunAsync($"{Options.DotnetRoot}/dotnet",`
`199`	`220`	`$"vstest {Options.Target} -lt",`
`200`		`- environmentVariables: EnvironmentVariables);`
	`221`	`+ environmentVariables: EnvironmentVariables,`
	`222`	`+ cancellationToken: new CancellationTokenSource(TimeSpan.FromMinutes(2)).Token);`
`201`	`223`
`202`	`224`	`if (discoveryResult.StandardOutput.Contains("Exception thrown"))`
`203`	`225`	`{`
`@@ -217,8 +239,10 @@ public async Task<bool> CheckTestDiscoveryAsync()`
`217`	`239`	`public async Task<int> RunTestsAsync()`
`218`	`240`	`{`
`219`	`241`	`var exitCode = 0;`
`220`		`- try`
	`242`	`+ try`
`221`	`243`	`{`
	`244`	`+ // Timeout test run 5 minutes before the Helix job would timeout`
	`245`	`+ var cts = new CancellationTokenSource(Options.Timeout.Subtract(TimeSpan.FromMinutes(5)));`
`222`	`246`	`var commonTestArgs = $"vstest {Options.Target} --logger:xunit --logger:\"console;verbosity=normal\" --blame";`
`223`	`247`	`if (Options.Quarantined)`
`224`	`248`	`{`
`@@ -230,7 +254,8 @@ public async Task<int> RunTestsAsync()`
`230`	`254`	`environmentVariables: EnvironmentVariables,`
`231`	`255`	`outputDataReceived: Console.WriteLine,`
`232`	`256`	`errorDataReceived: Console.Error.WriteLine,`
`233`		`- throwOnError: false);`
	`257`	`+ throwOnError: false,`
	`258`	`+ cancellationToken: cts.Token);`
`234`	`259`
`235`	`260`	`if (result.ExitCode != 0)`
`236`	`261`	`{`
`@@ -247,7 +272,8 @@ public async Task<int> RunTestsAsync()`
`247`	`272`	`environmentVariables: EnvironmentVariables,`
`248`	`273`	`outputDataReceived: Console.WriteLine,`
`249`	`274`	`errorDataReceived: Console.Error.WriteLine,`
`250`		`- throwOnError: false);`
	`275`	`+ throwOnError: false,`
	`276`	`+ cancellationToken: cts.Token);`
`251`	`277`
`252`	`278`	`if (result.ExitCode != 0)`
`253`	`279`	`{`