Skip to content

Add retries to some FSW tests #68368

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Apr 23, 2022
Merged

Add retries to some FSW tests #68368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
512a89b
Add retries to some FSW tests
danmoseley Apr 22, 2022
b125ece
another
danmoseley Apr 22, 2022
23f530a
Add Helix logging to console
danmoseley Apr 22, 2022
d3c284b
feedback
danmoseley Apr 22, 2022
2160989
more
danmoseley Apr 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/libraries/Common/tests/TestUtilities/System/PlatformDetection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Security;
using System.Security.Authentication;
Expand All @@ -22,6 +23,8 @@ public static partial class PlatformDetection
// do it in a way that failures don't cascade.
//

public static readonly bool IsInHelix = Environment.GetEnvironmentVariables().Keys.Cast<string>().Where(key => key.StartsWith("HELIX")).Any();
Copy link
Member

@jkotas jkotas Apr 23, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be done lazily, similar to how other expensive properties in PlatformDetection are initialized lazily.

Copy link
Member Author

@danmoseley danmoseley Apr 23, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll fix


public static bool IsNetCore => Environment.Version.Major >= 5 || RuntimeInformation.FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase);
public static bool IsMonoRuntime => Type.GetType("Mono.RuntimeStructs") != null;
public static bool IsNotMonoRuntime => !IsMonoRuntime;
Expand Down
17 changes: 15 additions & 2 deletions src/libraries/Common/tests/TestUtilities/System/RetryHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;

Expand All @@ -11,13 +14,14 @@ public static partial class RetryHelper
{
private static readonly Func<int, int> s_defaultBackoffFunc = i => Math.Min(i * 100, 60_000);
private static readonly Predicate<Exception> s_defaultRetryWhenFunc = _ => true;
private static readonly bool s_debug = Environment.GetEnvironmentVariable("DEBUG_RETRYHELPER") == "1";

/// <summary>Executes the <paramref name="test"/> action up to a maximum of <paramref name="maxAttempts"/> times.</summary>
/// <param name="maxAttempts">The maximum number of times to invoke <paramref name="test"/>.</param>
/// <param name="test">The test to invoke.</param>
/// <param name="backoffFunc">After a failure, invoked to determine how many milliseconds to wait before the next attempt. It's passed the number of iterations attempted.</param>
/// <param name="retryWhen">Invoked to select the exceptions to retry on. If not set, any exception will trigger a retry.</param>
public static void Execute(Action test, int maxAttempts = 5, Func<int, int> backoffFunc = null, Predicate<Exception> retryWhen = null)
public static void Execute(Action test, int maxAttempts = 5, Func<int, int> backoffFunc = null, Predicate<Exception> retryWhen = null, [CallerMemberName] string? testName = null)
{
// Validate arguments
if (maxAttempts < 1)
Expand All @@ -35,20 +39,29 @@ public static void Execute(Action test, int maxAttempts = 5, Func<int, int> back
var exceptions = new List<Exception>();
for (int i = 1; i <= maxAttempts; i++)
{
Exception lastException;
try
{
test();
return;
}
catch (Exception e) when (retryWhen(e))
{
lastException = e;
exceptions.Add(e);
if (i == maxAttempts)
{
throw new AggregateException(exceptions);
}
}

if (s_debug)
{
string diagnostic = $"RetryHelper: retrying {testName} {i}th time of {maxAttempts}: got {lastException.Message}";
Console.WriteLine(diagnostic);
Debug.WriteLine(diagnostic);
}

Thread.Sleep((backoffFunc ?? s_defaultBackoffFunc)(i));
}
}
Expand All @@ -58,7 +71,7 @@ public static void Execute(Action test, int maxAttempts = 5, Func<int, int> back
/// <param name="test">The test to invoke.</param>
/// <param name="backoffFunc">After a failure, invoked to determine how many milliseconds to wait before the next attempt. It's passed the number of iterations attempted.</param>
/// <param name="retryWhen">Invoked to select the exceptions to retry on. If not set, any exception will trigger a retry.</param>
public static async Task ExecuteAsync(Func<Task> test, int maxAttempts = 5, Func<int, int> backoffFunc = null, Predicate<Exception> retryWhen = null)
public static async Task ExecuteAsync(Func<Task> test, int maxAttempts = 5, Func<int, int> backoffFunc = null, Predicate<Exception> retryWhen = null, [CallerMemberName] string? testName = null)
{
// Validate arguments
if (maxAttempts < 1)
Expand Down
18 changes: 11 additions & 7 deletions src/libraries/System.IO.FileSystem.Watcher/tests/FileSystemWatcher.Directory.Changed.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Runtime.InteropServices;
using Xunit;
using Xunit.Sdk;

namespace System.IO.Tests
{
Expand All @@ -11,15 +12,18 @@ public class Directory_Changed_Tests : FileSystemWatcherTest
[Fact]
public void FileSystemWatcher_Directory_Changed_LastWrite()
{
using (var testDirectory = new TempDirectory(GetTestFilePath()))
using (var dir = new TempDirectory(Path.Combine(testDirectory.Path, "dir")))
using (var watcher = new FileSystemWatcher(testDirectory.Path, Path.GetFileName(dir.Path)))
FileSystemWatcherTest.Execute(() =>
{
Action action = () => Directory.SetLastWriteTime(dir.Path, DateTime.Now + TimeSpan.FromSeconds(10));
using (var testDirectory = new TempDirectory(GetTestFilePath()))
using (var dir = new TempDirectory(Path.Combine(testDirectory.Path, "dir")))
using (var watcher = new FileSystemWatcher(testDirectory.Path, Path.GetFileName(dir.Path)))
{
Action action = () => Directory.SetLastWriteTime(dir.Path, DateTime.Now + TimeSpan.FromSeconds(10));

WatcherChangeTypes expected = WatcherChangeTypes.Changed;
ExpectEvent(watcher, expected, action, expectedPath: dir.Path);
}
WatcherChangeTypes expected = WatcherChangeTypes.Changed;
ExpectEvent(watcher, expected, action, expectedPath: dir.Path);
}
}, maxAttempts: DefaultAttemptsForExpectedEvent, backoffFunc: (iteration) => RetryDelayMilliseconds, retryWhen: e => e is XunitException);
}

[Fact]
Expand Down
41 changes: 23 additions & 18 deletions src/libraries/System.IO.FileSystem.Watcher/tests/FileSystemWatcher.Directory.NotifyFilter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using Xunit;
using Xunit.Sdk;

namespace System.IO.Tests
{
Expand All @@ -21,27 +23,30 @@ private static partial uint SetSecurityInfoByHandle( string name, uint objectTyp
[MemberData(nameof(FilterTypes))]
public void FileSystemWatcher_Directory_NotifyFilter_Attributes(NotifyFilters filter)
{
using (var testDirectory = new TempDirectory(GetTestFilePath()))
using (var dir = new TempDirectory(Path.Combine(testDirectory.Path, "dir")))
using (var watcher = new FileSystemWatcher(testDirectory.Path, Path.GetFileName(dir.Path)))
FileSystemWatcherTest.Execute(() =>
{
watcher.NotifyFilter = filter;
var attributes = File.GetAttributes(dir.Path);
using (var testDirectory = new TempDirectory(GetTestFilePath()))
using (var dir = new TempDirectory(Path.Combine(testDirectory.Path, "dir")))
using (var watcher = new FileSystemWatcher(testDirectory.Path, Path.GetFileName(dir.Path)))
{
watcher.NotifyFilter = filter;
var attributes = File.GetAttributes(dir.Path);

Action action = () => File.SetAttributes(dir.Path, attributes | FileAttributes.ReadOnly);
Action cleanup = () => File.SetAttributes(dir.Path, attributes);
Action action = () => File.SetAttributes(dir.Path, attributes | FileAttributes.ReadOnly);
Action cleanup = () => File.SetAttributes(dir.Path, attributes);

WatcherChangeTypes expected = 0;
if (filter == NotifyFilters.Attributes)
expected |= WatcherChangeTypes.Changed;
else if (OperatingSystem.IsLinux() && ((filter & LinuxFiltersForAttribute) > 0))
expected |= WatcherChangeTypes.Changed;
else if (OperatingSystem.IsMacOS() && ((filter & OSXFiltersForModify) > 0))
expected |= WatcherChangeTypes.Changed;
else if (OperatingSystem.IsMacOS() && ((filter & NotifyFilters.Security) > 0))
expected |= WatcherChangeTypes.Changed; // Attribute change on OSX is a ChangeOwner operation which passes the Security NotifyFilter.
ExpectEvent(watcher, expected, action, cleanup, dir.Path);
}
WatcherChangeTypes expected = 0;
if (filter == NotifyFilters.Attributes)
expected |= WatcherChangeTypes.Changed;
else if (OperatingSystem.IsLinux() && ((filter & LinuxFiltersForAttribute) > 0))
expected |= WatcherChangeTypes.Changed;
else if (OperatingSystem.IsMacOS() && ((filter & OSXFiltersForModify) > 0))
expected |= WatcherChangeTypes.Changed;
else if (OperatingSystem.IsMacOS() && ((filter & NotifyFilters.Security) > 0))
expected |= WatcherChangeTypes.Changed; // Attribute change on OSX is a ChangeOwner operation which passes the Security NotifyFilter.
ExpectEvent(watcher, expected, action, cleanup, dir.Path);
}
}, maxAttempts: DefaultAttemptsForExpectedEvent, backoffFunc: (iteration) => RetryDelayMilliseconds, retryWhen: e => e is XunitException);
}

[Theory]
Expand Down
31 changes: 18 additions & 13 deletions src/libraries/System.IO.FileSystem.Watcher/tests/FileSystemWatcher.File.Create.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Xunit;
using Xunit.Sdk;

namespace System.IO.Tests
{
Expand All @@ -30,7 +32,7 @@ public void FileSystemWatcher_File_Create()
[OuterLoop]
public void FileSystemWatcher_File_Create_EnablingDisablingNotAffectRaisingEvent()
{
ExecuteWithRetry(() =>
FileSystemWatcherTest.Execute(() =>
{
using (var testDirectory = new TempDirectory(GetTestFilePath()))
using (var watcher = new FileSystemWatcher(testDirectory.Path))
Expand Down Expand Up @@ -62,7 +64,7 @@ public void FileSystemWatcher_File_Create_EnablingDisablingNotAffectRaisingEvent
Assert.False(autoResetEvent.WaitOne(SubsequentExpectedWait));
Assert.True(numberOfRaisedEvents == 1);
}
});
}, DefaultAttemptsForExpectedEvent, (iteration) => RetryDelayMilliseconds);
}

[Fact]
Expand Down Expand Up @@ -144,21 +146,24 @@ public void FileSystemWatcher_File_Create_SymLink()
[Fact]
public void FileSystemWatcher_File_Create_SynchronizingObject()
{
using (var testDirectory = new TempDirectory(GetTestFilePath()))
using (var watcher = new FileSystemWatcher(testDirectory.Path))
FileSystemWatcherTest.Execute(() =>
{
TestISynchronizeInvoke invoker = new TestISynchronizeInvoke();
watcher.SynchronizingObject = invoker;
using (var testDirectory = new TempDirectory(GetTestFilePath()))
using (var watcher = new FileSystemWatcher(testDirectory.Path))
{
TestISynchronizeInvoke invoker = new TestISynchronizeInvoke();
watcher.SynchronizingObject = invoker;

string fileName = Path.Combine(testDirectory.Path, "file");
watcher.Filter = Path.GetFileName(fileName);
string fileName = Path.Combine(testDirectory.Path, "file");
watcher.Filter = Path.GetFileName(fileName);

Action action = () => File.Create(fileName).Dispose();
Action cleanup = () => File.Delete(fileName);
Action action = () => File.Create(fileName).Dispose();
Action cleanup = () => File.Delete(fileName);

ExpectEvent(watcher, WatcherChangeTypes.Created, action, cleanup, fileName);
Assert.True(invoker.BeginInvoke_Called);
}
ExpectEvent(watcher, WatcherChangeTypes.Created, action, cleanup, fileName);
Assert.True(invoker.BeginInvoke_Called);
}
}, maxAttempts: DefaultAttemptsForExpectedEvent, backoffFunc: (iteration) => RetryDelayMilliseconds, retryWhen: e => e is XunitException);
}
}
}
28 changes: 16 additions & 12 deletions src/libraries/System.IO.FileSystem.Watcher/tests/FileSystemWatcher.File.Delete.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Collections.Generic;
using System.Threading;
using Xunit;
using Xunit.Sdk;

namespace System.IO.Tests
{
Expand Down Expand Up @@ -91,19 +92,22 @@ public void FileSystemWatcher_File_Delete_DeepDirectoryStructure()
[ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))]
public void FileSystemWatcher_File_Delete_SymLink()
{
using (var testDirectory = new TempDirectory(GetTestFilePath()))
using (var dir = new TempDirectory(Path.Combine(testDirectory.Path, "dir")))
using (var temp = new TempFile(GetTestFilePath()))
using (var watcher = new FileSystemWatcher(dir.Path, "*"))
FileSystemWatcherTest.Execute(() =>
{
// Make the symlink in our path (to the temp file) and make sure an event is raised
string symLinkPath = Path.Combine(dir.Path, GetRandomLinkName());
Action action = () => File.Delete(symLinkPath);
Action cleanup = () => Assert.True(MountHelper.CreateSymbolicLink(symLinkPath, temp.Path, false));
cleanup();

ExpectEvent(watcher, WatcherChangeTypes.Deleted, action, cleanup, symLinkPath);
}
using (var testDirectory = new TempDirectory(GetTestFilePath()))
using (var dir = new TempDirectory(Path.Combine(testDirectory.Path, "dir")))
using (var temp = new TempFile(GetTestFilePath()))
using (var watcher = new FileSystemWatcher(dir.Path, "*"))
{
// Make the symlink in our path (to the temp file) and make sure an event is raised
string symLinkPath = Path.Combine(dir.Path, GetRandomLinkName());
Action action = () => File.Delete(symLinkPath);
Action cleanup = () => Assert.True(MountHelper.CreateSymbolicLink(symLinkPath, temp.Path, false));
cleanup();

ExpectEvent(watcher, WatcherChangeTypes.Deleted, action, cleanup, symLinkPath);
}
}, maxAttempts: DefaultAttemptsForExpectedEvent, backoffFunc: (iteration) => RetryDelayMilliseconds, retryWhen: e => e is XunitException);
}

[Fact]
Expand Down
33 changes: 19 additions & 14 deletions src/libraries/System.IO.FileSystem.Watcher/tests/FileSystemWatcher.File.NotifyFilter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Xunit;
using Xunit.Sdk;

namespace System.IO.Tests
{
Expand Down Expand Up @@ -51,23 +53,26 @@ public void FileSystemWatcher_File_NotifyFilter_Attributes(NotifyFilters filter)
[MemberData(nameof(FilterTypes))]
public void FileSystemWatcher_File_NotifyFilter_CreationTime(NotifyFilters filter)
{
using (var testDirectory = new TempDirectory(GetTestFilePath()))
using (var file = new TempFile(Path.Combine(testDirectory.Path, "file")))
using (var watcher = new FileSystemWatcher(testDirectory.Path, Path.GetFileName(file.Path)))
FileSystemWatcherTest.Execute(() =>
{
watcher.NotifyFilter = filter;
Action action = () => File.SetCreationTime(file.Path, DateTime.Now + TimeSpan.FromSeconds(10));
using (var testDirectory = new TempDirectory(GetTestFilePath()))
using (var file = new TempFile(Path.Combine(testDirectory.Path, "file")))
using (var watcher = new FileSystemWatcher(testDirectory.Path, Path.GetFileName(file.Path)))
{
watcher.NotifyFilter = filter;
Action action = () => File.SetCreationTime(file.Path, DateTime.Now + TimeSpan.FromSeconds(10));

WatcherChangeTypes expected = 0;
if (filter == NotifyFilters.CreationTime)
expected |= WatcherChangeTypes.Changed;
else if (OperatingSystem.IsLinux() && ((filter & LinuxFiltersForAttribute) > 0))
expected |= WatcherChangeTypes.Changed;
else if (OperatingSystem.IsMacOS() && ((filter & OSXFiltersForModify) > 0))
expected |= WatcherChangeTypes.Changed;
WatcherChangeTypes expected = 0;
if (filter == NotifyFilters.CreationTime)
expected |= WatcherChangeTypes.Changed;
else if (OperatingSystem.IsLinux() && ((filter & LinuxFiltersForAttribute) > 0))
expected |= WatcherChangeTypes.Changed;
else if (OperatingSystem.IsMacOS() && ((filter & OSXFiltersForModify) > 0))
expected |= WatcherChangeTypes.Changed;

ExpectEvent(watcher, expected, action, expectedPath: file.Path);
}
ExpectEvent(watcher, expected, action, expectedPath: file.Path);
}
}, maxAttempts: DefaultAttemptsForExpectedEvent, backoffFunc: (iteration) => RetryDelayMilliseconds, retryWhen: e => e is XunitException);
}

[Theory]
Expand Down
Loading