#2578: Retry some file reads and writes to avoid 'file in use' on gitversion.json

src/GitVersion.Core/Core/GitPreparer.cs

@@ -3,6 +3,7 @@
 using System.Linq;
 using GitVersion.Common;
 using GitVersion.Extensions;
+using GitVersion.Helpers;
 using GitVersion.Logging;
 using GitVersion.Model.Configuration;
 using Microsoft.Extensions.Options;
@@ -148,7 +149,10 @@ private void CloneRepository(string repositoryUrl, string gitDirectory, Authenti
         {
             using (log.IndentLog($"Cloning repository from url '{repositoryUrl}'"))
             {
-                repository.Clone(repositoryUrl, gitDirectory, auth);
+                new OperationWithExponentialBackoff<LockedFileException>(new ThreadSleep(), log, () =>
+                {
+                    repository.Clone(repositoryUrl, gitDirectory, auth);
+                }).ExecuteAsync().Wait();
             }
         }
 
@@ -178,7 +182,9 @@ private void NormalizeGitDirectory(bool noFetch, string currentBranchName, bool
                 {
                     var refSpecs = string.Join(", ", remote.FetchRefSpecs.Select(r => r.Specification));
                     log.Info($"Fetching from remote '{remote.Name}' using the following refspecs: {refSpecs}.");
-                    repository.Fetch(remote.Name, Enumerable.Empty<string>(), authentication, null);
+                    new OperationWithExponentialBackoff<LockedFileException>(new ThreadSleep(), log,
+                        () => repository.Fetch(remote.Name, Enumerable.Empty<string>(), authentication, null))
+                    .ExecuteAsync().Wait();
                 }
 
                 EnsureLocalBranchExistsForCurrentBranch(remote, currentBranchName);
@@ -220,7 +226,7 @@ private void NormalizeGitDirectory(bool noFetch, string currentBranchName, bool
                 if (matchingCurrentBranch != null)
                 {
                     log.Info($"Checking out local branch '{currentBranchName}'.");
-                    repository.Checkout(matchingCurrentBranch.Name.Canonical);
+                    Checkout(matchingCurrentBranch.Name.Canonical);
                 }
                 else if (localBranchesWhereCommitShaIsHead.Count > 1)
                 {
@@ -233,7 +239,7 @@ private void NormalizeGitDirectory(bool noFetch, string currentBranchName, bool
                     if (main != null)
                     {
                         log.Warning("Because one of the branches is 'main', will build main." + moveBranchMsg);
-                        repository.Checkout(Config.MainBranchKey);
+                        Checkout(Config.MainBranchKey);
                     }
                     else
                     {
@@ -242,7 +248,7 @@ private void NormalizeGitDirectory(bool noFetch, string currentBranchName, bool
                         {
                             var branchWithoutSeparator = branchesWithoutSeparators[0];
                             log.Warning($"Choosing {branchWithoutSeparator.Name.Canonical} as it is the only branch without / or - in it. " + moveBranchMsg);
-                            repository.Checkout(branchWithoutSeparator.Name.Canonical);
+                            Checkout(branchWithoutSeparator.Name.Canonical);
                         }
                         else
                         {
@@ -253,12 +259,15 @@ private void NormalizeGitDirectory(bool noFetch, string currentBranchName, bool
                 else if (localBranchesWhereCommitShaIsHead.Count == 0)
                 {
                     log.Info($"No local branch pointing at the commit '{headSha}'. Fake branch needs to be created.");
-                    repository.CreateBranchForPullRequestBranch(authentication);
+                    new OperationWithExponentialBackoff<LockedFileException>(new ThreadSleep(), log, () =>
+                    {
+                        repository.CreateBranchForPullRequestBranch(authentication);
+                    }).ExecuteAsync().Wait();
                 }
                 else
                 {
                     log.Info($"Checking out local branch 'refs/heads/{localBranchesWhereCommitShaIsHead[0]}'.");
-                    repository.Checkout(localBranchesWhereCommitShaIsHead[0].Name.Friendly);
+                    Checkout(localBranchesWhereCommitShaIsHead[0].Name.Friendly);
                 }
             }
             finally
@@ -329,7 +338,7 @@ private void CreateOrUpdateLocalBranchesFromRemoteTrackingOnes(string remoteName
                     }
                     var remoteRefTipId = remoteTrackingReference.ReferenceTargetId;
                     log.Info($"Updating local ref '{localRef.Name.Canonical}' to point at {remoteRefTipId}.");
-                    repository.Refs.UpdateTarget(localRef, remoteRefTipId);
+                    new OperationWithExponentialBackoff<LockedFileException>(new ThreadSleep(), log, () => repository.Refs.UpdateTarget(localRef, remoteRefTipId)).ExecuteAsync().Wait();
                     continue;
                 }
 
@@ -382,10 +391,16 @@ public void EnsureLocalBranchExistsForCurrentBranch(IRemote remote, string curre
                 log.Info(isBranch ? $"Updating local branch {referenceName} to point at {repoTip}"
                     : $"Updating local branch {referenceName} to match ref {currentBranch}");
                 var localRef = repository.Refs[localCanonicalName];
-                repository.Refs.UpdateTarget(localRef, repoTipId);
+                new OperationWithExponentialBackoff<LockedFileException>(new ThreadSleep(), log, () => repository.Refs.UpdateTarget(localRef, repoTipId)).ExecuteAsync().Wait();
             }
 
-            repository.Checkout(localCanonicalName);
+            Checkout(localCanonicalName);
+        }
+
+        private void Checkout(string commitOrBranchSpec)
+        {
+            new OperationWithExponentialBackoff<LockedFileException>(new ThreadSleep(), log, () =>
+                    repository.Checkout(commitOrBranchSpec)).ExecuteAsync().Wait();
         }
     }
 }

src/GitVersion.Core/Core/ThreadSleep.cs

@@ -2,7 +2,7 @@
 
 namespace GitVersion
 {
-    internal class ThreadSleep : IThreadSleep
+    public class ThreadSleep : IThreadSleep
     {
         public async Task SleepAsync(int milliseconds)
         {

src/GitVersion.Core/Git/IReferenceCollection.cs

@@ -10,4 +10,5 @@ public interface IReferenceCollection : IEnumerable<IReference>
         void UpdateTarget(IReference directRef, IObjectId targetId);
         IEnumerable<IReference> FromGlob(string prefix);
     }
+
 }

src/GitVersion.Core/Helpers/OperationWithExponentialBackoff.cs

@@ -5,25 +5,37 @@
 
 namespace GitVersion.Helpers
 {
-    internal class OperationWithExponentialBackoff<T> where T : Exception
+    public class OperationWithExponentialBackoff<T> : OperationWithExponentialBackoff<T, bool> where T : Exception
+    {
+        public OperationWithExponentialBackoff(IThreadSleep threadSleep, ILog log, Action operation, int maxRetries = 5)
+            : base(threadSleep, log, () => { operation(); return false; }, maxRetries)
+        {
+        }
+
+        public new Task ExecuteAsync()
+        {
+            return base.ExecuteAsync();
+        }
+    }
+    public class OperationWithExponentialBackoff<T, Result> where T : Exception
     {
         private readonly IThreadSleep threadSleep;
         private readonly ILog log;
-        private readonly Action operation;
+        private readonly Func<Result> operation;
         private readonly int maxRetries;
 
-        public OperationWithExponentialBackoff(IThreadSleep threadSleep, ILog log, Action operation, int maxRetries = 5)
+        public OperationWithExponentialBackoff(IThreadSleep threadSleep, ILog log, Func<Result> operation, int maxRetries = 5)
         {
             if (maxRetries < 0)
                 throw new ArgumentOutOfRangeException(nameof(maxRetries));
 
             this.threadSleep = threadSleep ?? throw new ArgumentNullException(nameof(threadSleep));
-            this.log = log;
+            this.log = log ?? throw new ArgumentNullException(nameof(log));
             this.operation = operation;
             this.maxRetries = maxRetries;
         }
 
-        public async Task ExecuteAsync()
+        public async Task<Result> ExecuteAsync()
         {
             var exceptions = new List<Exception>();
 
@@ -36,8 +48,7 @@ public async Task ExecuteAsync()
 
                 try
                 {
-                    operation();
-                    break;
+                    return operation();
                 }
                 catch (T e)
                 {
@@ -53,6 +64,7 @@ public async Task ExecuteAsync()
 
                 sleepMSec *= 2;
             }
+            return default;
         }
     }
 }

src/GitVersion.Core/Model/Exceptions/LockedFileException.cs

@@ -0,0 +1,11 @@
+using System;
+
+namespace GitVersion
+{
+    public class LockedFileException : Exception
+    {
+        public LockedFileException(Exception inner) : base(inner.Message, inner)
+        {
+        }
+    }
+}

src/GitVersion.Core/Model/VersionVariables.cs

@@ -1,3 +1,5 @@
+using GitVersion.Helpers;
+using GitVersion.Logging;
 using System;
 using System.Collections;
 using System.Collections.Generic;
@@ -164,7 +166,29 @@ public static VersionVariables FromJson(string json)
             return FromDictionary(variablePairs);
         }
 
-        public static VersionVariables FromFile(string filePath, IFileSystem fileSystem)
+        public static VersionVariables FromFile(string filePath, IFileSystem fileSystem, ILog log)
+        {
+            try
+            {
+                if (log == null)
+                {
+                    return FromFileInternal(filePath, fileSystem);
+                }
+                var retryOperation = new OperationWithExponentialBackoff<IOException, VersionVariables>(new ThreadSleep(), log, () => FromFileInternal(filePath, fileSystem));
+                var versionVariables = retryOperation.ExecuteAsync().Result;
+                return versionVariables;
+            }
+            catch (AggregateException ex)
+            {
+                var lastException = ex.InnerExceptions?.LastOrDefault() ?? ex.InnerException;
+                if (lastException != null)
+                {
+                    throw lastException;
+                }
+                throw;
+            }
+        }
+        private static VersionVariables FromFileInternal(string filePath, IFileSystem fileSystem)
         {
             using var stream = fileSystem.OpenRead(filePath);
             using var reader = new StreamReader(stream);

src/GitVersion.Core/VersionCalculation/Cache/GitVersionCache.cs

@@ -74,7 +74,7 @@ public VersionVariables LoadVersionVariablesFromDiskCache(GitVersionCacheKey key
                 {
                     try
                     {
-                        var loadedVariables = VersionVariables.FromFile(cacheFileName, fileSystem);
+                        var loadedVariables = VersionVariables.FromFile(cacheFileName, fileSystem, log);
                         return loadedVariables;
                     }
                     catch (Exception ex)

src/GitVersion.Core/VersionConverters/OutputGenerator/OutputGenerator.cs

@@ -1,4 +1,6 @@
 using System;
+using System.IO;
+using GitVersion.Helpers;
 using GitVersion.Logging;
 using GitVersion.Model;
 using GitVersion.OutputVariables;
@@ -13,13 +15,15 @@ public interface IOutputGenerator : IVersionConverter<OutputContext>
     public class OutputGenerator : IOutputGenerator
     {
         private readonly IConsole console;
+        private readonly ILog log;
         private readonly IFileSystem fileSystem;
         private readonly IOptions<GitVersionOptions> options;
         private readonly ICurrentBuildAgent buildAgent;
 
-        public OutputGenerator(ICurrentBuildAgent buildAgent, IConsole console, IFileSystem fileSystem, IOptions<GitVersionOptions> options)
+        public OutputGenerator(ICurrentBuildAgent buildAgent, IConsole console, ILog log, IFileSystem fileSystem, IOptions<GitVersionOptions> options)
         {
             this.console = console ?? throw new ArgumentNullException(nameof(console));
+            this.log = log ?? throw new ArgumentNullException(nameof(log));
             this.fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
             this.options = options ?? throw new ArgumentNullException(nameof(options));
             this.buildAgent = buildAgent;
@@ -34,7 +38,8 @@ public void Execute(VersionVariables variables, OutputContext context)
             }
             if (gitVersionOptions.Output.Contains(OutputType.File))
             {
-                fileSystem.WriteAllText(context.OutputFile, variables.ToString());
+                var retryOperation = new OperationWithExponentialBackoff<IOException>(new ThreadSleep(), log, () => fileSystem.WriteAllText(context.OutputFile, variables.ToString()));
+                retryOperation.ExecuteAsync().Wait();
             }
             if (gitVersionOptions.Output.Contains(OutputType.Json))
             {