From efb587612d928f8c87a6cf718e3c4df7f7fe6b99 Mon Sep 17 00:00:00 2001 From: Anatoli Beliaev Date: Tue, 23 Jul 2019 18:21:17 -0700 Subject: [PATCH 1/5] Remove dot from snapshot folder name postfixes --- docs/designs/PowerShell-AzF-Overall-Design.md | 2 +- .../DependencySnapshotFolderNameTools.cs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/designs/PowerShell-AzF-Overall-Design.md b/docs/designs/PowerShell-AzF-Overall-Design.md index 9162d7bb..ed238106 100644 --- a/docs/designs/PowerShell-AzF-Overall-Design.md +++ b/docs/designs/PowerShell-AzF-Overall-Design.md @@ -492,7 +492,7 @@ A snapshot is considered _acceptable_ if it contains any version _allowed_ by th However, if the latest snapshot is _not acceptable_ (i.e. it does not contain module versions required by the manifest), the worker starts installing the dependencies into a new snapshot, and all the subsequent function invocation requests are blocked, waiting for the new snapshot installation to complete. -When a snapshot installation starts, the dependencies are first installed into a folder with a name following a special pattern (`*.i`), so that this snapshot is not picked up by any worker prematurely, before the installation is complete. After _successful_ completion, the snapshot is _atomically promoted_ by renaming the folder to follow a different pattern (`*.r`), which indicates to other workers that this snapshot is ready to use. If the installation fails or cannot complete for any reason (for example, the worker restarts, crashes, or gets decommissioned), the folder stays in the installing state until removed. +When a snapshot installation starts, the dependencies are first installed into a folder with a name following a special pattern (`*i`), so that this snapshot is not picked up by any worker prematurely, before the installation is complete. After _successful_ completion, the snapshot is _atomically promoted_ by renaming the folder to follow a different pattern (`*r`), which indicates to other workers that this snapshot is ready to use. If the installation fails or cannot complete for any reason (for example, the worker restarts, crashes, or gets decommissioned), the folder stays in the installing state until removed. Incomplete and old snapshots that are no longer in use are periodically removed from the file storage. In order to allow detecting unused snapshots, each PowerShell worker keeps "touching" a file named `.used` at the root of the used snapshot folder every `PSWorkerHeartbeatPeriodMinutes` minutes. Before and after installing any new snapshot, every PowerShell worker looks for unused snapshots by checking the folder creation time and the `.used` file modification time. If both these time values are older than (`PSWorkerHeartbeatPeriodMinutes` + `PSWorkerOldSnapshotHeartbeatMarginMinutes`) minutes, the snapshot is considered unused, so the PowerShell worker removes it. The latest `PSWorkerMinNumberOfSnapshotsToKeep` snapshots will never be removed, regardless of usage. (`PSWorkerHeartbeatPeriodMinutes`, `PSWorkerOldSnapshotHeartbeatMarginMinutes`, and `PSWorkerMinNumberOfSnapshotsToKeep` are environment variables configurable via Application Settings of a Function App.) diff --git a/src/DependencyManagement/DependencySnapshotFolderNameTools.cs b/src/DependencyManagement/DependencySnapshotFolderNameTools.cs index 62b1a510..5e8a2f39 100644 --- a/src/DependencyManagement/DependencySnapshotFolderNameTools.cs +++ b/src/DependencyManagement/DependencySnapshotFolderNameTools.cs @@ -10,9 +10,9 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement internal static class DependencySnapshotFolderNameTools { - private const string InstallingPostfix = ".i"; + private const string InstallingPostfix = "i"; - public const string InstalledPostfix = ".r"; + public const string InstalledPostfix = "r"; public const string InstalledPattern = "*" + InstalledPostfix; @@ -29,11 +29,11 @@ public static string CreateUniqueName() /// appending a postfix, so that that the resulting path follows a different /// pattern and can be discovered using a different file mask. /// For example, for the _installed_ path - /// ".../20190710-1234.567890.r" + /// ".../20190710-1234.567890r" /// the _installing_ path will be: - /// ".../20190710-1234.567890.r.i" - /// This makes it possible to enumerate all the installed snapshots by using ".../*.r" mask, - /// and all the installing snapshots by using ".../*.i" mask. + /// ".../20190710-1234.567890ri" + /// This makes it possible to enumerate all the installed snapshots by using ".../*r" mask, + /// and all the installing snapshots by using ".../*i" mask. /// public static string ConvertInstalledToInstalling(string installedPath) { From 87dcaa8c074f48605aa7c5aed86b126fd185ec61 Mon Sep 17 00:00:00 2001 From: Anatoli Beliaev Date: Tue, 23 Jul 2019 18:22:36 -0700 Subject: [PATCH 2/5] Remove dashes and dots from snasphot folder names --- .../DependencySnapshotFolderNameTools.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DependencyManagement/DependencySnapshotFolderNameTools.cs b/src/DependencyManagement/DependencySnapshotFolderNameTools.cs index 5e8a2f39..0d6d33f6 100644 --- a/src/DependencyManagement/DependencySnapshotFolderNameTools.cs +++ b/src/DependencyManagement/DependencySnapshotFolderNameTools.cs @@ -20,7 +20,7 @@ internal static class DependencySnapshotFolderNameTools public static string CreateUniqueName() { - var uniqueBase = DateTime.UtcNow.ToString("yyyyMMdd-HHmmss.ffffff"); + var uniqueBase = DateTime.UtcNow.ToString("yyyyMMddHHmmssffffff"); return uniqueBase + InstalledPostfix; } @@ -29,9 +29,9 @@ public static string CreateUniqueName() /// appending a postfix, so that that the resulting path follows a different /// pattern and can be discovered using a different file mask. /// For example, for the _installed_ path - /// ".../20190710-1234.567890r" + /// ".../201907101234567890r" /// the _installing_ path will be: - /// ".../20190710-1234.567890ri" + /// ".../201907101234567890ri" /// This makes it possible to enumerate all the installed snapshots by using ".../*r" mask, /// and all the installing snapshots by using ".../*i" mask. /// From cb36ce5d8a139b3d694cb00fc346df60ed439f65 Mon Sep 17 00:00:00 2001 From: Anatoli Beliaev Date: Tue, 23 Jul 2019 18:24:57 -0700 Subject: [PATCH 3/5] Remove microseconds from snapshot folder names --- src/DependencyManagement/DependencySnapshotFolderNameTools.cs | 2 +- .../DependencySnapshotFolderNameToolsTests.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DependencyManagement/DependencySnapshotFolderNameTools.cs b/src/DependencyManagement/DependencySnapshotFolderNameTools.cs index 0d6d33f6..9b8bc52f 100644 --- a/src/DependencyManagement/DependencySnapshotFolderNameTools.cs +++ b/src/DependencyManagement/DependencySnapshotFolderNameTools.cs @@ -20,7 +20,7 @@ internal static class DependencySnapshotFolderNameTools public static string CreateUniqueName() { - var uniqueBase = DateTime.UtcNow.ToString("yyyyMMddHHmmssffffff"); + var uniqueBase = DateTime.UtcNow.ToString("yyyyMMddHHmmssfff"); return uniqueBase + InstalledPostfix; } diff --git a/test/Unit/DependencyManagement/DependencySnapshotFolderNameToolsTests.cs b/test/Unit/DependencyManagement/DependencySnapshotFolderNameToolsTests.cs index c274b3ec..7f997c40 100644 --- a/test/Unit/DependencyManagement/DependencySnapshotFolderNameToolsTests.cs +++ b/test/Unit/DependencyManagement/DependencySnapshotFolderNameToolsTests.cs @@ -17,7 +17,7 @@ public class DependencySnapshotFolderNameToolsTests public void CreatesUniqueEnoughNames() { var name1 = DependencySnapshotFolderNameTools.CreateUniqueName(); - Thread.Sleep(1); // A snapshot name created 1 millisecond later must be different + Thread.Sleep(2); // A snapshot name created 2 milliseconds later must be different var name2 = DependencySnapshotFolderNameTools.CreateUniqueName(); Assert.NotEqual(name1, name2); } @@ -41,7 +41,7 @@ public void NamesConvertedFromInstalledToInstallingDoNotHaveInstalledPostfix() public void UniqueNamesConvertedFromInstalledToInstallingAreStillUnique() { var name1 = DependencySnapshotFolderNameTools.CreateUniqueName(); - Thread.Sleep(1); // A snapshot name created 1 millisecond later must be different + Thread.Sleep(2); // A snapshot name created 2 milliseconds later must be different var name2 = DependencySnapshotFolderNameTools.CreateUniqueName(); var convertedToInstalling1 = DependencySnapshotFolderNameTools.ConvertInstalledToInstalling(name1); From c5e513bd21b6358184b8bf479cd3160c8d24fe36 Mon Sep 17 00:00:00 2001 From: Anatoli Beliaev Date: Tue, 23 Jul 2019 18:26:56 -0700 Subject: [PATCH 4/5] Remove century from the year number in snapshot folder names --- src/DependencyManagement/DependencySnapshotFolderNameTools.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DependencyManagement/DependencySnapshotFolderNameTools.cs b/src/DependencyManagement/DependencySnapshotFolderNameTools.cs index 9b8bc52f..6bc560ae 100644 --- a/src/DependencyManagement/DependencySnapshotFolderNameTools.cs +++ b/src/DependencyManagement/DependencySnapshotFolderNameTools.cs @@ -20,7 +20,7 @@ internal static class DependencySnapshotFolderNameTools public static string CreateUniqueName() { - var uniqueBase = DateTime.UtcNow.ToString("yyyyMMddHHmmssfff"); + var uniqueBase = DateTime.UtcNow.ToString("yyMMddHHmmssfff"); return uniqueBase + InstalledPostfix; } From 2001da5680ce7e035fbe354eeee2cb854a779696 Mon Sep 17 00:00:00 2001 From: Anatoli Beliaev Date: Tue, 23 Jul 2019 18:28:17 -0700 Subject: [PATCH 5/5] Fix comment --- src/DependencyManagement/DependencySnapshotFolderNameTools.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DependencyManagement/DependencySnapshotFolderNameTools.cs b/src/DependencyManagement/DependencySnapshotFolderNameTools.cs index 6bc560ae..163e4941 100644 --- a/src/DependencyManagement/DependencySnapshotFolderNameTools.cs +++ b/src/DependencyManagement/DependencySnapshotFolderNameTools.cs @@ -29,9 +29,9 @@ public static string CreateUniqueName() /// appending a postfix, so that that the resulting path follows a different /// pattern and can be discovered using a different file mask. /// For example, for the _installed_ path - /// ".../201907101234567890r" + /// ".../1907101234567r" /// the _installing_ path will be: - /// ".../201907101234567890ri" + /// ".../1907101234567ri" /// This makes it possible to enumerate all the installed snapshots by using ".../*r" mask, /// and all the installing snapshots by using ".../*i" mask. ///