Allow exact module versions in requirements.psd1 (#299)

Allow exact module versions in requirements.psd1

Author: AnatoliB

docs/designs/PowerShell-AzF-Overall-Design.md

@@ -466,15 +466,20 @@ Note that, checking out a PowerShell Manager instance from the pool is a blockin
 
 The goal is to let the user declare the dependencies required by functions, and rely on the service automatically locating and installing the dependencies from the PowerShell Gallery or other sources, taking care of selecting the proper versions, and automatically upgrading the dependencies to the latest versions (if allowed by the version specifications provided by the user).
 
-Dependencies are declared in the _requirements.psd1_ file (_manifest_) as a collection of pairs (<_name_>, <_version specification_>). Currently, the version specification should strictly match the following pattern: `<major version>.*`, so a typical manifest looks like this:
+Dependencies are declared in the _requirements.psd1_ file (_manifest_) as a collection of pairs (<_name_>, <_version specification_>). Currently, the version specification should either be an exact and complete version, or strictly match the following pattern: `<major version>.*`. So, a typical manifest may look like this:
 
 ``` PowerShell
 @{
   'Az' = '2.*'
   'PSDepend' = '0.*'
+  'Pester' = '5.0.0-alpha3'
 }
 ```
 
+When the `<major version>.*` format is used, the worker will retrieve the latest available module version (within the specified major version) from the PowerShell Gallery, ignoring prerelease versions.
+
+When the exact version is specified, the worker will retrieve the specified version only, ignoring any other version. Prerelease versions are allowed in this case.
+
 The number of entries in the _requirements.psd1_ file should not exceed **10**. This limit is not user-configurable.
 
 Installing and upgrading dependencies should be performed automatically, without requiring any interaction with the user, and without interfering with the currently running functions. This represents an important design challenge. In a different context, dependencies could be stored on a single location on the file system, managed by regular PowerShell tools (`Install-Module`/`Save-Module`, `PSDepend`, etc.), while having the same file system location added to _PSModulePath_ to make all the modules available to scripts running on this machine. This is what PowerShell users normally do, and this approach looks attractive because it is simple and conventional. However, in the contexts where multiple independent workers load modules and execute scripts concurrently, and at the same time some module versions are being added, upgraded, or removed, this simple approach causes many known problems. The root causes of these problems are in the fundamentals of PowerShell and PowerShell modules design. The managed dependencies design in Azure Functions must take this into account. The problems will be solved if we satisfy the following conditions:

src/DependencyManagement/DependencyInfo.cs

@@ -8,12 +8,12 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement
     internal class DependencyInfo
     {
         internal readonly string Name;
-        internal readonly string LatestVersion;
+        internal readonly string ExactVersion;
 
-        internal DependencyInfo(string name, string latestVersion)
+        internal DependencyInfo(string name, string exactVersion)
         {
             Name = name;
-            LatestVersion = latestVersion;
+            ExactVersion = exactVersion;
         }
     }
 }

src/DependencyManagement/DependencyManagerStorage.cs

@@ -66,6 +66,12 @@ public IEnumerable<string> GetInstalledModuleVersions(string snapshotPath, strin
             return Directory.EnumerateDirectories(modulePath, $"{majorVersion}.*");
         }
 
+        public bool IsModuleVersionInstalled(string snapshotPath, string moduleName, string version)
+        {
+            var moduleVersionPath = Path.Join(snapshotPath, moduleName, version);
+            return Directory.Exists(moduleVersionPath);
+        }
+
         public string CreateNewSnapshotPath()
         {
             return Path.Join(

src/DependencyManagement/DependencyManifest.cs

@@ -38,14 +38,52 @@ public IEnumerable<DependencyManifestEntry> GetEntries()
 
             foreach (DictionaryEntry entry in hashtable)
             {
-                // A valid entry is of the form: 'ModuleName'='MajorVersion.*"
-                var name = (string)entry.Key;
-                var version = (string)entry.Value;
+                // A valid entry is of the form:
+                //     'ModuleName'='MajorVersion.*'
+                // or
+                //     'ModuleName'='ExactVersion'
+
+                yield return CreateDependencyManifestEntry(
+                                name: (string)entry.Key,
+                                version: (string)entry.Value);
+            }
+        }
+
+        private static DependencyManifestEntry CreateDependencyManifestEntry(string name, string version)
+        {
+            ValidateModuleName(name);
 
-                ValidateModuleName(name);
+            var match = Regex.Match(version, @"^(\d+)(.*)");
+            if (match.Success)
+            {
+                // Look for the 'MajorVersion.*' pattern first.
+                var majorVersion = match.Groups[1].Value;
+                var afterMajorVersion = match.Groups[2].Value;
+                if (afterMajorVersion == ".*")
+                {
+                    return new DependencyManifestEntry(
+                        name,
+                        VersionSpecificationType.MajorVersion,
+                        majorVersion);
+                }
 
-                yield return new DependencyManifestEntry(name, GetMajorVersion(version));
+                // At this point, we know this is not the 'MajorVersion.*' pattern.
+                // We want to perform a very basic sanity check of the format to detect some
+                // obviously wrong cases: make sure afterMajorVersion starts with a dot,
+                // does not contain * anywhere, and ends with a word character.
+                // Not even trying to match the actual version format rules,
+                // as they are quite complex and controlled by the server side anyway.
+                if (Regex.IsMatch(afterMajorVersion, @"^(\.[^\*]*?\w)?$"))
+                {
+                    return new DependencyManifestEntry(
+                        name,
+                        VersionSpecificationType.ExactVersion,
+                        version);
+                }
             }
+
+            var errorMessage = string.Format(PowerShellWorkerStrings.InvalidVersionFormat, version, "MajorVersion.*");
+            throw new ArgumentException(errorMessage);
         }
 
         /// <summary>
@@ -106,38 +144,5 @@ private static void ValidateModuleName(string name)
                 throw new ArgumentException(PowerShellWorkerStrings.DependencyNameIsNullOrEmpty);
             }
         }
-
-        /// <summary>
-        /// Parses the given string version and extracts the major version.
-        /// Please note that the only version we currently support is of the form '1.*'.
-        /// </summary>
-        private static string GetMajorVersion(string version)
-        {
-            ValidateVersionFormat(version);
-            return version.Split(".")[0];
-        }
-
-        private static void ValidateVersionFormat(string version)
-        {
-            if (version == null)
-            {
-                throw new ArgumentNullException(version);
-            }
-
-            if (!IsValidVersionFormat(version))
-            {
-                var errorMessage = string.Format(PowerShellWorkerStrings.InvalidVersionFormat, "MajorVersion.*");
-                throw new ArgumentException(errorMessage);
-            }
-        }
-
-        /// <summary>
-        /// Validates the given version format. Currently, we only support 'Number.*'.
-        /// </summary>
-        private static bool IsValidVersionFormat(string version)
-        {
-            var pattern = @"^(\d)+(\.)(\*)";
-            return Regex.IsMatch(version, pattern);
-        }
     }
 }

src/DependencyManagement/DependencyManifestEntry.cs

@@ -9,12 +9,18 @@ internal class DependencyManifestEntry
     {
         public string Name { get; }
 
-        public string MajorVersion { get; }
+        public VersionSpecificationType VersionSpecificationType { get;  }
 
-        public DependencyManifestEntry(string name, string majorVersion)
+        public string VersionSpecification { get; }
+
+        public DependencyManifestEntry(
+            string name,
+            VersionSpecificationType versionSpecificationType,
+            string versionSpecification)
         {
             Name = name;
-            MajorVersion = majorVersion;
+            VersionSpecificationType = versionSpecificationType;
+            VersionSpecification = versionSpecification;
         }
     }
 }

src/DependencyManagement/DependencySnapshotInstaller.cs

@@ -43,30 +43,27 @@ public void InstallSnapshot(
 
             try
             {
-                foreach (DependencyInfo module in GetLatestPublishedVersionsOfDependencies(dependencies))
+                foreach (DependencyInfo module in GetExactVersionsOfDependencies(dependencies))
                 {
-                    string moduleName = module.Name;
-                    string latestVersion = module.LatestVersion;
-
-                    logger.Log(isUserOnlyLog: false, LogLevel.Trace, string.Format(PowerShellWorkerStrings.StartedInstallingModule, moduleName, latestVersion));
+                    logger.Log(isUserOnlyLog: false, LogLevel.Trace, string.Format(PowerShellWorkerStrings.StartedInstallingModule, module.Name, module.ExactVersion));
 
                     int tries = 1;
 
                     while (true)
                     {
                         try
                         {
-                            _moduleProvider.SaveModule(pwsh, moduleName, latestVersion, installingPath);
+                            _moduleProvider.SaveModule(pwsh, module.Name, module.ExactVersion, installingPath);
 
-                            var message = string.Format(PowerShellWorkerStrings.ModuleHasBeenInstalled, moduleName, latestVersion);
+                            var message = string.Format(PowerShellWorkerStrings.ModuleHasBeenInstalled, module.Name, module.ExactVersion);
                             logger.Log(isUserOnlyLog: false, LogLevel.Trace, message);
 
                             break;
                         }
                         catch (Exception e)
                         {
                             string currentAttempt = GetCurrentAttemptMessage(tries);
-                            var errorMsg = string.Format(PowerShellWorkerStrings.FailToInstallModule, moduleName, latestVersion, currentAttempt, e.Message);
+                            var errorMsg = string.Format(PowerShellWorkerStrings.FailToInstallModule, module.Name, module.ExactVersion, currentAttempt, e.Message);
                             logger.Log(isUserOnlyLog: false, LogLevel.Error, errorMsg);
 
                             if (tries >= MaxNumberOfTries)
@@ -127,22 +124,35 @@ internal static string GetCurrentAttemptMessage(int attempt)
             }
         }
 
-        private List<DependencyInfo> GetLatestPublishedVersionsOfDependencies(
+        private List<DependencyInfo> GetExactVersionsOfDependencies(
             IEnumerable<DependencyManifestEntry> dependencies)
         {
             var result = new List<DependencyInfo>();
 
             foreach (var entry in dependencies)
             {
-                var latestVersion = GetModuleLatestPublishedVersion(entry.Name, entry.MajorVersion);
-
-                var dependencyInfo = new DependencyInfo(entry.Name, latestVersion);
+                var dependencyInfo = new DependencyInfo(entry.Name, GetExactVersion(entry));
                 result.Add(dependencyInfo);
             }
 
             return result;
         }
 
+        private string GetExactVersion(DependencyManifestEntry entry)
+        {
+            switch (entry.VersionSpecificationType)
+            {
+                case VersionSpecificationType.ExactVersion:
+                    return entry.VersionSpecification;
+
+                case VersionSpecificationType.MajorVersion:
+                    return GetModuleLatestPublishedVersion(entry.Name, entry.VersionSpecification);
+
+                default:
+                    throw new ArgumentException($"Unknown version specification type: {entry.VersionSpecificationType}");
+            }
+        }
+
         /// <summary>
         /// Gets the latest published module version for the given module name and major version.
         /// </summary>

src/DependencyManagement/IDependencyManagerStorage.cs

@@ -20,6 +20,8 @@ internal interface IDependencyManagerStorage
 
         IEnumerable<string> GetInstalledModuleVersions(string snapshotPath, string moduleName, string majorVersion);
 
+        bool IsModuleVersionInstalled(string snapshotPath, string moduleName, string version);
+
         string CreateNewSnapshotPath();
 
         string CreateInstallingSnapshot(string path);

src/DependencyManagement/InstalledDependenciesLocator.cs

@@ -5,6 +5,7 @@
 
 namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement
 {
+    using System;
     using System.Linq;
 
     internal class InstalledDependenciesLocator : IInstalledDependenciesLocator
@@ -23,7 +24,7 @@ public string GetPathWithAcceptableDependencyVersionsInstalled()
             if (lastSnapshotPath != null)
             {
                 var dependencies = _storage.GetDependencies();
-                if (dependencies.All(entry => IsMajorVersionInstalled(lastSnapshotPath, entry)))
+                if (dependencies.All(entry => IsAcceptableVersionInstalled(lastSnapshotPath, entry)))
                 {
                     return lastSnapshotPath;
                 }
@@ -32,12 +33,26 @@ public string GetPathWithAcceptableDependencyVersionsInstalled()
             return null;
         }
 
-        private bool IsMajorVersionInstalled(string snapshotPath, DependencyManifestEntry dependency)
+        private bool IsAcceptableVersionInstalled(string snapshotPath, DependencyManifestEntry dependency)
         {
-            var installedVersions =
-                _storage.GetInstalledModuleVersions(
-                    snapshotPath, dependency.Name, dependency.MajorVersion);
+            switch (dependency.VersionSpecificationType)
+            {
+                case VersionSpecificationType.ExactVersion:
+                    return _storage.IsModuleVersionInstalled(
+                        snapshotPath, dependency.Name, dependency.VersionSpecification);
+
+                case VersionSpecificationType.MajorVersion:
+                    return IsMajorVersionInstalled(
+                        snapshotPath, dependency.Name, dependency.VersionSpecification);
+
+                default:
+                    throw new ArgumentException($"Unknown version specification type: {dependency.VersionSpecificationType}");
+            }
+        }
 
+        private bool IsMajorVersionInstalled(string snapshotPath, string name, string majorVersion)
+        {
+            var installedVersions = _storage.GetInstalledModuleVersions(snapshotPath, name, majorVersion);
             return installedVersions.Any();
         }
     }

src/DependencyManagement/PowerShellGalleryModuleProvider.cs

@@ -106,6 +106,7 @@ public void SaveModule(PowerShell pwsh, string moduleName, string version, strin
                 .AddParameter("Repository", Repository)
                 .AddParameter("Name", moduleName)
                 .AddParameter("RequiredVersion", version)
+                .AddParameter("AllowPrerelease", Utils.BoxedTrue)
                 .AddParameter("Path", path)
                 .AddParameter("Force", Utils.BoxedTrue)
                 .AddParameter("ErrorAction", "Stop")

src/DependencyManagement/VersionSpecificationType.cs

@@ -0,0 +1,15 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+#pragma warning disable 1591 // Missing XML comment for publicly visible type or member 'member'
+
+namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement
+{
+    public enum VersionSpecificationType
+    {
+        ExactVersion,
+        MajorVersion
+    }
+}