Managed dependency download on background thread

src/DependencyManagement/DependencyManager.cs

@@ -14,11 +14,14 @@
 using Microsoft.Azure.Functions.PowerShellWorker.PowerShell;
 using Microsoft.Azure.Functions.PowerShellWorker.Utility;
 using LogLevel = Microsoft.Azure.WebJobs.Script.Grpc.Messages.RpcLog.Types.Level;
+using Microsoft.Azure.Functions.PowerShellWorker.Messaging;
 
 namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement
 {
     using System.Management.Automation;
     using System.Management.Automation.Language;
+    using System.Management.Automation.Runspaces;
+    using System.Threading.Tasks;
 
     internal class DependencyManager
     {
@@ -28,6 +31,11 @@ internal class DependencyManager
         // This is the location where the dependent modules will be installed.
         internal static string DependenciesPath { get; private set; }
 
+        internal Exception DependencyError { get => _dependencyError; }
+
+        //The dependency download task
+        internal Task DependencyDownloadTask { get => _dependencyDownloadTask; }
+
         // Az module name.
         private const string AzModuleName = "Az";
 
@@ -51,15 +59,67 @@ internal class DependencyManager
         // Managed Dependencies folder name.
         private const string ManagedDependenciesFolderName = "ManagedDependencies";
 
+        private volatile Exception _dependencyError;
+
         // This flag is used to figure out if we need to install/reinstall all the function app dependencies.
         // If we do, we use it to clean up the module destination path.
         private bool _shouldUpdateFunctionAppDependencies;
 
+        private volatile Task _dependencyDownloadTask;
+
         internal DependencyManager()
         {
             Dependencies = new List<DependencyInfo>();
         }
 
+        internal void ProcessDependencyDownload(MessagingStream msgStream, StreamingMessage request)
+        {
+            if (request.FunctionLoadRequest.ManagedDependencyEnabled)
+            {
+                //Start dependency download on a separate thread
+                _dependencyDownloadTask = Task.Run(() => ProcessDependencies(msgStream, request)).ContinueWith((task) =>
+                {
+                    _dependencyDownloadTask = null;
+                });
+            }
+        }
+
+        private void ProcessDependencies(
+            MessagingStream msgStream,
+            StreamingMessage request)
+        {
+            try
+            {
+                _dependencyError = null;
+                var rpcLogger = new RpcLogger(msgStream);
+                rpcLogger.SetContext(request.RequestId, null);
+                if (!_shouldUpdateFunctionAppDependencies)
+                {
+                    // The function app already has the latest dependencies installed.
+                    rpcLogger.Log(LogLevel.Trace, PowerShellWorkerStrings.LatestFunctionAppDependenciesAlreadyInstalled, isUserLog: true);
+                    return;
+                }
+
+                var initialSessionState = InitialSessionState.CreateDefault();
+                initialSessionState.ThreadOptions = PSThreadOptions.UseCurrentThread;
+                initialSessionState.EnvironmentVariables.Add(new SessionStateVariableEntry("PSModulePath", FunctionLoader.FunctionModulePath, null));
+                // Setting the execution policy on macOS and Linux throws an exception so only update it on Windows
+                if (Platform.IsWindows)
+                {
+                    initialSessionState.ExecutionPolicy = Microsoft.PowerShell.ExecutionPolicy.Unrestricted;
+                }
+
+                using (PowerShell PowerShellInstance = PowerShell.Create(initialSessionState))
+                {
+                    InstallFunctionAppDependencies(PowerShellInstance, rpcLogger);
+                }
+            }
+            catch (Exception e)
+            {
+                _dependencyError = e;
+            }
+        }
+
         /// <summary>
         /// Initializes the dependency manger and performs the following:
         /// - Parse functionAppRoot\requirements.psd1 file and create a list of dependencies to install.
@@ -83,8 +143,8 @@ internal void Initialize(FunctionLoadRequest request)
                 foreach (DictionaryEntry entry in entries)
                 {
                     // A valid entry is of the form: 'ModuleName'='MajorVersion.*"
-                    string name = (string) entry.Key;
-                    string version = (string) entry.Value;
+                    string name = (string)entry.Key;
+                    string version = (string)entry.Value;
 
                     // Validates that the module name is a supported dependency.
                     ValidateModuleName(name);

src/PowerShell/PowerShellManager.cs

@@ -45,7 +45,7 @@ static PowerShellManager()
             addMethod.Invoke(null, new object[] { "HttpRequestContext", typeof(HttpRequestContext) });
         }
 
-        internal PowerShellManager(ILogger logger, Action<PowerShell, ILogger> initAction = null)
+        internal PowerShellManager(ILogger logger)
         {
             if (FunctionLoader.FunctionAppRootPath == null)
             {
@@ -77,9 +77,6 @@ internal PowerShellManager(ILogger logger, Action<PowerShell, ILogger> initActio
             _pwsh.Streams.Verbose.DataAdding += streamHandler.VerboseDataAdding;
             _pwsh.Streams.Warning.DataAdding += streamHandler.WarningDataAdding;
 
-            // Install function app dependent modules
-            initAction?.Invoke(_pwsh, logger);
-
             // Initialize the Runspace
             InvokeProfile(FunctionLoader.FunctionAppProfilePath);
         }

src/PowerShell/PowerShellManagerPool.cs

@@ -50,14 +50,14 @@ internal PowerShellManagerPool(MessagingStream msgStream)
         /// Initialize the pool and populate it with PowerShellManager instances.
         /// We instantiate PowerShellManager instances in a lazy way, starting from size 1 and increase the number of workers as needed.
         /// </summary>
-        internal void Initialize(string requestId, Action<PowerShell, ILogger> initAction = null)
+        internal void Initialize(string requestId)
         {
             var logger = new RpcLogger(_msgStream);
 
             try
             {
                 logger.SetContext(requestId, invocationId: null);
-                _pool.Add(new PowerShellManager(logger, initAction));
+                _pool.Add(new PowerShellManager(logger));
                 _poolSize = 1;
             }
             finally

src/RequestProcessor.cs

@@ -16,7 +16,7 @@
 using Microsoft.Azure.WebJobs.Script.Grpc.Messages;
 using LogLevel = Microsoft.Azure.WebJobs.Script.Grpc.Messages.RpcLog.Types.Level;
 
-namespace  Microsoft.Azure.Functions.PowerShellWorker
+namespace Microsoft.Azure.Functions.PowerShellWorker
 {
     internal class RequestProcessor
     {
@@ -57,7 +57,7 @@ internal RequestProcessor(MessagingStream msgStream)
 
             // Host sends required metadata to worker to load function
             _requestHandlers.Add(StreamingMessage.ContentOneofCase.FunctionLoadRequest, ProcessFunctionLoadRequest);
-            
+
             // Host requests a given invocation
             _requestHandlers.Add(StreamingMessage.ContentOneofCase.InvocationRequest, ProcessInvocationRequest);
 
@@ -183,7 +183,6 @@ internal StreamingMessage ProcessFunctionLoadRequest(StreamingMessage request)
 
             try
             {
-                // Load the metadata of the function.
                 _functionLoader.LoadFunction(functionLoadRequest);
             }
             catch (Exception e)
@@ -206,9 +205,33 @@ internal StreamingMessage ProcessInvocationRequest(StreamingMessage request)
 
             try
             {
+                if (_dependencyManager.DependencyDownloadTask != null
+                    && ((_dependencyManager.DependencyDownloadTask.Status != TaskStatus.Canceled)
+                    || _dependencyManager.DependencyDownloadTask.Status != TaskStatus.Faulted
+                    || _dependencyManager.DependencyDownloadTask.Status != TaskStatus.RanToCompletion))
+                {
+                    var rpcLogger = new RpcLogger(_msgStream);
+                    rpcLogger.SetContext(request.RequestId, request.InvocationRequest?.InvocationId);
+                    rpcLogger.Log(LogLevel.Information, PowerShellWorkerStrings.DependencyDownloadInProgress, null, true);
+                    rpcLogger.Log(LogLevel.Information, PowerShellWorkerStrings.DependencyDownloadInProgress, null);
+                    _dependencyManager.DependencyDownloadTask.Wait();
+                }
+
+                if (_dependencyManager.DependencyError != null)
+                {
+                    StreamingMessage response = NewStreamingMessageTemplate(request.RequestId,
+                        StreamingMessage.ContentOneofCase.InvocationResponse,
+                        out StatusResult status);
+                    status.Status = StatusResult.Types.Status.Failure;
+                    status.Exception = _dependencyManager.DependencyError.ToRpcException();
+                    response.InvocationResponse.InvocationId = request.InvocationRequest.InvocationId;
+                    return response;
+                }
+
                 functionInfo = _functionLoader.GetFunctionInfo(request.InvocationRequest.FunctionId);
                 psManager = _powershellPool.CheckoutIdleWorker(request, functionInfo);
 
+                //ProcessInvocationRequestImpl(request, functionInfo, psManager);
                 if (_powershellPool.UpperBound == 1)
                 {
                     // When the concurrency upper bound is 1, we can handle only one invocation at a time anyways,
@@ -225,7 +248,6 @@ internal StreamingMessage ProcessInvocationRequest(StreamingMessage request)
             catch (Exception e)
             {
                 _powershellPool.ReclaimUsedWorker(psManager);
-
                 StreamingMessage response = NewStreamingMessageTemplate(
                     request.RequestId,
                     StreamingMessage.ContentOneofCase.InvocationResponse,
@@ -234,7 +256,6 @@ internal StreamingMessage ProcessInvocationRequest(StreamingMessage request)
                 response.InvocationResponse.InvocationId = request.InvocationRequest.InvocationId;
                 status.Status = StatusResult.Types.Status.Failure;
                 status.Exception = e.ToRpcException();
-
                 return response;
             }
 
@@ -299,28 +320,15 @@ internal StreamingMessage ProcessFunctionEnvironmentReloadRequest(StreamingMessa
         private void InitializeForFunctionApp(StreamingMessage request, StreamingMessage response)
         {
             var functionLoadRequest = request.FunctionLoadRequest;
-
-            // If 'ManagedDependencyEnabled' is true, process the function app dependencies as defined in FunctionAppRoot\requirements.psd1.
-            // These dependencies will be installed via 'Save-Module' when the first PowerShellManager instance is being created.
             if (functionLoadRequest.ManagedDependencyEnabled)
             {
                 _dependencyManager.Initialize(functionLoadRequest);
             }
 
             // Setup the FunctionApp root path and module path.
             FunctionLoader.SetupWellKnownPaths(functionLoadRequest);
-
-            // Constructing the first PowerShellManager instance for the Pool.
-            if (DependencyManager.Dependencies.Count > 0)
-            {
-                // Do extra work to install the specified dependencies.
-                _powershellPool.Initialize(request.RequestId, _dependencyManager.InstallFunctionAppDependencies);
-                response.FunctionLoadResponse.IsDependencyDownloaded = true;
-            }
-            else
-            {
-                _powershellPool.Initialize(request.RequestId);
-            }
+            _dependencyManager.ProcessDependencyDownload(_msgStream, request);
+            _powershellPool.Initialize(request.RequestId);
         }
 
         /// <summary>
@@ -425,7 +433,7 @@ private void BindOutputFromResult(InvocationResponse response, AzFunctionInfo fu
                         TypedData dataToUse = transformedValue.ToTypedData();
 
                         // if one of the bindings is '$return' we need to set the ReturnValue
-                        if(string.Equals(outBindingName, AzFunctionInfo.DollarReturn, StringComparison.OrdinalIgnoreCase))
+                        if (string.Equals(outBindingName, AzFunctionInfo.DollarReturn, StringComparison.OrdinalIgnoreCase))
                         {
                             response.ReturnValue = dataToUse;
                             continue;

src/resources/PowerShellWorkerStrings.resx

@@ -1,17 +1,17 @@
 <?xml version="1.0" encoding="utf-8"?>
 <root>
-  <!--
-    Microsoft ResX Schema
-
+  <!-- 
+    Microsoft ResX Schema 
+    
     Version 2.0
-
-    The primary goals of this format is to allow a simple XML format
-    that is mostly human readable. The generation and parsing of the
-    various data types are done through the TypeConverter classes
+    
+    The primary goals of this format is to allow a simple XML format 
+    that is mostly human readable. The generation and parsing of the 
+    various data types are done through the TypeConverter classes 
     associated with the data types.
-
+    
     Example:
-
+    
     ... ado.net/XML headers & schema ...
     <resheader name="resmimetype">text/microsoft-resx</resheader>
     <resheader name="version">2.0</resheader>
@@ -26,36 +26,36 @@
         <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
         <comment>This is a comment</comment>
     </data>
-
-    There are any number of "resheader" rows that contain simple
+                
+    There are any number of "resheader" rows that contain simple 
     name/value pairs.
-
-    Each data row contains a name, and value. The row also contains a
-    type or mimetype. Type corresponds to a .NET class that support
-    text/value conversion through the TypeConverter architecture.
-    Classes that don't support this are serialized and stored with the
+    
+    Each data row contains a name, and value. The row also contains a 
+    type or mimetype. Type corresponds to a .NET class that support 
+    text/value conversion through the TypeConverter architecture. 
+    Classes that don't support this are serialized and stored with the 
     mimetype set.
-
-    The mimetype is used for serialized objects, and tells the
-    ResXResourceReader how to depersist the object. This is currently not
+    
+    The mimetype is used for serialized objects, and tells the 
+    ResXResourceReader how to depersist the object. This is currently not 
     extensible. For a given mimetype the value must be set accordingly:
-
-    Note - application/x-microsoft.net.object.binary.base64 is the format
-    that the ResXResourceWriter will generate, however the reader can
+    
+    Note - application/x-microsoft.net.object.binary.base64 is the format 
+    that the ResXResourceWriter will generate, however the reader can 
     read any of the formats listed below.
-
+    
     mimetype: application/x-microsoft.net.object.binary.base64
-    value   : The object must be serialized with
+    value   : The object must be serialized with 
             : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
             : and then encoded with base64 encoding.
-
+    
     mimetype: application/x-microsoft.net.object.soap.base64
-    value   : The object must be serialized with
+    value   : The object must be serialized with 
             : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
             : and then encoded with base64 encoding.
 
     mimetype: application/x-microsoft.net.object.bytearray.base64
-    value   : The object must be serialized into a byte array
+    value   : The object must be serialized into a byte array 
             : using a System.ComponentModel.TypeConverter
             : and then encoded with base64 encoding.
     -->
@@ -166,7 +166,7 @@
   <data name="SpecifiedCustomPipeName" xml:space="preserve">
     <value>Custom pipe name specified. You can attach to the process by using vscode or by running `Enter-PSHostProcess -CustomPipeName {0}`</value>
   </data>
-    <data name="CannotFindModuleVersion" xml:space="preserve">
+  <data name="CannotFindModuleVersion" xml:space="preserve">
     <value>Cannot find a supported version for module '{0}' with major version '{0}'.</value>
   </data>
   <data name="FunctionAppDoesNotHaveDependentModulesToInstall" xml:space="preserve">
@@ -194,7 +194,7 @@
     <value>Invalid major version for module '{0}'. The maximum available major version is '{1}'.</value>
   </data>
   <data name="FailToInstallFuncAppDependencies" xml:space="preserve">
-    <value>Fail to install function app dependencies. Error: '{0}'</value>
+    <value>Fail to install function app dependencies. Restarting the app may resolve the error. Error: '{0}'</value>
   </data>
   <data name="FailToResolveHomeDirectory" xml:space="preserve">
     <value>Fail to resolve '{0}' path in App Service.</value>
@@ -211,4 +211,7 @@
   <data name="LogNewPowerShellManagerCreated" xml:space="preserve">
     <value>A new PowerShell manager instance is added to the pool. Current pool size '{0}'.</value>
   </data>
-</root>
+  <data name="DependencyDownloadInProgress" xml:space="preserve">
+    <value>Managed dependency download is in progress, function execution will continue when it's done.</value>
+  </data>
+</root>