Added RetryContext (#712) (#713)

* Updated subtree from https://github.com/azure/azure-functions-language-worker-protobuf. Tag: v1.5.1-protobuf. Commit: 2fc14be59bbd8bb9f3f6275247047675b83fb3ca

* Add RetryContext

* Added tests

* Added unit tests and removed E2E tests

* Fix RetryContext unit test

* Updated release_notes.md

* Addressing requested changes

Author: michaelpeng36

protobuf/README.md

@@ -61,7 +61,7 @@ mkdir %MSGDIR%
 
 set OUTDIR=%MSGDIR%\DotNet
 mkdir %OUTDIR%
-%GRPC_TOOLS_PATH%\protoc.exe %PROTO% --csharp_out %OUTDIR% --grpc_out=%OUTDIR% --plugin=protoc-gen-grpc=%GRPC_TOOLS_PATH%\grpc_csharp_plugin.exe --proto_path=%PROTO_PATH% --proto_path=%PROTOBUF_TOOLS% 
+%GRPC_TOOLS_PATH%\protoc.exe %PROTO% --csharp_out %OUTDIR% --grpc_out=%OUTDIR% --plugin=protoc-gen-grpc=%GRPC_TOOLS_PATH%\grpc_csharp_plugin.exe --proto_path=%PROTO_PATH% --proto_path=%PROTOBUF_TOOLS%
 ```
 ## JavaScript
 In package.json, add to the build script the following commands to build .js files and to build .ts files. Use and install npm package `protobufjs`.
@@ -81,7 +81,10 @@ In pom.xml add following under configuration for this plugin
 <protoSourceRoot>${basedir}/<path to this repo>/azure-functions-language-worker-protobuf/src/proto</protoSourceRoot>
 
 ## Python
---TODO
+```
+python -m pip install -e .[dev] -U
+python setup.py build
+```
 
 ## Contributing
 

protobuf/src/proto/FunctionRpc.proto

@@ -67,10 +67,18 @@ message StreamingMessage {
 
     // Worker logs a message back to the host
     RpcLog rpc_log = 2;
-
+    
     FunctionEnvironmentReloadRequest function_environment_reload_request = 25;
 
     FunctionEnvironmentReloadResponse function_environment_reload_response = 26;
+
+    // Ask the worker to close any open shared memory resources for a given invocation
+    CloseSharedMemoryResourcesRequest close_shared_memory_resources_request = 27;
+    CloseSharedMemoryResourcesResponse close_shared_memory_resources_response = 28;
+
+    // Worker indexing message types
+    FunctionsMetadataRequest functions_metadata_request = 29;
+    FunctionMetadataResponses function_metadata_responses = 30;
   }
 }
 
@@ -201,6 +209,17 @@ message FunctionEnvironmentReloadResponse {
   StatusResult result = 3;
 }
 
+// Tell the out-of-proc worker to close any shared memory maps it allocated for given invocation
+message CloseSharedMemoryResourcesRequest {
+  repeated string map_names = 1;
+}
+
+// Response from the worker indicating which of the shared memory maps have been successfully closed and which have not been closed
+// The key (string) is the map name and the value (bool) is true if it was closed, false if not
+message CloseSharedMemoryResourcesResponse {
+  map<string, bool> close_map_results = 1;
+}
+
 // Host tells the worker to load a Function
 message FunctionLoadRequest {
   // unique function identifier (avoid name collisions, facilitate reload case)
@@ -245,6 +264,30 @@ message RpcFunctionMetadata {
 
   // Is set to true for proxy
   bool is_proxy = 7;
+
+  // Function indexing status
+  StatusResult status = 8;
+
+  // Function language
+  string language = 9;
+
+  // Raw binding info
+  repeated string raw_bindings = 10;
+}
+
+// Host tells worker it is ready to receive metadata
+message FunctionsMetadataRequest {
+  // base directory for function app
+  string function_app_directory = 1;
+}
+
+// Worker sends function metadata back to host
+message FunctionMetadataResponses {
+  // list of function indexing responses
+  repeated FunctionLoadRequest function_load_requests_results = 1;
+
+  // status of overall metadata request
+  StatusResult result = 2;
 }
 
 // Host requests worker to invoke a Function
@@ -263,6 +306,9 @@ message InvocationRequest {
 
   // Populates activityId, tracestate and tags from host
   RpcTraceContext trace_context = 5;
+
+  // Current retry context
+  RetryContext retry_context = 6;
 }
 
 // Host sends ActivityId, traceStateString and Tags from host
@@ -277,6 +323,18 @@ message RpcTraceContext {
 	map<string, string> attributes = 3;
 }
 
+// Host sends retry context for a function invocation
+message RetryContext {
+  // Current retry count
+  int32 retry_count = 1;
+
+  // Max retry count
+  int32 max_retry_count = 2;
+
+  // Exception that caused the retry
+  RpcException exception = 3;
+}
+
 // Host requests worker to cancel invocation
 message InvocationCancel {
   // Unique id for invocation
@@ -318,6 +376,34 @@ message TypedData {
   }
 }
 
+// Specify which type of data is contained in the shared memory region being read
+enum RpcDataType {
+  unknown = 0;
+  string = 1;
+  json = 2;
+  bytes = 3;
+  stream = 4;
+  http = 5;
+  int = 6;
+  double = 7;
+  collection_bytes = 8;
+  collection_string = 9;
+  collection_double = 10;
+  collection_sint64 = 11;
+}
+
+// Used to provide metadata about shared memory region to read data from
+message RpcSharedMemory {
+  // Name of the shared memory map containing data
+  string name = 1;
+  // Offset in the shared memory map to start reading data from
+  int64 offset = 2;
+  // Number of bytes to read (starting from the offset)
+  int64 count = 3;
+  // Final type to which the read data (in bytes) is to be interpreted as
+  RpcDataType type = 4;
+}
+
 // Used to encapsulate collection string
 message CollectionString {
   repeated string string = 1;
@@ -343,8 +429,13 @@ message ParameterBinding {
   // Name for the binding
   string name = 1;
 
-  // Data for the binding
-  TypedData data = 2;
+  oneof rpc_data {
+    // Data for the binding
+    TypedData data = 2;
+
+    // Metadata about the shared memory region to read data from
+    RpcSharedMemory rpc_shared_memory = 3;
+  }
 }
 
 // Used to describe a given binding on load
@@ -390,8 +481,9 @@ message RpcLog {
 
   // Category of the log. Defaults to User if not specified.
   enum RpcLogCategory {
-	User = 0;
-	System = 1;
+    User = 0;
+    System = 1;
+    CustomMetric = 2;
   }
 
   // Unique id for invocation (if exists)
@@ -413,11 +505,14 @@ message RpcLog {
   // Exception (if exists)
   RpcException exception = 6;
 
-  // json serialized property bag, or could use a type scheme like map<string, TypedData>
+  // json serialized property bag
   string properties = 7;
 
-  // Category of the log. Either user(default) or system.
+  // Category of the log. Either user(default), system, or custom metric.
   RpcLogCategory log_category = 8;
+
+  // strongly-typed (ish) property bag
+  map<string, TypedData> propertiesMap = 9;
 }
 
 // Encapsulates an Exception 
@@ -484,4 +579,7 @@ message RpcHttp {
   TypedData rawBody = 17;
   repeated RpcClaimsIdentity identities = 18;
   repeated RpcHttpCookie cookies = 19;
+  map<string,NullableString> nullable_headers = 20;
+  map<string,NullableString> nullable_params = 21;
+  map<string,NullableString> nullable_query = 22;
 }

src/FunctionInfo.cs

@@ -24,10 +24,12 @@ internal class AzFunctionInfo
     {
         internal const string TriggerMetadata = "TriggerMetadata";
         internal const string TraceContext = "TraceContext";
+        internal const string RetryContext = "RetryContext";
         internal const string DollarReturn = "$return";
 
         internal readonly bool HasTriggerMetadataParam;
         internal readonly bool HasTraceContextParam;
+        internal readonly bool HasRetryContextParam;
 
         internal readonly string FuncDirectory;
         internal readonly string FuncName;
@@ -76,6 +78,7 @@ internal AzFunctionInfo(RpcFunctionMetadata metadata)
             var parametersCopy = new Dictionary<string, PSScriptParamInfo>(psScriptParams, StringComparer.OrdinalIgnoreCase);
             HasTriggerMetadataParam = parametersCopy.Remove(TriggerMetadata);
             HasTraceContextParam = parametersCopy.Remove(TraceContext);
+            HasRetryContextParam = parametersCopy.Remove(RetryContext);
 
             var allBindings = new Dictionary<string, ReadOnlyBindingInfo>(StringComparer.OrdinalIgnoreCase);
             var inputBindings = new Dictionary<string, ReadOnlyBindingInfo>(StringComparer.OrdinalIgnoreCase);

src/PowerShell/PowerShellManager.cs

@@ -199,6 +199,7 @@ public Hashtable InvokeFunction(
             AzFunctionInfo functionInfo,
             Hashtable triggerMetadata,
             TraceContext traceContext,
+            RetryContext retryContext,
             IList<ParameterBinding> inputData,
             FunctionInvocationPerformanceStopwatch stopwatch)
         {
@@ -213,7 +214,7 @@ public Hashtable InvokeFunction(
                 AddEntryPointInvocationCommand(functionInfo);
                 stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.FunctionCodeReady);
 
-                SetInputBindingParameterValues(functionInfo, inputData, durableController, triggerMetadata, traceContext);
+                SetInputBindingParameterValues(functionInfo, inputData, durableController, triggerMetadata, traceContext, retryContext);
                 stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.InputBindingValuesReady);
 
                 if (!durableController.ShouldSuppressPipelineTraces())
@@ -258,7 +259,8 @@ private void SetInputBindingParameterValues(
             IEnumerable<ParameterBinding> inputData,
             DurableController durableController,
             Hashtable triggerMetadata,
-            TraceContext traceContext)
+            TraceContext traceContext,
+            RetryContext retryContext)
         {
             foreach (var binding in inputData)
             {
@@ -284,6 +286,11 @@ private void SetInputBindingParameterValues(
             {
                 _pwsh.AddParameter(AzFunctionInfo.TraceContext, traceContext);
             }
+
+            if (functionInfo.HasRetryContextParam)
+            {
+                _pwsh.AddParameter(AzFunctionInfo.RetryContext, retryContext);
+            }
         }
 
         /// <summary>

src/Public/RetryContext.cs

@@ -0,0 +1,28 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using Microsoft.Azure.WebJobs.Script.Grpc.Messages;
+
+namespace Microsoft.Azure.Functions.PowerShellWorker
+{
+    /// <summary>
+    /// Custom RetryContext constructed from the RpcRetryContext member received from the host.
+    /// </summary>
+    internal class RetryContext
+    {
+        public RetryContext(int retryCount, int maxRetryCount, RpcException exception)
+        {
+            RetryCount = retryCount;
+            MaxRetryCount = maxRetryCount;
+            Exception = exception;
+        }
+
+        public int RetryCount { get; }
+
+        public int MaxRetryCount { get; }
+
+        public RpcException Exception { get; }
+    }
+}

src/RequestProcessor.cs

@@ -329,9 +329,10 @@ private Hashtable InvokeFunction(
         {
             var triggerMetadata = GetTriggerMetadata(functionInfo, invocationRequest);
             var traceContext = GetTraceContext(functionInfo, invocationRequest);
+            var retryContext = GetRetryContext(functionInfo, invocationRequest);
             stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.MetadataAndTraceContextReady);
 
-            return psManager.InvokeFunction(functionInfo, triggerMetadata, traceContext, invocationRequest.InputData, stopwatch);
+            return psManager.InvokeFunction(functionInfo, triggerMetadata, traceContext, retryContext, invocationRequest.InputData, stopwatch);
         }
 
         internal StreamingMessage ProcessInvocationCancelRequest(StreamingMessage request)
@@ -452,6 +453,19 @@ private static TraceContext GetTraceContext(AzFunctionInfo functionInfo, Invocat
                 invocationRequest.TraceContext.Attributes);
         }
 
+        private static RetryContext GetRetryContext(AzFunctionInfo functionInfo, InvocationRequest invocationRequest)
+        {
+            if (!functionInfo.HasRetryContextParam)
+            {
+                return null;
+            }
+
+            return new RetryContext(
+                invocationRequest.RetryContext.RetryCount,
+                invocationRequest.RetryContext.MaxRetryCount,
+                invocationRequest.RetryContext.Exception);
+        }
+
         /// <summary>
         /// Set the 'ReturnValue' and 'OutputData' based on the invocation results appropriately.
         /// </summary>

src/Utility/TypeExtensions.cs

@@ -153,9 +153,9 @@ internal static RpcException ToRpcException(this Exception exception)
         {
             return new RpcException
             {
-                Message = exception.Message,
                 Source = exception.Source ?? "",
-                StackTrace = exception.StackTrace ?? ""
+                StackTrace = exception.StackTrace ?? "",
+                Message = exception.Message
             };
         }
 

test/Unit/Function/FunctionLoaderTests.cs

@@ -100,10 +100,12 @@ public void TestFunctionLoaderGetFuncWithRequires()
             Assert.Single(funcInfo.OutputBindings);
         }
 
-        [Fact]
-        public void TestFunctionLoaderGetFuncWithTriggerMetadataParam()
+        [Theory]
+        [InlineData("TriggerMetadata")]
+        [InlineData("RetryContext")]
+        public void TestFunctionLoaderGetFuncWithSingleParam(string paramName)
         {
-            var scriptFileToUse = Path.Join(_functionDirectory, "BasicFuncScriptWithTriggerMetadata.ps1");
+            var scriptFileToUse = Path.Join(_functionDirectory, $"BasicFuncScriptWith{paramName}.ps1");
             var entryPointToUse = string.Empty;
             var functionLoadRequest = GetFuncLoadRequest(scriptFileToUse, entryPointToUse);
 
@@ -118,7 +120,7 @@ public void TestFunctionLoaderGetFuncWithTriggerMetadataParam()
             Assert.Equal(3, funcInfo.FuncParameters.Count);
             Assert.True(funcInfo.FuncParameters.ContainsKey("req"));
             Assert.True(funcInfo.FuncParameters.ContainsKey("inputBlob"));
-            Assert.True(funcInfo.FuncParameters.ContainsKey("TriggerMetadata"));
+            Assert.True(funcInfo.FuncParameters.ContainsKey(paramName));
 
             Assert.Equal(3, funcInfo.AllBindings.Count);
             Assert.Equal(2, funcInfo.InputBindings.Count);
@@ -229,10 +231,12 @@ public void ParametersShouldMatchInputBinding()
             Assert.Contains("inputBlob", exception.Message);
         }
 
-        [Fact]
-        public void ParametersShouldMatchInputBindingWithTriggerMetadataParam()
+        [Theory]
+        [InlineData("TriggerMetadata")]
+        [InlineData("RetryContext")]
+        public void ParametersShouldMatchInputBindingWithSingleParam(string paramName)
         {
-            var scriptFileToUse = Path.Join(_functionDirectory, "BasicFuncScriptWithTriggerMetadata.ps1");
+            var scriptFileToUse = Path.Join(_functionDirectory, $"BasicFuncScriptWith{paramName}.ps1");
             var entryPointToUse = string.Empty;
 
             var functionLoadRequest = GetFuncLoadRequest(scriptFileToUse, entryPointToUse);
@@ -262,9 +266,9 @@ public void EntryPointParametersShouldMatchInputBinding()
         }
 
         [Fact]
-        public void EntryPointParametersShouldMatchInputBindingWithTriggerMetadataParam()
+        public void EntryPointParametersShouldMatchInputBindingWithTriggerMetadataAndRetryContextParams()
         {
-            var scriptFileToUse = Path.Join(_functionDirectory, "FuncWithEntryPointAndTriggerMetadata.psm1");
+            var scriptFileToUse = Path.Join(_functionDirectory, "FuncWithEntryPointAndTriggerMetadataAndRetryContext.psm1");
             var entryPointToUse = "Run";
 
             var functionLoadRequest = GetFuncLoadRequest(scriptFileToUse, entryPointToUse);