Merge pull request #516 from hjgraca/lambda-log-level

feat: New log level

Author: hjgraca

docs/core/logging.md

@@ -69,6 +69,51 @@ Here is an example using the AWS SAM [Globals section](https://docs.aws.amazon.c
 | **POWERTOOLS_LOGGER_LOG_EVENT** | Logs incoming event |  `false` |
 | **POWERTOOLS_LOGGER_SAMPLE_RATE** | Debug log sampling |  `0` |
 
+
+### Using AWS Lambda Advanced Logging Controls (ALC)
+
+With [AWS Lambda Advanced Logging Controls (ALC)](https://docs.aws.amazon.com/lambda/latest/dg/monitoring-cloudwatchlogs.html#monitoring-cloudwatchlogs-advanced), you can control the output format of your logs as either TEXT or JSON and specify the minimum accepted log level for your application. Regardless of the output format setting in Lambda, Powertools for AWS Lambda will always output JSON formatted logging messages.
+
+When you have this feature enabled, log messages that don’t meet the configured log level are discarded by Lambda. For example, if you set the minimum log level to WARN, you will only receive WARN and ERROR messages in your AWS CloudWatch Logs, all other log levels will be discarded by Lambda.
+
+!!! warning "When using AWS Lambda Advanced Logging Controls (ALC)"
+    - When Powertools Logger output is set to `PascalCase` **`Level`**  property name will be replaced by **`LogLevel`** as a property name.
+    - ALC takes precedence over **`POWERTOOLS_LOG_LEVEL`** and when setting it in code using **`[Logging(LogLevel = )]`**
+
+```mermaid
+sequenceDiagram
+    title Lambda ALC allows WARN logs only
+    participant Lambda service
+    participant Lambda function
+    participant Application Logger
+    
+    Note over Lambda service: AWS_LAMBDA_LOG_LEVEL="WARN"
+    Lambda service->>Lambda function: Invoke (event)
+    Lambda function->>Lambda function: Calls handler
+    Lambda function->>Application Logger: Logger.Warning("Something happened")
+    Lambda function-->>Application Logger: Logger.Debug("Something happened")
+    Lambda function-->>Application Logger: Logger.Information("Something happened")
+    
+    Lambda service->>Lambda service: DROP INFO and DEBUG logs
+
+    Lambda service->>CloudWatch Logs: Ingest error logs
+```
+
+Logger will automatically listen for the AWS_LAMBDA_LOG_FORMAT and AWS_LAMBDA_LOG_LEVEL environment variables, and change behaviour if they’re found to ensure as much compatibility as possible.
+
+**Priority of log level settings in Powertools for AWS Lambda**
+
+When the Advanced Logging Controls feature is enabled, we are unable to increase the minimum log level below the AWS_LAMBDA_LOG_LEVEL environment variable value, see [AWS Lambda service documentation](https://docs.aws.amazon.com/lambda/latest/dg/monitoring-cloudwatchlogs.html#monitoring-cloudwatchlogs-log-level) for more details.
+
+We prioritise log level settings in this order:
+
+1. AWS_LAMBDA_LOG_LEVEL environment variable
+2. Setting the log level in code using `[Logging(LogLevel = )]`
+3. POWERTOOLS_LOG_LEVEL environment variable
+
+In the event you have set POWERTOOLS_LOG_LEVEL to a level lower than the ACL setting, Powertools for AWS Lambda will output a warning log message informing you that your messages will be discarded by Lambda.
+
+
 ## Standard structured keys
 
 Your logs will always include the following keys to your structured logging:

libraries/src/AWS.Lambda.Powertools.Common/Core/Constants.cs

@@ -54,6 +54,11 @@ internal static class Constants
     ///     Constant for POWERTOOLS_LOG_LEVEL environment variable
     /// </summary>
     internal const string LogLevelNameEnv = "POWERTOOLS_LOG_LEVEL";
+    
+    /// <summary>
+    ///     Constant for POWERTOOLS_LOG_LEVEL environment variable
+    /// </summary>
+    internal const string AWSLambdaLogLevelNameEnv = "AWS_LAMBDA_LOG_LEVEL";
 
     /// <summary>
     ///     Constant for POWERTOOLS_LOGGER_SAMPLE_RATE environment variable

libraries/src/AWS.Lambda.Powertools.Common/Core/IPowertoolsConfigurations.cs

@@ -57,10 +57,16 @@ public interface IPowertoolsConfigurations
     string MetricsNamespace { get; }
 
     /// <summary>
-    ///     Gets the log level.
+    ///     Gets the Powertools log level.
     /// </summary>
     /// <value>The log level.</value>
     string LogLevel { get; }
+    
+    /// <summary>
+    ///     Gets the AWS Lambda log level.
+    /// </summary>
+    /// <value>The log level.</value>
+    string AWSLambdaLogLevel { get; }
 
     /// <summary>
     ///     Gets the logger sample rate.

libraries/src/AWS.Lambda.Powertools.Common/Core/PowertoolsConfigurations.cs

@@ -147,12 +147,11 @@ public bool GetEnvironmentVariableOrDefault(string variable, bool defaultValue)
     public string MetricsNamespace =>
         GetEnvironmentVariable(Constants.MetricsNamespaceEnv);
 
-    /// <summary>
-    ///     Gets the log level.
-    /// </summary>
-    /// <value>The log level.</value>
-    public string LogLevel =>
-        GetEnvironmentVariable(Constants.LogLevelNameEnv);
+    /// <inheritdoc />
+    public string LogLevel => GetEnvironmentVariable(Constants.LogLevelNameEnv);
+
+    /// <inheritdoc />
+    public string AWSLambdaLogLevel => GetEnvironmentVariable(Constants.AWSLambdaLogLevelNameEnv);
 
     /// <summary>
     ///     Gets the logger sample rate.

libraries/src/AWS.Lambda.Powertools.Common/Core/SystemWrapper.cs

@@ -1,19 +1,20 @@
 /*
  * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- * 
+ *
  * Licensed under the Apache License, Version 2.0 (the "License").
  * You may not use this file except in compliance with the License.
  * A copy of the License is located at
- * 
+ *
  *  http://aws.amazon.com/apache2.0
- * 
+ *
  * or in the "license" file accompanying this file. This file is distributed
  * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
  * express or implied. See the License for the specific language governing
  * permissions and limitations under the License.
  */
 
 using System;
+using System.IO;
 using System.Text;
 
 namespace AWS.Lambda.Powertools.Common;
@@ -39,6 +40,14 @@ public SystemWrapper(IPowertoolsEnvironment powertoolsEnvironment)
     {
         _powertoolsEnvironment = powertoolsEnvironment;
         _instance ??= this;
+        
+        // Clear AWS SDK Console injected parameters StdOut and StdErr
+        var standardOutput = new StreamWriter(Console.OpenStandardOutput());
+        standardOutput.AutoFlush = true;
+        Console.SetOut(standardOutput);
+        var errordOutput = new StreamWriter(Console.OpenStandardError());
+        errordOutput.AutoFlush = true;
+        Console.SetError(errordOutput);
     }
 
     /// <summary>
@@ -97,21 +106,21 @@ public void SetExecutionEnvironment<T>(T type)
         var envValue = new StringBuilder();
         var currentEnvValue = GetEnvironmentVariable(envName);
         var assemblyName = ParseAssemblyName(_powertoolsEnvironment.GetAssemblyName(type));
-        
+
         // If there is an existing execution environment variable add the annotations package as a suffix.
-        if(!string.IsNullOrEmpty(currentEnvValue))
+        if (!string.IsNullOrEmpty(currentEnvValue))
         {
             // Avoid duplication - should not happen since the calling Instances are Singletons - defensive purposes
             if (currentEnvValue.Contains(assemblyName))
             {
                 return;
             }
-            
+
             envValue.Append($"{currentEnvValue} ");
         }
 
         var assemblyVersion = _powertoolsEnvironment.GetAssemblyVersion(type);
-        
+
         envValue.Append($"{assemblyName}/{assemblyVersion}");
 
         SetEnvironmentVariable(envName, envValue.ToString());
@@ -127,13 +136,14 @@ private string ParseAssemblyName(string assemblyName)
     {
         try
         {
-            var parsedName = assemblyName.Substring(assemblyName.LastIndexOf(".", StringComparison.Ordinal)+1);
+            var parsedName = assemblyName.Substring(assemblyName.LastIndexOf(".", StringComparison.Ordinal) + 1);
             return $"{Constants.FeatureContextIdentifier}/{parsedName}";
         }
         catch
         {
             //NOOP
         }
+
         return $"{Constants.FeatureContextIdentifier}/{assemblyName}";
     }
 }

libraries/src/AWS.Lambda.Powertools.Logging/Internal/PowertoolsConfigurations.cs

@@ -14,6 +14,7 @@
  */
 
 using System;
+using System.Collections.Generic;
 using AWS.Lambda.Powertools.Common;
 using Microsoft.Extensions.Logging;
 
@@ -42,6 +43,18 @@ internal static LogLevel GetLogLevel(this IPowertoolsConfigurations powertoolsCo
         return LoggingConstants.DefaultLogLevel;
     }
 
+    internal static LogLevel GetLambdaLogLevel(this IPowertoolsConfigurations powertoolsConfigurations)
+    {
+        AwsLogLevelMapper.TryGetValue((powertoolsConfigurations.AWSLambdaLogLevel ?? "").Trim().ToUpper(), out var awsLogLevel);
+
+        if (Enum.TryParse(awsLogLevel, true, out LogLevel result))
+        {
+            return result;
+        }
+
+        return LogLevel.None;
+    }
+    
     internal static LoggerOutputCase GetLoggerOutputCase(this IPowertoolsConfigurations powertoolsConfigurations,
         LoggerOutputCase? loggerOutputCase = null)
     {
@@ -53,4 +66,14 @@ internal static LoggerOutputCase GetLoggerOutputCase(this IPowertoolsConfigurati
 
         return LoggingConstants.DefaultLoggerOutputCase;
     }
+
+    private static Dictionary<string, string> AwsLogLevelMapper = new()
+    {
+        { "TRACE", "TRACE" },
+        { "DEBUG", "DEBUG" },
+        { "INFO", "INFORMATION" },
+        { "WARN", "WARNING" },
+        { "ERROR", "ERROR" },
+        { "FATAL", "CRITICAL" }
+    };
 }

libraries/src/AWS.Lambda.Powertools.Logging/Internal/PowertoolsLogger.cs

@@ -50,17 +50,16 @@ internal sealed class PowertoolsLogger : ILogger
     ///     The system wrapper
     /// </summary>
     private readonly ISystemWrapper _systemWrapper;
-
-    /// <summary>
-    ///     The current configuration
-    /// </summary>
-    private LoggerConfiguration _currentConfig;
 
     /// <summary>
     ///     The JsonSerializer options
     /// </summary>
     private JsonSerializerOptions _jsonSerializerOptions;
 
+    private LogLevel _lambdaLogLevel;
+    private LogLevel _logLevel;
+    private bool _lambdaLogLevelEnabled;
+
     /// <summary>
     ///     Initializes a new instance of the <see cref="PowertoolsLogger" /> class.
     /// </summary>
@@ -78,14 +77,17 @@ public PowertoolsLogger(
             powertoolsConfigurations, systemWrapper, getCurrentConfig);
 
         _powertoolsConfigurations.SetExecutionEnvironment(this);
+        CurrentConfig = GetCurrentConfig();
+        
+        if (_lambdaLogLevelEnabled && _logLevel < _lambdaLogLevel)
+        {
+            var message =
+                $"Current log level ({_logLevel}) does not match AWS Lambda Advanced Logging Controls minimum log level ({_lambdaLogLevel}). This can lead to data loss, consider adjusting them.";
+            this.LogWarning(message);
+        }
     }
 
-    /// <summary>
-    ///     Sets the current configuration.
-    /// </summary>
-    /// <value>The current configuration.</value>
-    private LoggerConfiguration CurrentConfig =>
-        _currentConfig ??= GetCurrentConfig();
+    private LoggerConfiguration CurrentConfig { get; set; }
 
     /// <summary>
     ///     Sets the minimum level.
@@ -255,8 +257,15 @@ private Dictionary<string, object> GetLogEntry(LogLevel logLevel, DateTime times
             }
         }
 
+        var keyLogLevel = LoggingConstants.KeyLogLevel;
+        // If ALC is enabled and PascalCase we need to convert Level to LogLevel for it to be parsed and sent to CW
+        if (_lambdaLogLevelEnabled && CurrentConfig.LoggerOutputCase == LoggerOutputCase.PascalCase)
+        {
+            keyLogLevel = "LogLevel";
+        }
+
         logEntry.TryAdd(LoggingConstants.KeyTimestamp, timestamp.ToString("o"));
-        logEntry.TryAdd(LoggingConstants.KeyLogLevel, logLevel.ToString());
+        logEntry.TryAdd(keyLogLevel, logLevel.ToString());
         logEntry.TryAdd(LoggingConstants.KeyService, Service);
         logEntry.TryAdd(LoggingConstants.KeyLoggerName, _name);
         logEntry.TryAdd(LoggingConstants.KeyMessage, message);
@@ -361,7 +370,7 @@ private object GetFormattedLogEntry(LogLevel logLevel, DateTime timestamp, objec
     /// </summary>
     internal void ClearConfig()
     {
-        _currentConfig = null;
+        CurrentConfig = null;
     }
 
     /// <summary>
@@ -371,14 +380,22 @@ internal void ClearConfig()
     private LoggerConfiguration GetCurrentConfig()
     {
         var currConfig = _getCurrentConfig();
-        var minimumLevel = _powertoolsConfigurations.GetLogLevel(currConfig?.MinimumLevel);
+        _logLevel = _powertoolsConfigurations.GetLogLevel(currConfig?.MinimumLevel);
         var samplingRate = currConfig?.SamplingRate ?? _powertoolsConfigurations.LoggerSampleRate;
         var loggerOutputCase =  _powertoolsConfigurations.GetLoggerOutputCase(currConfig?.LoggerOutputCase);
-
+        _lambdaLogLevel = _powertoolsConfigurations.GetLambdaLogLevel();
+        _lambdaLogLevelEnabled = _lambdaLogLevel != LogLevel.None;
+        
+        var minLogLevel = _logLevel;
+        if (_lambdaLogLevelEnabled)
+        {
+            minLogLevel = _lambdaLogLevel;
+        }
+        
         var config = new LoggerConfiguration
         {
             Service = currConfig?.Service,
-            MinimumLevel = minimumLevel,
+            MinimumLevel = minLogLevel,
             SamplingRate = samplingRate,
             LoggerOutputCase = loggerOutputCase
         };
@@ -388,7 +405,7 @@ private LoggerConfiguration GetCurrentConfig()
 
         if (samplingRate.Value < 0 || samplingRate.Value > 1)
         {
-            if (minimumLevel is LogLevel.Debug or LogLevel.Trace)
+            if (minLogLevel is LogLevel.Debug or LogLevel.Trace)
                 _systemWrapper.LogLine(
                     $"Skipping sampling rate configuration because of invalid value. Sampling rate: {samplingRate.Value}");
             config.SamplingRate = null;