From 2150743a176468b5ced7cadf23181896705991e4 Mon Sep 17 00:00:00 2001 From: Pankaj Agrawal Date: Fri, 24 Dec 2021 15:38:52 +0100 Subject: [PATCH 1/4] feat: Modern way of configuring json layout via JsonTemplateLayout --- pom.xml | 5 ++ powertools-logging/pom.xml | 5 +- .../logging/PowertoolsResolver.java | 51 +++++++++++ .../logging/PowertoolsResolverFactory.java | 34 +++++++ .../logging/internal/LambdaJsonLayout.java | 3 + .../src/main/resources/LambdaJsonLayout.json | 89 +++++++++++++++++++ .../resources/log4j2.component.properties | 2 + .../src/test/resources/log4j2.xml | 4 +- 8 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolver.java create mode 100644 powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolverFactory.java create mode 100644 powertools-logging/src/main/resources/LambdaJsonLayout.json create mode 100644 powertools-logging/src/main/resources/log4j2.component.properties diff --git a/pom.xml b/pom.xml index 16cc90f4f..99214bd43 100644 --- a/pom.xml +++ b/pom.xml @@ -167,6 +167,11 @@ log4j-api ${log4j.version} + + org.apache.logging.log4j + log4j-layout-template-json + ${log4j.version} + org.apache.logging.log4j log4j-jcl diff --git a/powertools-logging/pom.xml b/powertools-logging/pom.xml index bc9214ca5..eaf0b1cfc 100644 --- a/powertools-logging/pom.xml +++ b/powertools-logging/pom.xml @@ -53,7 +53,10 @@ com.fasterxml.jackson.core jackson-databind - + + org.apache.logging.log4j + log4j-layout-template-json + org.apache.logging.log4j log4j-core diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolver.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolver.java new file mode 100644 index 000000000..025834f32 --- /dev/null +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolver.java @@ -0,0 +1,51 @@ +package software.amazon.lambda.powertools.logging; + +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.layout.template.json.resolver.EventResolver; +import org.apache.logging.log4j.layout.template.json.util.JsonWriter; +import org.apache.logging.log4j.util.ReadOnlyStringMap; + +public final class PowertoolsResolver implements EventResolver { + + private final EventResolver internalResolver; + + PowertoolsResolver() { + internalResolver = new EventResolver() { + @Override + public boolean isResolvable(LogEvent value) { + ReadOnlyStringMap contextData = value.getContextData(); + return null != contextData && !contextData.isEmpty(); + } + + @Override + public void resolve(LogEvent logEvent, JsonWriter jsonWriter) { + StringBuilder stringBuilder = jsonWriter.getStringBuilder(); + // remove dummy field to kick inn powertools resolver + stringBuilder.setLength(stringBuilder.length() - 4); + + // Inject all the context information. + ReadOnlyStringMap contextData = logEvent.getContextData(); + contextData.forEach((key, value) -> { + jsonWriter.writeSeparator(); + jsonWriter.writeString(key); + stringBuilder.append(':'); + jsonWriter.writeValue(value); + }); + } + }; + } + + static String getName() { + return "powertools"; + } + + @Override + public void resolve(LogEvent value, JsonWriter jsonWriter) { + internalResolver.resolve(value, jsonWriter); + } + + @Override + public boolean isResolvable(LogEvent value) { + return internalResolver.isResolvable(value); + } +} diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolverFactory.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolverFactory.java new file mode 100644 index 000000000..dd256fcee --- /dev/null +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolverFactory.java @@ -0,0 +1,34 @@ +package software.amazon.lambda.powertools.logging; + +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.layout.template.json.resolver.EventResolverContext; +import org.apache.logging.log4j.layout.template.json.resolver.EventResolverFactory; +import org.apache.logging.log4j.layout.template.json.resolver.TemplateResolver; +import org.apache.logging.log4j.layout.template.json.resolver.TemplateResolverConfig; +import org.apache.logging.log4j.layout.template.json.resolver.TemplateResolverFactory; + +@Plugin(name = "PowertoolsResolverFactory", category = TemplateResolverFactory.CATEGORY) +public final class PowertoolsResolverFactory implements EventResolverFactory { + + private static final PowertoolsResolverFactory INSTANCE = new PowertoolsResolverFactory(); + + private PowertoolsResolverFactory() {} + + @PluginFactory + public static PowertoolsResolverFactory getInstance() { + return INSTANCE; + } + + @Override + public String getName() { + return PowertoolsResolver.getName(); + } + + @Override + public TemplateResolver create(EventResolverContext context, + TemplateResolverConfig config) { + return new PowertoolsResolver(); + } +} diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/LambdaJsonLayout.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/LambdaJsonLayout.java index ad7ca3cf7..c10ceaa42 100644 --- a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/LambdaJsonLayout.java +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/LambdaJsonLayout.java @@ -43,6 +43,9 @@ import static java.time.Instant.ofEpochMilli; import static java.time.format.DateTimeFormatter.ISO_ZONED_DATE_TIME; +/*** + * Note: The LambdaJsonLayout should be considered to be deprecated. Please use JsonTemplateLayout instead. + */ @Plugin(name = "LambdaJsonLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true) public final class LambdaJsonLayout extends AbstractJacksonLayoutCopy { private static final String DEFAULT_FOOTER = "]"; diff --git a/powertools-logging/src/main/resources/LambdaJsonLayout.json b/powertools-logging/src/main/resources/LambdaJsonLayout.json new file mode 100644 index 000000000..dfc1fc78f --- /dev/null +++ b/powertools-logging/src/main/resources/LambdaJsonLayout.json @@ -0,0 +1,89 @@ +{ + "timestamp": { + "$resolver": "timestamp" + }, + "instant": { + "epochSecond": { + "$resolver": "timestamp", + "epoch": { + "unit": "secs", + "rounded": true + } + }, + "nanoOfSecond": { + "$resolver": "timestamp", + "epoch": { + "unit": "secs.nanos" + } + } + }, + "thread": { + "$resolver": "thread", + "field": "name" + }, + "level": { + "$resolver": "level", + "field": "name" + }, + "loggerName": { + "$resolver": "logger", + "field": "name" + }, + "message": { + "$resolver": "message", + "stringified": true + }, + "thrown": { + "message": { + "$resolver": "exception", + "field": "message" + }, + "name": { + "$resolver": "exception", + "field": "className" + }, + "extendedStackTrace": { + "$resolver": "exception", + "field": "stackTrace" + } + }, + "contextStack": { + "$resolver": "ndc" + }, + "endOfBatch": { + "$resolver": "endOfBatch" + }, + "loggerFqcn": { + "$resolver": "logger", + "field": "fqcn" + }, + "threadId": { + "$resolver": "thread", + "field": "id" + }, + "threadPriority": { + "$resolver": "thread", + "field": "priority" + }, + "source": { + "class": { + "$resolver": "source", + "field": "className" + }, + "method": { + "$resolver": "source", + "field": "methodName" + }, + "file": { + "$resolver": "source", + "field": "fileName" + }, + "line": { + "$resolver": "source", + "field": "lineNumber" + } + }, + "": { + "$resolver": "powertools" + } +} diff --git a/powertools-logging/src/main/resources/log4j2.component.properties b/powertools-logging/src/main/resources/log4j2.component.properties new file mode 100644 index 000000000..3c392dd13 --- /dev/null +++ b/powertools-logging/src/main/resources/log4j2.component.properties @@ -0,0 +1,2 @@ +log4j.layout.jsonTemplate.timestampFormatPattern=yyyy-MM-dd'T'HH:mm:ss.SSSZz +#log4j.layout.jsonTemplate.timeZone= \ No newline at end of file diff --git a/powertools-logging/src/test/resources/log4j2.xml b/powertools-logging/src/test/resources/log4j2.xml index 108e32b75..22a44ee8b 100644 --- a/powertools-logging/src/test/resources/log4j2.xml +++ b/powertools-logging/src/test/resources/log4j2.xml @@ -1,8 +1,8 @@ - + - + From d04932e110ab5fd8fe8e8a478330e1de16106a98 Mon Sep 17 00:00:00 2001 From: Pankaj Agrawal Date: Sat, 25 Dec 2021 15:04:53 +0100 Subject: [PATCH 2/4] feat: Sample ECSLayout with powertools preconfigured --- .../src/main/resources/LambdaEcsLayout.json | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 powertools-logging/src/main/resources/LambdaEcsLayout.json diff --git a/powertools-logging/src/main/resources/LambdaEcsLayout.json b/powertools-logging/src/main/resources/LambdaEcsLayout.json new file mode 100644 index 000000000..4ab9c7ce2 --- /dev/null +++ b/powertools-logging/src/main/resources/LambdaEcsLayout.json @@ -0,0 +1,52 @@ +{ + "@timestamp": { + "$resolver": "timestamp", + "pattern": { + "format": "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", + "timeZone": "UTC" + } + }, + "ecs.version": "1.2.0", + "log.level": { + "$resolver": "level", + "field": "name" + }, + "message": { + "$resolver": "message", + "stringified": true + }, + "process.thread.name": { + "$resolver": "thread", + "field": "name" + }, + "log.logger": { + "$resolver": "logger", + "field": "name" + }, + "labels": { + "$resolver": "mdc", + "flatten": true, + "stringified": true + }, + "tags": { + "$resolver": "ndc" + }, + "error.type": { + "$resolver": "exception", + "field": "className" + }, + "error.message": { + "$resolver": "exception", + "field": "message" + }, + "error.stack_trace": { + "$resolver": "exception", + "field": "stackTrace", + "stackTrace": { + "stringified": true + } + }, + "": { + "$resolver": "powertools" + } +} \ No newline at end of file From 01dbd30b4d3eeb046202ab277164f33283cc9a01 Mon Sep 17 00:00:00 2001 From: Pankaj Agrawal Date: Sat, 25 Dec 2021 15:49:32 +0100 Subject: [PATCH 3/4] docs: Modern way of configuring json layout via JsonTemplateLayout --- docs/core/logging.md | 102 +++++++++++++++++- .../logging/internal/LambdaJsonLayout.java | 1 + 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/docs/core/logging.md b/docs/core/logging.md index 273b9ecb7..7acaaba73 100644 --- a/docs/core/logging.md +++ b/docs/core/logging.md @@ -13,16 +13,23 @@ Logging provides an opinionated logger with output structured as JSON. ## Initialization -Powertools extends the functionality of Log4J. Below is an example `#!xml log4j2.xml` file, with the `#!java LambdaJsonLayout` configured. +Powertools extends the functionality of Log4J. Below is an example `#!xml log4j2.xml` file, with the `JsonTemplateLayout` using `#!json LambdaJsonLayout.json` configured. + +!!! info "LambdaJsonLayout is now deprecated" + + Configuring utiltiy using `` plugin is deprecated now. While utility still supports the old configuration, we strongly recommend upgrading the + `log4j2.xml` configuration to `JsonTemplateLayout` instead. [JsonTemplateLayout](https://logging.apache.org/log4j/2.x/manual/json-template-layout.html) is recommended way of doing structured logging. + + Please follow [this guide](#upgrade-to-jsontemplatelayout-from-deprecated-lambdajsonlayout-configuration-in-log4j2xml) for upgrade steps. === "log4j2.xml" ```xml hl_lines="5" - + - + @@ -123,6 +130,44 @@ to customise what is logged. } ``` +### Customising fields in logs + +- Utility by default emits `timestamp` field in the logs in format `yyyy-MM-dd'T'HH:mm:ss.SSSZz` and in system default timezone. +If you need to customize format and timezone, you can do so by configuring `log4j2.component.properties` and configuring properties as shown in example below: + +=== "log4j2.component.properties" + + ```properties hl_lines="1 2" + log4j.layout.jsonTemplate.timestampFormatPattern=yyyy-MM-dd'T'HH:mm:ss.SSSZz + log4j.layout.jsonTemplate.timeZone=Europe/Oslo + ``` + +- Utility also provides sample template for [Elastic Common Schema(ECS)](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) layout. +The filed emitted in logs will follow specs from [ECS](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) together with field captured by utility as mentioned [above](#standard-structured-keys). + + Use `LambdaEcsLayout.json` as `eventTemplateUri` when configuring `JsonTemplateLayout`. + +=== "log4j2.xml" + + ```xml hl_lines="5" + + + + + + + + + + + + + + + + + ``` + ## Setting a Correlation ID You can set a Correlation ID using `correlationIdPath` attribute by passing a [JSON Pointer expression](https://datatracker.ietf.org/doc/html/draft-ietf-appsawg-json-pointer-03){target="_blank"}. @@ -417,3 +462,54 @@ via `samplingRate` attribute on annotation. Variables: POWERTOOLS_LOGGER_SAMPLE_RATE: 0.5 ``` + + +## Upgrade to JsonTemplateLayout from deprecated LambdaJsonLayout configuration in log4j2.xml + +Prior to version [1.10.0](https://github.com/awslabs/aws-lambda-powertools-java/releases/tag/v1.10.0), only supported way of configuring `log4j2.xml` was via ``. This plugin is +deprecated now and will be removed in future version. Switching to `JsonTemplateLayout` is straight forward. + +Below examples shows deprecated and new configuration of `log4j2.xml`. + +=== "Deprecated configuration of log4j2.xml" + + ```xml hl_lines="5" + + + + + + + + + + + + + + + + + ``` + +=== "New configuration of log4j2.xml" + + ```xml hl_lines="5" + + + + + + + + + + + + + + + + + ``` + diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/LambdaJsonLayout.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/LambdaJsonLayout.java index c10ceaa42..578937231 100644 --- a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/LambdaJsonLayout.java +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/LambdaJsonLayout.java @@ -46,6 +46,7 @@ /*** * Note: The LambdaJsonLayout should be considered to be deprecated. Please use JsonTemplateLayout instead. */ +@Deprecated @Plugin(name = "LambdaJsonLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true) public final class LambdaJsonLayout extends AbstractJacksonLayoutCopy { private static final String DEFAULT_FOOTER = "]"; From 3063711c1443b3173cfd35eabda12c7294d73657 Mon Sep 17 00:00:00 2001 From: Pankaj Agrawal Date: Sat, 25 Dec 2021 15:56:39 +0100 Subject: [PATCH 4/4] chore: Move modern plugins to internal package --- docs/core/logging.md | 2 +- .../logging/internal/AbstractJacksonLayoutCopy.java | 1 + .../powertools/logging/internal/JacksonFactoryCopy.java | 1 + .../powertools/logging/{ => internal}/PowertoolsResolver.java | 4 ++-- .../logging/{ => internal}/PowertoolsResolverFactory.java | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) rename powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/{ => internal}/PowertoolsResolver.java (93%) rename powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/{ => internal}/PowertoolsResolverFactory.java (95%) diff --git a/docs/core/logging.md b/docs/core/logging.md index 7acaaba73..f35c88067 100644 --- a/docs/core/logging.md +++ b/docs/core/logging.md @@ -143,7 +143,7 @@ If you need to customize format and timezone, you can do so by configuring `log4 ``` - Utility also provides sample template for [Elastic Common Schema(ECS)](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) layout. -The filed emitted in logs will follow specs from [ECS](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) together with field captured by utility as mentioned [above](#standard-structured-keys). +The field emitted in logs will follow specs from [ECS](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) together with field captured by utility as mentioned [above](#standard-structured-keys). Use `LambdaEcsLayout.json` as `eventTemplateUri` when configuring `JsonTemplateLayout`. diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/AbstractJacksonLayoutCopy.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/AbstractJacksonLayoutCopy.java index b1fafe3b7..3ceda4b79 100644 --- a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/AbstractJacksonLayoutCopy.java +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/AbstractJacksonLayoutCopy.java @@ -33,6 +33,7 @@ import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectWriter; +@Deprecated abstract class AbstractJacksonLayoutCopy extends AbstractStringLayout { protected static final String DEFAULT_EOL = "\r\n"; diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/JacksonFactoryCopy.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/JacksonFactoryCopy.java index 67ec01058..41247cfdb 100644 --- a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/JacksonFactoryCopy.java +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/JacksonFactoryCopy.java @@ -14,6 +14,7 @@ import java.util.HashSet; import java.util.Set; +@Deprecated abstract class JacksonFactoryCopy { static class JSON extends JacksonFactoryCopy { diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolver.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/PowertoolsResolver.java similarity index 93% rename from powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolver.java rename to powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/PowertoolsResolver.java index 025834f32..c392e2ed9 100644 --- a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolver.java +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/PowertoolsResolver.java @@ -1,11 +1,11 @@ -package software.amazon.lambda.powertools.logging; +package software.amazon.lambda.powertools.logging.internal; import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.layout.template.json.resolver.EventResolver; import org.apache.logging.log4j.layout.template.json.util.JsonWriter; import org.apache.logging.log4j.util.ReadOnlyStringMap; -public final class PowertoolsResolver implements EventResolver { +final class PowertoolsResolver implements EventResolver { private final EventResolver internalResolver; diff --git a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolverFactory.java b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/PowertoolsResolverFactory.java similarity index 95% rename from powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolverFactory.java rename to powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/PowertoolsResolverFactory.java index dd256fcee..5683c9688 100644 --- a/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/PowertoolsResolverFactory.java +++ b/powertools-logging/src/main/java/software/amazon/lambda/powertools/logging/internal/PowertoolsResolverFactory.java @@ -1,4 +1,4 @@ -package software.amazon.lambda.powertools.logging; +package software.amazon.lambda.powertools.logging.internal; import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.core.config.plugins.Plugin;