From 8d17e97340dc96feaa8c733fd475eac7a0b3aa8a Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Sun, 29 Mar 2020 13:25:37 +0200 Subject: [PATCH 1/2] Serialize MDC under top level Makes it easier to set ECS fields via MDC. Previously, you'd have to configure topLevelLabels for each ECS field added via MDC. Now, if you want to have a field nested under labels, you have to explicitly prefix it with `labels.`. --- README.md | 9 ++++- .../co/elastic/logging/EcsJsonSerializer.java | 13 ++----- .../logging/AbstractEcsLoggingTest.java | 14 ++----- .../co/elastic/logging/log4j/EcsLayout.java | 11 +----- log4j2-ecs-layout/README.md | 1 - .../co/elastic/logging/log4j2/EcsLayout.java | 39 +++---------------- .../log4j2/AbstractLog4j2EcsLayoutTest.java | 8 ++-- .../logging/log4j2/Log4j2EcsLayoutTest.java | 1 - .../src/test/resources/log4j2-test.xml | 2 +- .../elastic/logging/logback/EcsEncoder.java | 9 +---- 10 files changed, 28 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index 74f2a7dd..f90e1822 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,16 @@ We recommend using this library to log into a JSON log file and let Filebeat sen |[`error.message`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html)|[`Throwable#getMessage()`](https://docs.oracle.com/javase/7/docs/api/java/lang/Throwable.html#getMessage())| |[`error.stack_trace`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html)|[`Throwable#getStackTrace()`](https://docs.oracle.com/javase/7/docs/api/java/lang/Throwable.html#getStackTrace())| |[`process.thread.name`](https://www.elastic.co/guide/en/ecs/current/ecs-process.html)|[`LogEvent#getThreadName()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getThreadName()) | -|[`labels`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html)|[`LogEvent#getContextMap()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextMap())| +|Each MDC entry is a top-level field 1|[`LogEvent#getContextMap()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextMap())| |[`tags`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html)|[`LogEvent#getContextStack()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextStack())| +1 It's recommended to use existing [ECS fields](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for MDC values. + +If there is no appropriate ECS field, +consider prefixing your fields with `labels.`, as in `labels.foo`, for simple key/value pairs. +For nested structures consider prefixing with `custom.` to make sure you won't get conflicts if ECS later adds the same fields but with a different mapping. + + ## Getting Started ### Step 1: Configure application logging diff --git a/ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java b/ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java index 829912f3..f8b8a90f 100644 --- a/ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java +++ b/ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java @@ -26,14 +26,10 @@ import java.io.PrintWriter; import java.io.Writer; -import java.util.Arrays; -import java.util.List; import java.util.Map; -import java.util.Set; public class EcsJsonSerializer { - public static final List DEFAULT_TOP_LEVEL_LABELS = Arrays.asList("trace.id", "transaction.id", "span.id", "error.id", "service.name"); private static final TimestampSerializer TIMESTAMP_SERIALIZER = new TimestampSerializer(); private static final ThreadLocal messageStringBuilder = new ThreadLocal(); private static final String NEW_LINE = System.getProperty("line.separator"); @@ -138,14 +134,11 @@ public static void serializeOrigin(StringBuilder builder, String fileName, Strin builder.append("},"); } - public static void serializeLabels(StringBuilder builder, Map labels, Set topLevelLabels) { - if (!labels.isEmpty()) { - for (Map.Entry entry : labels.entrySet()) { + public static void serializeMDC(StringBuilder builder, Map properties) { + if (!properties.isEmpty()) { + for (Map.Entry entry : properties.entrySet()) { builder.append('\"'); String key = entry.getKey(); - if (!topLevelLabels.contains(key)) { - builder.append("labels."); - } JsonUtils.quoteAsString(key, builder); builder.append("\":\""); JsonUtils.quoteAsString(toNullSafeString(String.valueOf(entry.getValue())), builder); diff --git a/ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java b/ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java index bf84d436..50e8a088 100644 --- a/ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java +++ b/ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java @@ -62,7 +62,7 @@ void testSimpleLog() throws Exception { void testThreadContext() throws Exception { putMdc("foo", "bar"); debug("test"); - assertThat(getLastLogLine().get("labels.foo").textValue()).isEqualTo("bar"); + assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar"); } @Test @@ -74,21 +74,15 @@ void testThreadContextStack() throws Exception { } @Test - void testTopLevelLabels() throws Exception { + void testMdc() throws Exception { putMdc("transaction.id", "0af7651916cd43dd8448eb211c80319c"); putMdc("span.id", "foo"); + putMdc("foo", "bar"); debug("test"); assertThat(getLastLogLine().get("labels.transaction.id")).isNull(); assertThat(getLastLogLine().get("transaction.id").textValue()).isEqualTo("0af7651916cd43dd8448eb211c80319c"); assertThat(getLastLogLine().get("span.id").textValue()).isEqualTo("foo"); - } - - @Test - void testCustomTopLevelLabels() throws Exception { - putMdc("top_level", "foo"); - debug("test"); - assertThat(getLastLogLine().get("labels.top_level")).isNull(); - assertThat(getLastLogLine().get("top_level").textValue()).isEqualTo("foo"); + assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar"); } @Test diff --git a/log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java b/log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java index b50d1730..0b6b0380 100644 --- a/log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java +++ b/log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java @@ -30,14 +30,10 @@ import org.apache.log4j.spi.LoggingEvent; import org.apache.log4j.spi.ThrowableInformation; -import java.util.HashSet; -import java.util.Set; - public class EcsLayout extends Layout { private boolean stackTraceAsArray = false; private String serviceName; - private Set topLevelLabels = new HashSet(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS); private boolean includeOrigin; @Override @@ -49,7 +45,7 @@ public String format(LoggingEvent event) { EcsJsonSerializer.serializeServiceName(builder, serviceName); EcsJsonSerializer.serializeThreadName(builder, event.getThreadName()); EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName()); - EcsJsonSerializer.serializeLabels(builder, event.getProperties(), topLevelLabels); + EcsJsonSerializer.serializeMDC(builder, event.getProperties()); EcsJsonSerializer.serializeTag(builder, event.getNDC()); if (includeOrigin) { LocationInfo locationInformation = event.getLocationInformation(); @@ -99,8 +95,5 @@ public void setIncludeOrigin(boolean includeOrigin) { public void setStackTraceAsArray(boolean stackTraceAsArray) { this.stackTraceAsArray = stackTraceAsArray; } - - public void addTopLevelLabel(String topLevelLabel) { - this.topLevelLabels.add(topLevelLabel); - } + } diff --git a/log4j2-ecs-layout/README.md b/log4j2-ecs-layout/README.md index 5f016b0e..ed89c09c 100644 --- a/log4j2-ecs-layout/README.md +++ b/log4j2-ecs-layout/README.md @@ -43,7 +43,6 @@ Instead of the usual ``, use `` |Parameter name |Type |Default|Description| |-----------------|-------|-------|-----------| |serviceName |String | |Sets the `service.name` field so you can filter your logs by a particular service | -|topLevelLabels |String |`trace.id, transaction.id, span.id, error.id, service.name`|Usually, MDC keys are nested under [`labels`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html). You can specify a comma-separated list of properties which should be on the top level. | |includeMarkers |boolean|`false`|Log [Markers](https://logging.apache.org/log4j/2.0/manual/markers.html) as [`tags`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html) | |stackTraceAsArray|boolean|`false`|Serializes the [`error.stack_trace`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html) as a JSON array where each element is in a new line to improve readability. Note that this requires a slightly more complex [Filebeat configuration](../README.md#when-stacktraceasarray-is-enabled).| |includeOrigin |boolean|`false`|If `true`, adds the [`log.origin.file.name`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html), [`log.origin.file.line`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) and [`log.origin.function`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) fields. Note that you also have to set `includeLocation="true"` on your loggers and appenders if you are using the async ones. | diff --git a/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java b/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java index 052ee852..45ce19b1 100644 --- a/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java +++ b/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java @@ -51,11 +51,7 @@ import org.apache.logging.log4j.util.TriConsumer; import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -65,13 +61,10 @@ public class EcsLayout extends AbstractStringLayout { public static final Charset UTF_8 = Charset.forName("UTF-8"); public static final String[] JSON_FORMAT = {"JSON"}; - private final TriConsumer WRITE_KEY_VALUES_INTO = new TriConsumer() { + private final TriConsumer WRITE_MDC = new TriConsumer() { @Override public void accept(final String key, final Object value, final StringBuilder stringBuilder) { stringBuilder.append('\"'); - if (!topLevelLabels.contains(key)) { - stringBuilder.append("labels."); - } JsonUtils.quoteAsString(key, stringBuilder); stringBuilder.append("\":\""); JsonUtils.quoteAsString(EcsJsonSerializer.toNullSafeString(String.valueOf(value)), stringBuilder); @@ -81,7 +74,6 @@ public void accept(final String key, final Object value, final StringBuilder str private final KeyValuePair[] additionalFields; private final PatternFormatter[][] fieldValuePatternFormatter; - private final Set topLevelLabels; private final boolean stackTraceAsArray; private final String serviceName; private final boolean includeMarkers; @@ -89,12 +81,10 @@ public void accept(final String key, final Object value, final StringBuilder str private final ConcurrentMap, Boolean> supportsJson = new ConcurrentHashMap, Boolean>(); private final ObjectMessageJacksonSerializer objectMessageJacksonSerializer = ObjectMessageJacksonSerializer.Resolver.INSTANCE.resolve(); - private EcsLayout(Configuration config, String serviceName, boolean includeMarkers, KeyValuePair[] additionalFields, Collection topLevelLabels, boolean includeOrigin, boolean stackTraceAsArray) { + private EcsLayout(Configuration config, String serviceName, boolean includeMarkers, KeyValuePair[] additionalFields, boolean includeOrigin, boolean stackTraceAsArray) { super(config, UTF_8, null, null); this.serviceName = serviceName; this.includeMarkers = includeMarkers; - this.topLevelLabels = new HashSet(topLevelLabels); - this.topLevelLabels.addAll(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS); this.includeOrigin = includeOrigin; this.stackTraceAsArray = stackTraceAsArray; this.additionalFields = additionalFields; @@ -138,7 +128,7 @@ private StringBuilder toText(LogEvent event, StringBuilder builder, boolean gcFr EcsJsonSerializer.serializeServiceName(builder, serviceName); EcsJsonSerializer.serializeThreadName(builder, event.getThreadName()); EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName()); - serializeLabels(event, builder); + serializeAdditionalFieldsAndMDC(event, builder); serializeTags(event, builder); if (includeOrigin) { EcsJsonSerializer.serializeOrigin(builder, event.getSource()); @@ -148,7 +138,7 @@ private StringBuilder toText(LogEvent event, StringBuilder builder, boolean gcFr return builder; } - private void serializeLabels(LogEvent event, StringBuilder builder) { + private void serializeAdditionalFieldsAndMDC(LogEvent event, StringBuilder builder) { final int length = additionalFields.length; if (!event.getContextData().isEmpty() || length > 0) { if (length > 0) { @@ -182,7 +172,7 @@ private void serializeLabels(LogEvent event, StringBuilder builder) { } } } - event.getContextData().forEach(WRITE_KEY_VALUES_INTO, builder); + event.getContextData().forEach(WRITE_MDC, builder); } } @@ -332,8 +322,6 @@ public static class Builder extends AbstractStringLayout.Builder topLevelLabelsList = new ArrayList(); - if (topLevelLabels != null) { - for (String label : topLevelLabels.split(",")) { - topLevelLabelsList.add(label.trim()); - } - } - return new EcsLayout(getConfiguration(), serviceName, includeMarkers, additionalFields, topLevelLabelsList, includeOrigin, stackTraceAsArray); + return new EcsLayout(getConfiguration(), serviceName, includeMarkers, additionalFields, includeOrigin, stackTraceAsArray); } public boolean isStackTraceAsArray() { diff --git a/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java b/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java index dfd350d4..ed4e8390 100644 --- a/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java +++ b/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java @@ -51,10 +51,9 @@ void tearDown() throws Exception { } @Test - void globalLabels() throws Exception { + void testAdditionalFields() throws Exception { putMdc("trace.id", "foo"); - putMdc("top_level", "foo"); - putMdc("nested_under_labels", "foo"); + putMdc("foo", "bar"); debug("test"); assertThat(getLastLogLine().get("cluster.uuid").textValue()).isEqualTo("9fe9134b-20b0-465e-acf9-8cc09ac9053b"); assertThat(getLastLogLine().get("node.id").textValue()).isEqualTo("foo"); @@ -62,8 +61,7 @@ void globalLabels() throws Exception { assertThat(getLastLogLine().get("emptyPattern")).isNull(); assertThat(getLastLogLine().get("clazz").textValue()).startsWith(getClass().getPackageName()); assertThat(getLastLogLine().get("404")).isNull(); - assertThat(getLastLogLine().get("top_level").textValue()).isEqualTo("foo"); - assertThat(getLastLogLine().get("labels.nested_under_labels").textValue()).isEqualTo("foo"); + assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar"); } @Test diff --git a/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java b/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java index d837c9e5..72c0d1fc 100644 --- a/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java +++ b/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java @@ -70,7 +70,6 @@ void setUp() { .setIncludeMarkers(true) .setIncludeOrigin(true) .setStackTraceAsArray(true) - .setTopLevelLabels("top_level") .setAdditionalFields(new KeyValuePair[]{ new KeyValuePair("cluster.uuid", "9fe9134b-20b0-465e-acf9-8cc09ac9053b"), new KeyValuePair("node.id", "${node.id}"), diff --git a/log4j2-ecs-layout/src/test/resources/log4j2-test.xml b/log4j2-ecs-layout/src/test/resources/log4j2-test.xml index c7188613..de20628f 100644 --- a/log4j2-ecs-layout/src/test/resources/log4j2-test.xml +++ b/log4j2-ecs-layout/src/test/resources/log4j2-test.xml @@ -5,7 +5,7 @@ - + diff --git a/logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java b/logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java index 1d9098c0..1d689c81 100644 --- a/logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java +++ b/logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java @@ -35,10 +35,8 @@ import java.nio.charset.Charset; import java.util.ArrayList; -import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Set; public class EcsEncoder extends EncoderBase { @@ -47,7 +45,6 @@ public class EcsEncoder extends EncoderBase { private String serviceName; private boolean includeMarkers = false; private ThrowableProxyConverter throwableProxyConverter; - private Set topLevelLabels = new HashSet(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS); private boolean includeOrigin; private List additionalFields = new ArrayList(); @@ -74,7 +71,7 @@ public byte[] encode(ILoggingEvent event) { EcsJsonSerializer.serializeThreadName(builder, event.getThreadName()); EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName()); serializeAdditionalFields(builder); - EcsJsonSerializer.serializeLabels(builder, event.getMDCPropertyMap(), topLevelLabels); + EcsJsonSerializer.serializeMDC(builder, event.getMDCPropertyMap()); if (includeOrigin) { StackTraceElement[] callerData = event.getCallerData(); if (callerData != null && callerData.length > 0) { @@ -107,10 +104,6 @@ private void serializeAdditionalFields(StringBuilder builder) { } } - public void addTopLevelLabel(String topLevelLabel) { - this.topLevelLabels.add(topLevelLabel); - } - private void serializeMarkers(ILoggingEvent event, StringBuilder builder) { Marker marker = event.getMarker(); if (includeMarkers && marker != null) { From 65c4227d85dec77083e21985d070da67cb43f31f Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 30 Mar 2020 08:54:20 +0200 Subject: [PATCH 2/2] Fix compile errors --- .../test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java | 1 - .../src/test/java/co/elastic/logging/logback/EcsEncoderTest.java | 1 - 2 files changed, 2 deletions(-) diff --git a/log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java b/log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java index a58500be..a8cc7a83 100644 --- a/log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java +++ b/log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java @@ -52,7 +52,6 @@ void setUp() { ecsLayout.setServiceName("test"); ecsLayout.setStackTraceAsArray(true); ecsLayout.setIncludeOrigin(true); - ecsLayout.addTopLevelLabel("top_level"); appender.setLayout(ecsLayout); } diff --git a/logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java b/logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java index bf51adb6..55f1256d 100644 --- a/logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java +++ b/logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java @@ -46,7 +46,6 @@ void setUp() { ecsEncoder.setIncludeMarkers(true); ecsEncoder.setStackTraceAsArray(true); ecsEncoder.setIncludeOrigin(true); - ecsEncoder.addTopLevelLabel("top_level"); ecsEncoder.addAdditionalField(new EcsEncoder.Pair("foo", "bar")); ecsEncoder.addAdditionalField(new EcsEncoder.Pair("baz", "qux")); ecsEncoder.start();