diff --git a/README.md b/README.md index 74f2a7dd..f90e1822 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,16 @@ We recommend using this library to log into a JSON log file and let Filebeat sen |[`error.message`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html)|[`Throwable#getMessage()`](https://docs.oracle.com/javase/7/docs/api/java/lang/Throwable.html#getMessage())| |[`error.stack_trace`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html)|[`Throwable#getStackTrace()`](https://docs.oracle.com/javase/7/docs/api/java/lang/Throwable.html#getStackTrace())| |[`process.thread.name`](https://www.elastic.co/guide/en/ecs/current/ecs-process.html)|[`LogEvent#getThreadName()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getThreadName()) | -|[`labels`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html)|[`LogEvent#getContextMap()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextMap())| +|Each MDC entry is a top-level field 1|[`LogEvent#getContextMap()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextMap())| |[`tags`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html)|[`LogEvent#getContextStack()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextStack())| +1 It's recommended to use existing [ECS fields](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for MDC values. + +If there is no appropriate ECS field, +consider prefixing your fields with `labels.`, as in `labels.foo`, for simple key/value pairs. +For nested structures consider prefixing with `custom.` to make sure you won't get conflicts if ECS later adds the same fields but with a different mapping. + + ## Getting Started ### Step 1: Configure application logging diff --git a/ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java b/ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java index 3e026d10..c23f4a0e 100644 --- a/ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java +++ b/ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java @@ -26,14 +26,10 @@ import java.io.PrintWriter; import java.io.Writer; -import java.util.Arrays; -import java.util.List; import java.util.Map; -import java.util.Set; public class EcsJsonSerializer { - public static final List DEFAULT_TOP_LEVEL_LABELS = Arrays.asList("trace.id", "transaction.id", "span.id", "error.id", "service.name"); private static final TimestampSerializer TIMESTAMP_SERIALIZER = new TimestampSerializer(); private static final ThreadLocal messageStringBuilder = new ThreadLocal(); private static final String NEW_LINE = System.getProperty("line.separator"); @@ -144,14 +140,11 @@ public static void serializeOrigin(StringBuilder builder, String fileName, Strin builder.append("},"); } - public static void serializeLabels(StringBuilder builder, Map labels, Set topLevelLabels) { - if (!labels.isEmpty()) { - for (Map.Entry entry : labels.entrySet()) { + public static void serializeMDC(StringBuilder builder, Map properties) { + if (!properties.isEmpty()) { + for (Map.Entry entry : properties.entrySet()) { builder.append('\"'); String key = entry.getKey(); - if (!topLevelLabels.contains(key)) { - builder.append("labels."); - } JsonUtils.quoteAsString(key, builder); builder.append("\":\""); JsonUtils.quoteAsString(toNullSafeString(String.valueOf(entry.getValue())), builder); diff --git a/ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java b/ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java index 1b97f421..a1c1d32d 100644 --- a/ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java +++ b/ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java @@ -63,7 +63,7 @@ void testSimpleLog() throws Exception { void testThreadContext() throws Exception { putMdc("foo", "bar"); debug("test"); - assertThat(getLastLogLine().get("labels.foo").textValue()).isEqualTo("bar"); + assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar"); } @Test @@ -75,21 +75,15 @@ void testThreadContextStack() throws Exception { } @Test - void testTopLevelLabels() throws Exception { + void testMdc() throws Exception { putMdc("transaction.id", "0af7651916cd43dd8448eb211c80319c"); putMdc("span.id", "foo"); + putMdc("foo", "bar"); debug("test"); assertThat(getLastLogLine().get("labels.transaction.id")).isNull(); assertThat(getLastLogLine().get("transaction.id").textValue()).isEqualTo("0af7651916cd43dd8448eb211c80319c"); assertThat(getLastLogLine().get("span.id").textValue()).isEqualTo("foo"); - } - - @Test - void testCustomTopLevelLabels() throws Exception { - putMdc("top_level", "foo"); - debug("test"); - assertThat(getLastLogLine().get("labels.top_level")).isNull(); - assertThat(getLastLogLine().get("top_level").textValue()).isEqualTo("foo"); + assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar"); } @Test diff --git a/log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java b/log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java index 6b081593..aee21222 100644 --- a/log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java +++ b/log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java @@ -30,14 +30,10 @@ import org.apache.log4j.spi.LoggingEvent; import org.apache.log4j.spi.ThrowableInformation; -import java.util.HashSet; -import java.util.Set; - public class EcsLayout extends Layout { private boolean stackTraceAsArray = false; private String serviceName; - private Set topLevelLabels = new HashSet(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS); private boolean includeOrigin; private String eventDataset; @@ -51,7 +47,7 @@ public String format(LoggingEvent event) { EcsJsonSerializer.serializeEventDataset(builder, eventDataset); EcsJsonSerializer.serializeThreadName(builder, event.getThreadName()); EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName()); - EcsJsonSerializer.serializeLabels(builder, event.getProperties(), topLevelLabels); + EcsJsonSerializer.serializeMDC(builder, event.getProperties()); EcsJsonSerializer.serializeTag(builder, event.getNDC()); if (includeOrigin) { LocationInfo locationInformation = event.getLocationInformation(); @@ -102,10 +98,6 @@ public void setStackTraceAsArray(boolean stackTraceAsArray) { this.stackTraceAsArray = stackTraceAsArray; } - public void addTopLevelLabel(String topLevelLabel) { - this.topLevelLabels.add(topLevelLabel); - } - public void setEventDataset(String eventDataset) { this.eventDataset = eventDataset; } diff --git a/log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java b/log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java index 3d2a37f5..3f4174fd 100644 --- a/log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java +++ b/log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java @@ -52,7 +52,6 @@ void setUp() { ecsLayout.setServiceName("test"); ecsLayout.setStackTraceAsArray(true); ecsLayout.setIncludeOrigin(true); - ecsLayout.addTopLevelLabel("top_level"); ecsLayout.setEventDataset("testdataset.log"); ecsLayout.activateOptions(); appender.setLayout(ecsLayout); diff --git a/log4j2-ecs-layout/README.md b/log4j2-ecs-layout/README.md index 9114fd55..1afd8474 100644 --- a/log4j2-ecs-layout/README.md +++ b/log4j2-ecs-layout/README.md @@ -44,7 +44,6 @@ Instead of the usual ``, use `` |-----------------|-------|-------|-----------| |serviceName |String | |Sets the `service.name` field so you can filter your logs by a particular service | |eventDataset |String |`${serviceName}.log`|Sets the `event.dataset` field used by the machine learning job of the Logs app to look for anomalies in the log rate. | -|topLevelLabels |String |`trace.id, transaction.id, span.id, error.id, service.name`|Usually, MDC keys are nested under [`labels`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html). You can specify a comma-separated list of properties which should be on the top level. | |includeMarkers |boolean|`false`|Log [Markers](https://logging.apache.org/log4j/2.0/manual/markers.html) as [`tags`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html) | |stackTraceAsArray|boolean|`false`|Serializes the [`error.stack_trace`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html) as a JSON array where each element is in a new line to improve readability. Note that this requires a slightly more complex [Filebeat configuration](../README.md#when-stacktraceasarray-is-enabled).| |includeOrigin |boolean|`false`|If `true`, adds the [`log.origin.file.name`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html), [`log.origin.file.line`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) and [`log.origin.function`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) fields. Note that you also have to set `includeLocation="true"` on your loggers and appenders if you are using the async ones. | diff --git a/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java b/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java index 33bbd295..2550e40e 100644 --- a/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java +++ b/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java @@ -51,11 +51,7 @@ import org.apache.logging.log4j.util.TriConsumer; import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -65,13 +61,10 @@ public class EcsLayout extends AbstractStringLayout { public static final Charset UTF_8 = Charset.forName("UTF-8"); public static final String[] JSON_FORMAT = {"JSON"}; - private final TriConsumer WRITE_KEY_VALUES_INTO = new TriConsumer() { + private final TriConsumer WRITE_MDC = new TriConsumer() { @Override public void accept(final String key, final Object value, final StringBuilder stringBuilder) { stringBuilder.append('\"'); - if (!topLevelLabels.contains(key)) { - stringBuilder.append("labels."); - } JsonUtils.quoteAsString(key, stringBuilder); stringBuilder.append("\":\""); JsonUtils.quoteAsString(EcsJsonSerializer.toNullSafeString(String.valueOf(value)), stringBuilder); @@ -81,7 +74,6 @@ public void accept(final String key, final Object value, final StringBuilder str private final KeyValuePair[] additionalFields; private final PatternFormatter[][] fieldValuePatternFormatter; - private final Set topLevelLabels; private final boolean stackTraceAsArray; private final String serviceName; private final String eventDataset; @@ -90,13 +82,11 @@ public void accept(final String key, final Object value, final StringBuilder str private final ConcurrentMap, Boolean> supportsJson = new ConcurrentHashMap, Boolean>(); private final ObjectMessageJacksonSerializer objectMessageJacksonSerializer = ObjectMessageJacksonSerializer.Resolver.INSTANCE.resolve(); - private EcsLayout(Configuration config, String serviceName, String eventDataset, boolean includeMarkers, KeyValuePair[] additionalFields, Collection topLevelLabels, boolean includeOrigin, boolean stackTraceAsArray) { + private EcsLayout(Configuration config, String serviceName, String eventDataset, boolean includeMarkers, KeyValuePair[] additionalFields, boolean includeOrigin, boolean stackTraceAsArray) { super(config, UTF_8, null, null); this.serviceName = serviceName; this.eventDataset = eventDataset; this.includeMarkers = includeMarkers; - this.topLevelLabels = new HashSet(topLevelLabels); - this.topLevelLabels.addAll(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS); this.includeOrigin = includeOrigin; this.stackTraceAsArray = stackTraceAsArray; this.additionalFields = additionalFields; @@ -141,7 +131,7 @@ private StringBuilder toText(LogEvent event, StringBuilder builder, boolean gcFr EcsJsonSerializer.serializeEventDataset(builder, eventDataset); EcsJsonSerializer.serializeThreadName(builder, event.getThreadName()); EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName()); - serializeLabels(event, builder); + serializeAdditionalFieldsAndMDC(event, builder); serializeTags(event, builder); if (includeOrigin) { EcsJsonSerializer.serializeOrigin(builder, event.getSource()); @@ -151,7 +141,7 @@ private StringBuilder toText(LogEvent event, StringBuilder builder, boolean gcFr return builder; } - private void serializeLabels(LogEvent event, StringBuilder builder) { + private void serializeAdditionalFieldsAndMDC(LogEvent event, StringBuilder builder) { final int length = additionalFields.length; if (!event.getContextData().isEmpty() || length > 0) { if (length > 0) { @@ -185,7 +175,7 @@ private void serializeLabels(LogEvent event, StringBuilder builder) { } } } - event.getContextData().forEach(WRITE_KEY_VALUES_INTO, builder); + event.getContextData().forEach(WRITE_MDC, builder); } } @@ -337,8 +327,6 @@ public static class Builder extends AbstractStringLayout.Builder topLevelLabelsList = new ArrayList(); - if (topLevelLabels != null) { - for (String label : topLevelLabels.split(",")) { - topLevelLabelsList.add(label.trim()); - } - } - return new EcsLayout(getConfiguration(), serviceName, EcsJsonSerializer.computeEventDataset(eventDataset, serviceName), includeMarkers, additionalFields, topLevelLabelsList, includeOrigin, stackTraceAsArray); + return new EcsLayout(getConfiguration(), serviceName, EcsJsonSerializer.computeEventDataset(eventDataset, serviceName), includeMarkers, additionalFields, includeOrigin, stackTraceAsArray); } public boolean isStackTraceAsArray() { diff --git a/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java b/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java index dfd350d4..ed4e8390 100644 --- a/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java +++ b/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java @@ -51,10 +51,9 @@ void tearDown() throws Exception { } @Test - void globalLabels() throws Exception { + void testAdditionalFields() throws Exception { putMdc("trace.id", "foo"); - putMdc("top_level", "foo"); - putMdc("nested_under_labels", "foo"); + putMdc("foo", "bar"); debug("test"); assertThat(getLastLogLine().get("cluster.uuid").textValue()).isEqualTo("9fe9134b-20b0-465e-acf9-8cc09ac9053b"); assertThat(getLastLogLine().get("node.id").textValue()).isEqualTo("foo"); @@ -62,8 +61,7 @@ void globalLabels() throws Exception { assertThat(getLastLogLine().get("emptyPattern")).isNull(); assertThat(getLastLogLine().get("clazz").textValue()).startsWith(getClass().getPackageName()); assertThat(getLastLogLine().get("404")).isNull(); - assertThat(getLastLogLine().get("top_level").textValue()).isEqualTo("foo"); - assertThat(getLastLogLine().get("labels.nested_under_labels").textValue()).isEqualTo("foo"); + assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar"); } @Test diff --git a/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java b/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java index 355222af..3b55260e 100644 --- a/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java +++ b/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java @@ -70,7 +70,6 @@ void setUp() { .setIncludeMarkers(true) .setIncludeOrigin(true) .setStackTraceAsArray(true) - .setTopLevelLabels("top_level") .setEventDataset("testdataset.log") .setAdditionalFields(new KeyValuePair[]{ new KeyValuePair("cluster.uuid", "9fe9134b-20b0-465e-acf9-8cc09ac9053b"), diff --git a/log4j2-ecs-layout/src/test/resources/log4j2-test.xml b/log4j2-ecs-layout/src/test/resources/log4j2-test.xml index 6709816f..6cba9f3e 100644 --- a/log4j2-ecs-layout/src/test/resources/log4j2-test.xml +++ b/log4j2-ecs-layout/src/test/resources/log4j2-test.xml @@ -5,7 +5,7 @@ - diff --git a/logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java b/logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java index 49fc02a0..fd6bf854 100644 --- a/logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java +++ b/logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java @@ -35,10 +35,8 @@ import java.nio.charset.Charset; import java.util.ArrayList; -import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Set; public class EcsEncoder extends EncoderBase { @@ -48,7 +46,6 @@ public class EcsEncoder extends EncoderBase { private String eventDataset; private boolean includeMarkers = false; private ThrowableProxyConverter throwableProxyConverter; - private Set topLevelLabels = new HashSet(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS); private boolean includeOrigin; private List additionalFields = new ArrayList(); @@ -77,7 +74,7 @@ public byte[] encode(ILoggingEvent event) { EcsJsonSerializer.serializeThreadName(builder, event.getThreadName()); EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName()); serializeAdditionalFields(builder); - EcsJsonSerializer.serializeLabels(builder, event.getMDCPropertyMap(), topLevelLabels); + EcsJsonSerializer.serializeMDC(builder, event.getMDCPropertyMap()); if (includeOrigin) { StackTraceElement[] callerData = event.getCallerData(); if (callerData != null && callerData.length > 0) { @@ -110,10 +107,6 @@ private void serializeAdditionalFields(StringBuilder builder) { } } - public void addTopLevelLabel(String topLevelLabel) { - this.topLevelLabels.add(topLevelLabel); - } - private void serializeMarkers(ILoggingEvent event, StringBuilder builder) { Marker marker = event.getMarker(); if (includeMarkers && marker != null) { diff --git a/logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java b/logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java index 887a75dc..7794a5de 100644 --- a/logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java +++ b/logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java @@ -46,7 +46,6 @@ void setUp() { ecsEncoder.setIncludeMarkers(true); ecsEncoder.setStackTraceAsArray(true); ecsEncoder.setIncludeOrigin(true); - ecsEncoder.addTopLevelLabel("top_level"); ecsEncoder.addAdditionalField(new EcsEncoder.Pair("foo", "bar")); ecsEncoder.addAdditionalField(new EcsEncoder.Pair("baz", "qux")); ecsEncoder.setEventDataset("testdataset.log");