Add support for filtering stack traces in the log4j2 implementation

docs/tab-widgets/ecs-encoder.asciidoc

@@ -176,6 +176,15 @@ To include any custom field in the output, use following syntax:
 
 Custom fields are included in the order they are declared. The values support https://logging.apache.org/log4j/2.x/manual/lookups.html[lookups].
 
+To suppress lines containing `com.example.SomeClass` in an exceptions stack trace, use the following syntax:
+
+[source,xml]
+----
+  <EcsLayout>
+    <StackTraceFilter filter="com.example.SomeClass" />
+  </EcsLayout>
+----
+
 NOTE: The log4j2 `EcsLayout` does not allocate any memory (unless the log event contains an `Exception`) to reduce GC pressure.
 This is achieved by manually serializing JSON so that no intermediate JSON or map representation of a log event is needed.
 // end::log4j2[]

ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java

@@ -26,6 +26,7 @@
 
 import java.io.PrintWriter;
 import java.io.Writer;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
@@ -188,6 +189,10 @@ public static void serializeMDC(StringBuilder builder, Map<String, ?> properties
     }
 
     public static void serializeException(StringBuilder builder, Throwable thrown, boolean stackTraceAsArray) {
+        serializeException(builder, thrown, Collections.<String>emptyList(), stackTraceAsArray);
+    }
+
+    public static void serializeException(StringBuilder builder, Throwable thrown, List<String> stackTraceFilters, boolean stackTraceAsArray) {
         if (thrown != null) {
             builder.append("\"error.type\":\"");
             JsonUtils.quoteAsString(thrown.getClass().getName(), builder);
@@ -201,11 +206,11 @@ public static void serializeException(StringBuilder builder, Throwable thrown, b
             }
             if (stackTraceAsArray) {
                 builder.append("\"error.stack_trace\":[").append(NEW_LINE);
-                formatThrowableAsArray(builder, thrown);
+                formatThrowableAsArray(builder, thrown, stackTraceFilters);
                 builder.append("]");
             } else {
                 builder.append("\"error.stack_trace\":\"");
-                JsonUtils.quoteAsString(formatThrowable(thrown), builder);
+                JsonUtils.quoteAsString(formatThrowable(thrown, stackTraceFilters), builder);
                 builder.append("\"");
             }
         }
@@ -232,19 +237,77 @@ public static void serializeException(StringBuilder builder, String exceptionCla
         }
     }
 
-    private static CharSequence formatThrowable(final Throwable throwable) {
-        StringBuilder buffer = getMessageStringBuilder();
-        final PrintWriter pw = new PrintWriter(new StringBuilderWriter(buffer));
+    private static CharSequence formatThrowable(final Throwable throwable, final List<String> stackTraceFilters) {
+        final StringBuilder buffer = getMessageStringBuilder();
+        final PrintWriter pw = new PrintWriter(new StringBuilderWriter(buffer)) {
+
+            private int endOfLastLine;
+            private int numberOfSuppressedLines;
+
+            @Override
+            public void println() {
+                if (buffer.indexOf("\tat", endOfLastLine) == endOfLastLine) {
+                    for (String filter : stackTraceFilters) {
+                        if (buffer.indexOf(filter, endOfLastLine) != -1) {
+                            ++numberOfSuppressedLines;
+                            buffer.setLength(endOfLastLine);
+
+                            return;
+                        }
+                    }
+                }
+                if (numberOfSuppressedLines > 0) {
+                    int lengthOfNextLine = buffer.length() - endOfLastLine;
+                    if (numberOfSuppressedLines > 1) {
+                        buffer.insert(endOfLastLine, "\t... suppressed ")
+                                .insert(buffer.length() - lengthOfNextLine, numberOfSuppressedLines)
+                                .insert(buffer.length() - lengthOfNextLine, " lines");
+                    } else {
+                        buffer.insert(endOfLastLine, "\t...");
+                    }
+                    buffer.insert(buffer.length() - lengthOfNextLine, NEW_LINE);
+
+                    numberOfSuppressedLines = 0;
+                }
+                buffer.append(NEW_LINE);
+                endOfLastLine = buffer.length();
+            }
+        };
         throwable.printStackTrace(pw);
         pw.flush();
         return buffer;
     }
 
-    private static void formatThrowableAsArray(final StringBuilder jsonBuilder, final Throwable throwable) {
+    private static void formatThrowableAsArray(final StringBuilder jsonBuilder, final Throwable throwable, final List<String> stackTraceFilters) {
         final StringBuilder buffer = getMessageStringBuilder();
         final PrintWriter pw = new PrintWriter(new StringBuilderWriter(buffer), true) {
+
+            private int numberOfSuppressedLines;
+
             @Override
             public void println() {
+                if (buffer.indexOf("\tat") == 0) {
+                    for (String filter : stackTraceFilters) {
+                        if (buffer.indexOf(filter) != -1) {
+                            ++numberOfSuppressedLines;
+                            buffer.setLength(0);
+
+                            return;
+                        }
+                    }
+                }
+                if (numberOfSuppressedLines > 0) {
+                    jsonBuilder.append("\t\"");
+                    if (numberOfSuppressedLines > 1) {
+                        jsonBuilder.append("\\t... suppressed ").append(numberOfSuppressedLines).append(" lines");
+                    } else {
+                        jsonBuilder.append("\\t...");
+                    }
+                    jsonBuilder.append("\",");
+                    jsonBuilder.append(NEW_LINE);
+
+                    numberOfSuppressedLines = 0;
+                }
                 flush();
                 jsonBuilder.append("\t\"");
                 JsonUtils.quoteAsString(buffer, jsonBuilder);

ecs-logging-core/src/test/java/co/elastic/logging/EcsJsonSerializerTest.java

@@ -173,4 +173,94 @@ private void assertRemoveIfEndsWith(String builder, String ending, String expect
         EcsJsonSerializer.removeIfEndsWith(sb, ending);
         assertThat(sb.toString()).isEqualTo(expected);
     }
+
+    @Test
+    void testStackTraceSuppressOneLine() throws IOException {
+        try {
+            First.m();
+        } catch (Exception exception) {
+            StringBuilder jsonBuilder = new StringBuilder();
+            jsonBuilder.append('{');
+            List<String> stackTraceFilters = List.of("co.elastic.logging.EcsJsonSerializerTest$First");
+            EcsJsonSerializer.serializeException(jsonBuilder, exception, stackTraceFilters, false);
+            jsonBuilder.append('}');
+
+            String stackTrace = objectMapper.readTree(jsonBuilder.toString()).get(ERROR_STACK_TRACE).textValue();
+            assertThat(stackTrace).contains("\n\t...\n");
+            assertThat(stackTrace).doesNotContain("co.elastic.logging.EcsJsonSerializerTest$First");
+        }
+    }
+
+    @Test
+    void testWihStackTraceAsArrayStackTraceSuppressOneLine() throws IOException {
+        try {
+            First.m();
+        } catch (Exception exception) {
+            StringBuilder jsonBuilder = new StringBuilder();
+            jsonBuilder.append('{');
+            List<String> stackTraceFilters = List.of("co.elastic.logging.EcsJsonSerializerTest$First");
+            EcsJsonSerializer.serializeException(jsonBuilder, exception, stackTraceFilters, true);
+            jsonBuilder.append('}');
+
+            String stackTrace = objectMapper.readTree(jsonBuilder.toString()).get(ERROR_STACK_TRACE).toString();
+            assertThat(stackTrace).contains("\"\\t...\"");
+            assertThat(stackTrace).doesNotContain("co.elastic.logging.EcsJsonSerializerTest$First");
+        }
+    }
+
+    @Test
+    void testStackTraceSuppressMultipleLines() throws IOException {
+        try {
+            First.m();
+        } catch (Exception exception) {
+            StringBuilder jsonBuilder = new StringBuilder();
+            jsonBuilder.append('{');
+            List<String> stackTraceFilters = List.of("co.elastic.logging.EcsJsonSerializerTest$First", "co.elastic.logging.EcsJsonSerializerTest$Second");
+            EcsJsonSerializer.serializeException(jsonBuilder, exception, stackTraceFilters, false);
+            jsonBuilder.append('}');
+
+            String stackTrace = objectMapper.readTree(jsonBuilder.toString()).get(ERROR_STACK_TRACE).textValue();
+            assertThat(stackTrace).contains("\n\t... suppressed 2 lines\n");
+            assertThat(stackTrace).doesNotContain("co.elastic.logging.EcsJsonSerializerTest$First");
+            assertThat(stackTrace).doesNotContain("co.elastic.logging.EcsJsonSerializerTest$Second");
+        }
+    }
+
+    @Test
+    void testWithStackTraceAsArrayStackTraceSuppressMultipleLines() throws IOException {
+        try {
+            First.m();
+        } catch (Exception exception) {
+            StringBuilder jsonBuilder = new StringBuilder();
+            jsonBuilder.append('{');
+            List<String> stackTraceFilters = List.of("co.elastic.logging.EcsJsonSerializerTest$First", "co.elastic.logging.EcsJsonSerializerTest$Second");
+            EcsJsonSerializer.serializeException(jsonBuilder, exception, stackTraceFilters, true);
+            jsonBuilder.append('}');
+
+            String stackTrace = objectMapper.readTree(jsonBuilder.toString()).get(ERROR_STACK_TRACE).toString();
+            assertThat(stackTrace).contains("\"\\t... suppressed 2 lines\"");
+            assertThat(stackTrace).doesNotContain("co.elastic.logging.EcsJsonSerializerTest$First");
+            assertThat(stackTrace).doesNotContain("co.elastic.logging.EcsJsonSerializerTest$Second");
+        }
+    }
+
+
+    private static final class First {
+        public static void m() {
+            Second.m();
+        }
+    }
+
+
+    private static final class Second {
+        public static void m() {
+            Third.m();
+        }
+    }
+
+    private static final class Third {
+        public static void m() {
+            throw new RuntimeException();
+        }
+    }
 }

log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java

@@ -52,6 +52,7 @@
 import org.apache.logging.log4j.util.StringBuilderFormattable;
 
 import java.nio.charset.Charset;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
@@ -69,6 +70,7 @@ public class EcsLayout extends AbstractStringLayout {
 
     private final KeyValuePair[] additionalFields;
     private final PatternFormatter[][] fieldValuePatternFormatter;
+    private final List<String> stackTraceFilter;
     private final boolean stackTraceAsArray;
     private final String serviceName;
     private final String serviceNodeName;
@@ -78,13 +80,14 @@ public class EcsLayout extends AbstractStringLayout {
     private final ConcurrentMap<Class<? extends MultiformatMessage>, Boolean> supportsJson = new ConcurrentHashMap<Class<? extends MultiformatMessage>, Boolean>();
 
     private EcsLayout(Configuration config, String serviceName, String serviceNodeName, String eventDataset, boolean includeMarkers,
-                      KeyValuePair[] additionalFields, boolean includeOrigin, boolean stackTraceAsArray) {
+                      KeyValuePair[] additionalFields, boolean includeOrigin, List<String> stackTraceFilter, boolean stackTraceAsArray) {
         super(config, UTF_8, null, null);
         this.serviceName = serviceName;
         this.serviceNodeName = serviceNodeName;
         this.eventDataset = eventDataset;
         this.includeMarkers = includeMarkers;
         this.includeOrigin = includeOrigin;
+        this.stackTraceFilter = stackTraceFilter;
         this.stackTraceAsArray = stackTraceAsArray;
         this.additionalFields = additionalFields;
         fieldValuePatternFormatter = new PatternFormatter[additionalFields.length][];
@@ -140,7 +143,7 @@ private StringBuilder toText(LogEvent event, StringBuilder builder, boolean gcFr
         if (includeOrigin) {
             EcsJsonSerializer.serializeOrigin(builder, event.getSource());
         }
-        EcsJsonSerializer.serializeException(builder, event.getThrown(), stackTraceAsArray);
+        EcsJsonSerializer.serializeException(builder, event.getThrown(), stackTraceFilter, stackTraceAsArray);
         EcsJsonSerializer.serializeObjectEnd(builder);
         return builder;
     }
@@ -339,6 +342,8 @@ public static class Builder implements org.apache.logging.log4j.core.util.Builde
         private boolean includeMarkers = false;
         @PluginBuilderAttribute("stackTraceAsArray")
         private boolean stackTraceAsArray = false;
+        @PluginElement("StackTraceFilters")
+        private StackTraceFilter[] stackTraceFilters = new StackTraceFilter[]{};
         @PluginElement("AdditionalField")
         private KeyValuePair[] additionalFields = new KeyValuePair[]{};
         @PluginBuilderAttribute("includeOrigin")
@@ -380,6 +385,14 @@ public boolean isIncludeOrigin() {
             return includeOrigin;
         }
 
+        public boolean isStackTraceAsArray() {
+            return stackTraceAsArray;
+        }
+
+        public StackTraceFilter[] getStackTraceFilters() {
+            return stackTraceFilters;
+        }
+
         /**
          * Additional fields to set on each log event.
          *
@@ -415,19 +428,24 @@ public EcsLayout.Builder setIncludeOrigin(final boolean includeOrigin) {
             return this;
         }
 
+        public EcsLayout.Builder setStackTraceFilters(StackTraceFilter[] stackTraceFilters) {
+            this.stackTraceFilters = stackTraceFilters;
+            return this;
+        }
+
         public EcsLayout.Builder setStackTraceAsArray(boolean stackTraceAsArray) {
             this.stackTraceAsArray = stackTraceAsArray;
             return this;
         }
 
         @Override
         public EcsLayout build() {
+            List<String> stackTraceFilterPatterns = new ArrayList<String>(stackTraceFilters.length);
+            for (StackTraceFilter stackTraceFilter : stackTraceFilters) {
+                stackTraceFilterPatterns.add(stackTraceFilter.getFilter());
+            }
             return new EcsLayout(getConfiguration(), serviceName, serviceNodeName, EcsJsonSerializer.computeEventDataset(eventDataset, serviceName),
-                    includeMarkers, additionalFields, includeOrigin, stackTraceAsArray);
-        }
-
-        public boolean isStackTraceAsArray() {
-            return stackTraceAsArray;
+                    includeMarkers, additionalFields, includeOrigin, stackTraceFilterPatterns, stackTraceAsArray);
         }
     }
 }

log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/StackTraceFilter.java

@@ -0,0 +1,58 @@
+/*-
+ * #%L
+ * Java ECS logging
+ * %%
+ * Copyright (C) 2019 - 2022 Elastic and contributors
+ * %%
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * #L%
+ */
+package co.elastic.logging.log4j2;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.config.Node;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.status.StatusLogger;
+
+@Plugin(name = "StackTraceFilter", category = Node.CATEGORY)
+public final class StackTraceFilter {
+
+    private static final Logger LOGGER = StatusLogger.getLogger();
+
+    private final String filter;
+
+    public StackTraceFilter(String filter) {
+        this.filter = filter;
+    }
+
+    public String getFilter() {
+        return filter;
+    }
+
+    @PluginFactory
+    public static StackTraceFilter createStackTraceFilter(@PluginAttribute("filter") final String filter) {
+        if (filter == null) {
+            LOGGER.error("StackTraceFilter must contain a filter");
+            return null;
+        }
+
+        return new StackTraceFilter(filter);
+    }
+}