diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java index cf0664b9403..ad8f7762ba0 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java @@ -100,5 +100,7 @@ public final class GeneralConfig { public static final String APM_TRACING_ENABLED = "apm.tracing.enabled"; public static final String JDK_SOCKET_ENABLED = "jdk.socket.enabled"; + public static final String STACK_TRACE_LENGTH_LIMIT = "stack.trace.length.limit"; + private GeneralConfig() {} } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java index 78b364c1a17..a150f5246cf 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java @@ -27,8 +27,7 @@ import datadog.trace.bootstrap.instrumentation.api.ErrorPriorities; import datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities; import datadog.trace.bootstrap.instrumentation.api.Tags; -import java.io.PrintWriter; -import java.io.StringWriter; +import datadog.trace.core.util.StackTraces; import java.util.Collections; import java.util.List; import java.util.Map; @@ -350,9 +349,9 @@ public DDSpan addThrowable(Throwable error, byte errorPriority) { // or warming up - capturing the stack trace and keeping // the trace may exacerbate existing problems. setError(true, errorPriority); - final StringWriter errorString = new StringWriter(); - error.printStackTrace(new PrintWriter(errorString)); - setTag(DDTags.ERROR_STACK, errorString.toString()); + setTag( + DDTags.ERROR_STACK, + StackTraces.getStackTrace(error, Config.get().getStackTraceLengthLimit())); } setTag(DDTags.ERROR_MSG, message); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/util/StackTraces.java b/dd-trace-core/src/main/java/datadog/trace/core/util/StackTraces.java new file mode 100644 index 00000000000..3c6a00e599b --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/core/util/StackTraces.java @@ -0,0 +1,145 @@ +package datadog.trace.core.util; + +import java.io.BufferedReader; +import java.io.PrintWriter; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public final class StackTraces { + private StackTraces() {} + + public static String getStackTrace(Throwable t, int maxChars) { + StringWriter sw = new StringWriter(); + t.printStackTrace(new PrintWriter(sw)); + String trace = sw.toString(); + try { + return truncate(trace, maxChars); + } catch (Exception e) { + // If something goes wrong, return the original trace + return trace; + } + } + + static String truncate(String trace, int maxChars) { + if (trace.length() <= maxChars) { + return trace; + } + + trace = abbreviatePackageNames(trace); + if (trace.length() <= maxChars) { + return trace; + } + + trace = removeStackTraceMiddleForEachException(trace); + if (trace.length() <= maxChars) { + return trace; + } + + /* last-ditch centre cut to guarantee the limit */ + String cutMessage = "\t... trace centre-cut to " + maxChars + " chars ..."; + int retainedLength = maxChars - cutMessage.length() - 2; // 2 for the newlines + int half = retainedLength / 2; + return trace.substring(0, half) + + System.lineSeparator() + + cutMessage + + System.lineSeparator() + + trace.substring(trace.length() - (retainedLength - half)); + } + + private static final Pattern FRAME = Pattern.compile("^\\s*at ([^(]+)(\\(.*)$"); + + private static String abbreviatePackageNames(String trace) { + StringBuilder sb = new StringBuilder(trace.length()); + new BufferedReader(new StringReader(trace)) + .lines() + .forEach( + line -> { + Matcher m = FRAME.matcher(line); + if (m.matches()) { + sb.append("\tat ").append(abbreviatePackageName(m.group(1))).append(m.group(2)); + } else { + sb.append(line); + } + sb.append(System.lineSeparator()); + }); + return sb.toString(); + } + + /** + * Abbreviates only the package part of a fully qualified class name with member. For example, + * "com.myorg.MyClass.myMethod" to "c.m.MyClass.myMethod". If there is no package (e.g. + * "MyClass.myMethod"), returns the input unchanged. + */ + private static String abbreviatePackageName(String fqcnWithMember) { + int lastDot = fqcnWithMember.lastIndexOf('.'); + if (lastDot < 0) { + return fqcnWithMember; + } + int preClassDot = fqcnWithMember.lastIndexOf('.', lastDot - 1); + if (preClassDot < 0) { + return fqcnWithMember; + } + String packagePart = fqcnWithMember.substring(0, preClassDot); + String classAndAfter = fqcnWithMember.substring(preClassDot + 1); + + StringBuilder sb = new StringBuilder(fqcnWithMember.length()); + int segmentStart = 0; + for (int i = 0; i <= packagePart.length(); i++) { + if (i == packagePart.length() || packagePart.charAt(i) == '.') { + sb.append(packagePart.charAt(segmentStart)).append('.'); + segmentStart = i + 1; + } + } + sb.append(classAndAfter); + return sb.toString(); + } + + private static final int HEAD_LINES = 8, TAIL_LINES = 4; + + /** + * Removes lines from the middle of each exception stack trace, leaving {@link + * StackTraces#HEAD_LINES} lines at the beginning and {@link StackTraces#TAIL_LINES} lines at the + * end + */ + private static String removeStackTraceMiddleForEachException(String trace) { + List lines = + new BufferedReader(new StringReader(trace)).lines().collect(Collectors.toList()); + List out = new ArrayList<>(lines.size()); + int i = 0; + while (i < lines.size()) { + out.add(lines.get(i++)); // "Exception ..." / "Caused by: ..." + int start = i; + while (i < lines.size() && lines.get(i).startsWith("\tat")) { + i++; + } + + int total = i - start; + + int keepHead = Math.min(HEAD_LINES, total); + for (int j = 0; j < keepHead; j++) { + out.add(lines.get(start + j)); + } + + int keepTail = Math.min(TAIL_LINES, total - keepHead); + int skipped = total - keepHead - keepTail; + if (skipped > 0) { + out.add("\t... " + skipped + " trimmed ..."); + } + + for (int j = total - keepTail; j < total; j++) { + out.add(lines.get(start + j)); + } + + // "... n more" continuation markers + if (i < lines.size() && lines.get(i).startsWith("\t...")) { + out.add(lines.get(i++)); + } + } + return String.join(System.lineSeparator(), out) + System.lineSeparator(); + } +} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/core/util/StackTracesTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/core/util/StackTracesTest.groovy new file mode 100644 index 00000000000..646956708ff --- /dev/null +++ b/dd-trace-core/src/test/groovy/datadog/trace/core/util/StackTracesTest.groovy @@ -0,0 +1,188 @@ +package datadog.trace.core.util + + +import spock.lang.Specification + +class StackTracesTest extends Specification { + + def "test stack trace truncation: #limit"() { + given: + def trace = """ +Exception in thread "main" com.example.app.MainException: Unexpected application failure + at com.example.app.Application\$Runner.run(Application.java:102) + at com.example.app.Application.lambda\$start\$0(Application.java:75) + at java.base/java.util.Optional.ifPresent(Optional.java:178) + at com.example.app.Application.start(Application.java:74) + at com.example.app.Main.main(Main.java:21) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.base/java.lang.reflect.Method.invoke(Method.java:566) + at com.example.launcher.Bootstrap.run(Bootstrap.java:39) + at com.example.launcher.Bootstrap.main(Bootstrap.java:25) + at com.example.internal.\$Proxy1.start(Unknown Source) + at com.example.internal.Initializer\$1.run(Initializer.java:47) + at com.example.internal.Initializer.lambda\$init\$0(Initializer.java:38) + at java.base/java.util.concurrent.Executors\$RunnableAdapter.call(Executors.java:515) + at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) + at java.base/java.util.concurrent.ThreadPoolExecutor\$Worker.run(ThreadPoolExecutor.java:628) + at java.base/java.lang.Thread.run(Thread.java:834) + at com.example.synthetic.Helper.access\$100(Helper.java:14) +Caused by: com.example.db.DatabaseException: Failed to load user data + at com.example.db.UserDao.findUser(UserDao.java:88) + at com.example.db.UserDao.lambda\$cacheLookup\$1(UserDao.java:64) + at com.example.cache.Cache\$Entry.computeIfAbsent(Cache.java:111) + at com.example.cache.Cache.get(Cache.java:65) + at com.example.service.UserService.loadUser(UserService.java:42) + at com.example.service.UserService.lambda\$loadUserAsync\$0(UserService.java:36) + at com.example.util.SafeRunner.run(SafeRunner.java:27) + at java.base/java.util.concurrent.Executors\$RunnableAdapter.call(Executors.java:515) + at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) + at java.base/java.util.concurrent.ThreadPoolExecutor\$Worker.run(ThreadPoolExecutor.java:628) + at java.base/java.lang.Thread.run(Thread.java:834) + at com.example.synthetic.UserDao\$1.run(UserDao.java:94) + at com.example.synthetic.UserDao\$1.run(UserDao.java:94) + at com.example.db.ConnectionManager.getConnection(ConnectionManager.java:55) +Suppressed: java.io.IOException: Resource cleanup failed + at com.example.util.ResourceManager.close(ResourceManager.java:23) + at com.example.service.UserService.lambda\$loadUserAsync\$0(UserService.java:38) + ... 3 more +Caused by: java.nio.file.AccessDeniedException: /data/user/config.json + at java.base/sun.nio.fs.UnixException.translateToIOException(UnixException.java:90) + at java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) + at java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) + at java.base/sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:219) + at java.base/java.nio.file.Files.newByteChannel(Files.java:375) + at java.base/java.nio.file.Files.newInputStream(Files.java:489) + at com.example.util.FileUtils.readFile(FileUtils.java:22) + at com.example.util.ResourceManager.close(ResourceManager.java:21) + ... 3 more +""" + + expect: + StackTraces.truncate(trace, limit) == expected + + where: + limit | expected + 1000 | """ +Exception in thread "main" com.example.app.MainException: Unexpected application failure + at c.e.a.Application\$Runner.run(Application.java:102) + at c.e.a.Application.lambda\$start\$0(Application.java:75) + at j.b.u.Optional.ifPresent(Optional.java:178) + at c.e.a.Application.start(Application.java:74) + at c.e.a.Main.main(Main.java:21) + at s.r.NativeMethodAccessorImpl.invoke0(Native Method) + at s.r.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at s.r.Delegat + ... trace centre-cut to 1000 chars ... +ToIOException(UnixException.java:90) + at j.b.n.f.UnixException.rethrowAsIOException(UnixException.java:111) + at j.b.n.f.UnixException.rethrowAsIOException(UnixException.java:116) + at j.b.n.f.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:219) + at j.b.n.f.Files.newByteChannel(Files.java:375) + at j.b.n.f.Files.newInputStream(Files.java:489) + at c.e.u.FileUtils.readFile(FileUtils.java:22) + at c.e.u.ResourceManager.close(ResourceManager.java:21) + ... 3 more +""" + 2500 | """ +Exception in thread "main" com.example.app.MainException: Unexpected application failure + at c.e.a.Application\$Runner.run(Application.java:102) + at c.e.a.Application.lambda\$start\$0(Application.java:75) + at j.b.u.Optional.ifPresent(Optional.java:178) + at c.e.a.Application.start(Application.java:74) + at c.e.a.Main.main(Main.java:21) + at s.r.NativeMethodAccessorImpl.invoke0(Native Method) + at s.r.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at s.r.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + ... 8 trimmed ... + at j.b.u.c.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) + at j.b.u.c.ThreadPoolExecutor\$Worker.run(ThreadPoolExecutor.java:628) + at j.b.l.Thread.run(Thread.java:834) + at c.e.s.Helper.access\$100(Helper.java:14) +Caused by: com.example.db.DatabaseException: Failed to load user data + at c.e.d.UserDao.findUser(UserDao.java:88) + at c.e.d.UserDao.lambda\$cacheLookup\$1(UserDao.java:64) + at c.e.c.Cache\$Entry.computeIfAbsent(Cache.java:111) + at c.e.c.Cache.get(Cache.java:65) + at c.e.s.UserService.loadUser(UserService.java:42) + at c.e.s.UserService.lambda\$loadUserAsync\$0(UserService.java:36) + at c.e.u.SafeRunner.run(SafeRunner.java:27) + at j.b.u.c.Executors\$RunnableAdapter.call(Executors.java:515) + ... 3 trimmed ... + at j.b.l.Thread.run(Thread.java:834) + at c.e.s.UserDao\$1.run(UserDao.java:94) + at c.e.s.UserDao\$1.run(UserDao.java:94) + at c.e.d.ConnectionManager.getConnection(ConnectionManager.java:55) +Suppressed: java.io.IOException: Resource cleanup failed + at c.e.u.ResourceManager.close(ResourceManager.java:23) + at c.e.s.UserService.lambda\$loadUserAsync\$0(UserService.java:38) + ... 3 more +Caused by: java.nio.file.AccessDeniedException: /data/user/config.json + at j.b.n.f.UnixException.translateToIOException(UnixException.java:90) + at j.b.n.f.UnixException.rethrowAsIOException(UnixException.java:111) + at j.b.n.f.UnixException.rethrowAsIOException(UnixException.java:116) + at j.b.n.f.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:219) + at j.b.n.f.Files.newByteChannel(Files.java:375) + at j.b.n.f.Files.newInputStream(Files.java:489) + at c.e.u.FileUtils.readFile(FileUtils.java:22) + at c.e.u.ResourceManager.close(ResourceManager.java:21) + ... 3 more +""" + 3000 | """ +Exception in thread "main" com.example.app.MainException: Unexpected application failure + at c.e.a.Application\$Runner.run(Application.java:102) + at c.e.a.Application.lambda\$start\$0(Application.java:75) + at j.b.u.Optional.ifPresent(Optional.java:178) + at c.e.a.Application.start(Application.java:74) + at c.e.a.Main.main(Main.java:21) + at s.r.NativeMethodAccessorImpl.invoke0(Native Method) + at s.r.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at s.r.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at j.b.l.r.Method.invoke(Method.java:566) + at c.e.l.Bootstrap.run(Bootstrap.java:39) + at c.e.l.Bootstrap.main(Bootstrap.java:25) + at c.e.i.\$Proxy1.start(Unknown Source) + at c.e.i.Initializer\$1.run(Initializer.java:47) + at c.e.i.Initializer.lambda\$init\$0(Initializer.java:38) + at j.b.u.c.Executors\$RunnableAdapter.call(Executors.java:515) + at j.b.u.c.FutureTask.run(FutureTask.java:264) + at j.b.u.c.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) + at j.b.u.c.ThreadPoolExecutor\$Worker.run(ThreadPoolExecutor.java:628) + at j.b.l.Thread.run(Thread.java:834) + at c.e.s.Helper.access\$100(Helper.java:14) +Caused by: com.example.db.DatabaseException: Failed to load user data + at c.e.d.UserDao.findUser(UserDao.java:88) + at c.e.d.UserDao.lambda\$cacheLookup\$1(UserDao.java:64) + at c.e.c.Cache\$Entry.computeIfAbsent(Cache.java:111) + at c.e.c.Cache.get(Cache.java:65) + at c.e.s.UserService.loadUser(UserService.java:42) + at c.e.s.UserService.lambda\$loadUserAsync\$0(UserService.java:36) + at c.e.u.SafeRunner.run(SafeRunner.java:27) + at j.b.u.c.Executors\$RunnableAdapter.call(Executors.java:515) + at j.b.u.c.FutureTask.run(FutureTask.java:264) + at j.b.u.c.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) + at j.b.u.c.ThreadPoolExecutor\$Worker.run(ThreadPoolExecutor.java:628) + at j.b.l.Thread.run(Thread.java:834) + at c.e.s.UserDao\$1.run(UserDao.java:94) + at c.e.s.UserDao\$1.run(UserDao.java:94) + at c.e.d.ConnectionManager.getConnection(ConnectionManager.java:55) +Suppressed: java.io.IOException: Resource cleanup failed + at c.e.u.ResourceManager.close(ResourceManager.java:23) + at c.e.s.UserService.lambda\$loadUserAsync\$0(UserService.java:38) + ... 3 more +Caused by: java.nio.file.AccessDeniedException: /data/user/config.json + at j.b.n.f.UnixException.translateToIOException(UnixException.java:90) + at j.b.n.f.UnixException.rethrowAsIOException(UnixException.java:111) + at j.b.n.f.UnixException.rethrowAsIOException(UnixException.java:116) + at j.b.n.f.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:219) + at j.b.n.f.Files.newByteChannel(Files.java:375) + at j.b.n.f.Files.newInputStream(Files.java:489) + at c.e.u.FileUtils.readFile(FileUtils.java:22) + at c.e.u.ResourceManager.close(ResourceManager.java:21) + ... 3 more +""" + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index b3e4c533cd9..f9b55d7f3f7 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -579,6 +579,8 @@ public static String getHostName() { private final boolean jdkSocketEnabled; + private final int stackTraceLengthLimit; + // Read order: System Properties -> Env Variables, [-> properties file], [-> default value] private Config() { this(ConfigProvider.createDefault()); @@ -2036,6 +2038,13 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) this.jdkSocketEnabled = configProvider.getBoolean(JDK_SOCKET_ENABLED, true); + int defaultStackTraceLengthLimit = + instrumenterConfig.isCiVisibilityEnabled() + ? 5000 // EVP limit + : Integer.MAX_VALUE; // no effective limit (old behavior) + this.stackTraceLengthLimit = + configProvider.getInteger(STACK_TRACE_LENGTH_LIMIT, defaultStackTraceLengthLimit); + log.debug("New instance: {}", this); } @@ -3658,6 +3667,10 @@ public boolean isJdkSocketEnabled() { return jdkSocketEnabled; } + public int getStackTraceLengthLimit() { + return stackTraceLengthLimit; + } + /** @return A map of tags to be applied only to the local application root span. */ public Map getLocalRootSpanTags() { final Map runtimeTags = getRuntimeTags();