Flush traces when spark application is finished (#6670)

paul-laffon-dd · web-flow · commit 0c0f9cb79ba3 · 2024-02-13T10:05:35.000+01:00
Flush spans when the spark application finishes. Default timeout for the remoteWriter is 1 second so it will not block the main thread for more than 1 second

Motivation: Customer losing their spans because the JVM was shut down too quickly
diff --git a/dd-java-agent/instrumentation/spark/src/main/java/datadog/trace/instrumentation/spark/AbstractDatadogSparkListener.java b/dd-java-agent/instrumentation/spark/src/main/java/datadog/trace/instrumentation/spark/AbstractDatadogSparkListener.java
@@ -213,6 +213,10 @@ public synchronized void finishApplication(
         "spark.available_executor_time", computeCurrentAvailableExecutorTime(time));
 
     applicationSpan.finish(time * 1000);
+
+    // write traces synchronously:
+    // as soon as the application finishes, the JVM starts to shut down
+    tracer.flush();
   }
 
   private AgentSpan getOrCreateStreamingBatchSpan(

Original file line number	Diff line number	Diff line change
`@@ -213,6 +213,10 @@ public synchronized void finishApplication(`
`213`	`213`	`"spark.available_executor_time", computeCurrentAvailableExecutorTime(time));`
`214`	`214`
`215`	`215`	`applicationSpan.finish(time * 1000);`
	`216`	`+`
	`217`	`+ // write traces synchronously:`
	`218`	`+ // as soon as the application finishes, the JVM starts to shut down`
	`219`	`+ tracer.flush();`
`216`	`220`	`}`
`217`	`221`
`218`	`222`	`private AgentSpan getOrCreateStreamingBatchSpan(`