Potential memory leak and race with the JVMTI wallclock sampler

.github/scripts/test_alpine_aarch64.sh

@@ -29,6 +29,6 @@ JAVA_VERSION=$("${JAVA_TEST_HOME}/bin/java" -version 2>&1 | awk -F '"' '/version
 }')
 export JAVA_VERSION
 
-apk update && apk add curl moreutils wget hexdump linux-headers bash make g++ clang git cppcheck jq cmake gtest-dev gmock tar >/dev/null
+apk update && apk add curl moreutils wget hexdump linux-headers bash make g++ clang git cppcheck jq cmake gtest-dev gmock tar binutils >/dev/null
 
 ./gradlew -PCI -PkeepJFRs :ddprof-test:test${CONFIG} --no-daemon --parallel --build-cache --no-watch-fs

.github/workflows/test_workflow.yml

@@ -72,7 +72,7 @@ jobs:
         if: steps.set_enabled.outputs.enabled == 'true'
         run: |
           sudo apt-get update
-          sudo apt-get install -y curl zip unzip libgtest-dev libgmock-dev
+          sudo apt-get install -y curl zip unzip libgtest-dev libgmock-dev binutils
           if [[ ${{ matrix.java_version }} =~ "-zing" ]]; then
             sudo apt-get install -y g++-9 gcc-9
             sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 --slave /usr/bin/g++ g++ /usr/bin/g++-9
@@ -135,7 +135,7 @@ jobs:
     steps:
       - name: Setup OS
         run: |
-          apk update && apk add curl moreutils wget hexdump linux-headers bash make g++ clang git cppcheck jq cmake gtest-dev gmock tar >/dev/null
+          apk update && apk add curl moreutils wget hexdump linux-headers bash make g++ clang git cppcheck jq cmake gtest-dev gmock tar binutils >/dev/null
       - uses: actions/checkout@v3
       - name: Cache Gradle Wrapper Binaries
         uses: actions/cache@v4
@@ -286,7 +286,7 @@ jobs:
           sudo apt update -y
           sudo apt remove -y g++
           sudo apt autoremove -y
-          sudo apt install -y curl zip unzip clang make build-essential
+          sudo apt install -y curl zip unzip clang make build-essential binutils
           if [[ ${{ matrix.java_version }} =~ "-zing" ]]; then
             sudo apt -y install g++-9 gcc-9
             sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 --slave /usr/bin/g++ g++ /usr/bin/g++-9

.github/workflows/update_assets.yml

@@ -35,7 +35,7 @@ jobs:
             TAG="$GITHUB_REF_NAME"
           fi
           VERSION=$(echo "${TAG}" | sed -e 's/v_//g')
-          ASSET_URL="https://oss.sonatype.org/service/local/repositories/releases/content/com/datadoghq/ddprof/${VERSION}/ddprof-${VERSION}.jar"
+          ASSET_URL="https://repo1.maven.org/maven2/com/datadoghq/ddprof/${VERSION}/ddprof-${VERSION}.jar"
           RESULT=1
           while [ $RESULT -ne 0 ]; do
             wget -q $ASSET_URL

README.md

@@ -64,6 +64,39 @@ The project includes both Java and C++ unit tests. You can run them using:
 ### Cross-JDK Testing
 `JAVA_TEST_HOME=<path to test JDK> ./gradlew testDebug`
 
+## Release Builds and Debug Information
+
+### Split Debug Information
+Release builds automatically generate split debug information to optimize deployment size while preserving debugging capabilities:
+
+- **Stripped libraries** (~1.2MB): Production-ready binaries with symbols removed for deployment
+- **Debug symbol files** (~6.1MB): Separate `.debug` files containing full debugging information
+- **Debug links**: Stripped libraries include `.gnu_debuglink` sections pointing to debug files
+
+### Build Artifacts Structure
+```
+ddprof-lib/build/
+├── lib/main/release/linux/x64/
+│   ├── libjavaProfiler.so              # Original library with debug symbols
+│   ├── stripped/
+│   │   └── libjavaProfiler.so          # Stripped library (83% smaller)
+│   └── debug/
+│       └── libjavaProfiler.so.debug    # Debug symbols only
+├── native/release/
+│   └── META-INF/native-libs/linux-x64/
+│       └── libjavaProfiler.so          # Final stripped library (deployed)
+└── native/release-debug/
+    └── META-INF/native-libs/linux-x64/
+        └── libjavaProfiler.so.debug    # Debug symbols package
+```
+
+### Build Options
+- **Skip debug extraction**: `./gradlew buildRelease -Pskip-debug-extraction=true`
+- **Debug extraction requires**: `objcopy` (Linux) or `dsymutil` (macOS)
+  - Ubuntu/Debian: `sudo apt-get install binutils`
+  - Alpine: `apk add binutils`
+  - macOS: Included with Xcode command line tools
+
 ## Development
 
 ### Code Quality

build.gradle

@@ -14,7 +14,7 @@ plugins {
   id "com.diffplug.spotless" version "6.11.0"
 }
 
-version = "1.28.0"
+version = "1.29.0"
 
 apply plugin: "com.dipien.semantic-version"
 version = project.findProperty("ddprof_version") ?: version
@@ -39,7 +39,8 @@ repositories {
     mavenContent {
       snapshotsOnly()
     }
-    url 'https://oss.sonatype.org/content/repositories/snapshots/'
+    // see https://central.sonatype.org/publish/publish-portal-snapshots/#consuming-via-gradle
+    url 'https://central.sonatype.com/repository/maven-snapshots/'
   }
 }
 
@@ -75,7 +76,14 @@ nexusPublishing {
         password = "admin123"
       }
     } else {
+      // see https://github.com/gradle-nexus/publish-plugin#publishing-to-maven-central-via-sonatype-central
+      // For official documentation:
+      // staging repo publishing https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/#configuration
+      // snapshot publishing https://central.sonatype.org/publish/publish-portal-snapshots/#publishing-via-other-methods
       sonatype {
+        nexusUrl.set(uri("https://ossrh-staging-api.central.sonatype.com/service/local/"))
+        snapshotRepositoryUrl.set(uri("https://central.sonatype.com/repository/maven-snapshots/"))
+
         username = System.getenv("SONATYPE_USERNAME")
         password = System.getenv("SONATYPE_PASSWORD")
       }

ddprof-lib/build.gradle

@@ -8,6 +8,166 @@ plugins {
   id 'de.undercouch.download' version '4.1.1'
 }
 
+// Helper function to check if objcopy is available
+def checkObjcopyAvailable() {
+  try {
+    def process = ['objcopy', '--version'].execute()
+    process.waitFor()
+    return process.exitValue() == 0
+  } catch (Exception e) {
+    return false
+  }
+}
+
+// Helper function to check if dsymutil is available (for macOS)
+def checkDsymutilAvailable() {
+  try {
+    def process = ['dsymutil', '--version'].execute()
+    process.waitFor()
+    return process.exitValue() == 0
+  } catch (Exception e) {
+    return false
+  }
+}
+
+// Helper function to check if debug extraction should be skipped
+def shouldSkipDebugExtraction() {
+  // Skip if explicitly disabled
+  if (project.hasProperty('skip-debug-extraction')) {
+    return true
+  }
+
+  // Skip if required tools are not available
+  if (os().isLinux() && !checkObjcopyAvailable()) {
+    return true
+  }
+
+  if (os().isMacOsX() && !checkDsymutilAvailable()) {
+    return true
+  }
+
+  return false
+}
+
+// Helper function to get debug file path for a given config
+def getDebugFilePath(config) {
+  def extension = os().isLinux() ? 'so' : 'dylib'
+  return file("$buildDir/lib/main/${config.name}/${osIdentifier()}/${archIdentifier()}/debug/libjavaProfiler.${extension}.debug")
+}
+
+// Helper function to get stripped file path for a given config
+def getStrippedFilePath(config) {
+  def extension = os().isLinux() ? 'so' : 'dylib'
+  return file("$buildDir/lib/main/${config.name}/${osIdentifier()}/${archIdentifier()}/stripped/libjavaProfiler.${extension}")
+}
+
+// Helper function to create error message for missing tools
+def getMissingToolErrorMessage(toolName, installInstructions) {
+  return """
+    |${toolName} is not available but is required for split debug information.
+    |
+    |To fix this issue:
+    |${installInstructions}
+    |
+    |If you want to build without split debug info, set -Pskip-debug-extraction=true
+  """.stripMargin()
+}
+
+// Helper function to create debug extraction task
+def createDebugExtractionTask(config, linkTask) {
+  return tasks.register('extractDebugLibRelease', Exec) {
+    onlyIf {
+      !shouldSkipDebugExtraction()
+    }
+    dependsOn linkTask
+    description = 'Extract debug symbols from release library'
+    workingDir project.buildDir
+
+    doFirst {
+      def sourceFile = linkTask.get().linkedFile.get().asFile
+      def debugFile = getDebugFilePath(config)
+
+      // Ensure debug directory exists
+      debugFile.parentFile.mkdirs()
+
+      // Set the command line based on platform
+      if (os().isLinux()) {
+        commandLine = ['objcopy', '--only-keep-debug', sourceFile.absolutePath, debugFile.absolutePath]
+      } else {
+        // For macOS, we'll use dsymutil instead
+        commandLine = ['dsymutil', sourceFile.absolutePath, '-o', debugFile.absolutePath.replace('.debug', '.dSYM')]
+      }
+    }
+  }
+}
+
+// Helper function to create debug link task (Linux only)
+def createDebugLinkTask(config, linkTask, extractDebugTask) {
+  return tasks.register('addDebugLinkLibRelease', Exec) {
+    onlyIf {
+      os().isLinux() && !shouldSkipDebugExtraction()
+    }
+    dependsOn extractDebugTask
+    description = 'Add debug link to the original library'
+
+    doFirst {
+      def sourceFile = linkTask.get().linkedFile.get().asFile
+      def debugFile = getDebugFilePath(config)
+
+      commandLine = ['objcopy', '--add-gnu-debuglink=' + debugFile.absolutePath, sourceFile.absolutePath]
+    }
+  }
+}
+
+// Helper function to create debug file copy task
+def createDebugCopyTask(config, extractDebugTask) {
+  return tasks.register('copyReleaseDebugFiles', Copy) {
+    onlyIf {
+      !shouldSkipDebugExtraction()
+    }
+    dependsOn extractDebugTask
+    from file("$buildDir/lib/main/${config.name}/${osIdentifier()}/${archIdentifier()}/debug")
+    into file(libraryTargetPath(config.name + '-debug'))
+    include '**/*.debug'
+    include '**/*.dSYM/**'
+  }
+}
+
+// Main function to setup debug extraction for release builds
+def setupDebugExtraction(config, linkTask) {
+  if (config.name == 'release' && config.active && !project.hasProperty('skip-native')) {
+    // Create all debug-related tasks
+    def extractDebugTask = createDebugExtractionTask(config, linkTask)
+    def addDebugLinkTask = createDebugLinkTask(config, linkTask, extractDebugTask)
+
+    // Create the strip task and configure it properly
+    def stripTask = tasks.register('stripLibRelease', StripSymbols) {
+      // No onlyIf needed here - setupDebugExtraction already handles the main conditions
+      dependsOn addDebugLinkTask
+    }
+
+    // Configure the strip task after registration
+    stripTask.configure {
+      targetPlatform = linkTask.get().targetPlatform
+      toolChain = linkTask.get().toolChain
+      binaryFile = linkTask.get().linkedFile.get().asFile
+      outputFile = getStrippedFilePath(config)
+    }
+
+    def copyDebugTask = createDebugCopyTask(config, extractDebugTask)
+
+    // Wire up the copy task to use stripped binaries
+    def copyTask = tasks.findByName("copyReleaseLibs")
+    if (copyTask != null) {
+      copyTask.dependsOn stripTask
+      copyTask.inputs.files stripTask.get().outputs.files
+
+      // Create an extra folder for the debug symbols
+      copyTask.dependsOn copyDebugTask
+    }
+  }
+}
+
 def libraryName = "ddprof"
 
 description = "Datadog Java Profiler Library"
@@ -366,23 +526,7 @@ tasks.whenTaskAdded { task ->
             outputs.file linkedFile
           }
           if (config.name == 'release') {
-            def stripTask = tasks.register('stripLibRelease', StripSymbols) {
-              onlyIf {
-                config.active
-              }
-              dependsOn linkTask
-              targetPlatform = tasks.linkLibRelease.targetPlatform
-              toolChain = tasks.linkLibRelease.toolChain
-              binaryFile = tasks.linkLibRelease.linkedFile.get()
-              outputFile = file("$buildDir/lib/main/${config.name}/${osIdentifier()}/${archIdentifier()}/stripped/libjavaProfiler.${os().isLinux() ? 'so' : 'dylib'}")
-              inputs.file binaryFile
-              outputs.file outputFile
-            }
-            def copyTask = tasks.findByName("copyReleaseLibs")
-            if (copyTask != null) {
-              copyTask.dependsOn stripTask
-              copyTask.inputs.files stripTask.get().outputs.files
-            }
+            setupDebugExtraction(config, linkTask)
           }
         }
       }

ddprof-lib/src/main/cpp/profiler.cpp

@@ -972,6 +972,7 @@ void Profiler::updateJavaThreadNames() {
   JNIEnv *jni = VM::jni();
   for (int i = 0; i < thread_count; i++) {
     updateThreadName(jvmti, jni, thread_objects[i]);
+    jni->DeleteLocalRef(thread_objects[i]);
   }
 
   jvmti->Deallocate((unsigned char *)thread_objects);

ddprof-lib/src/main/cpp/safeAccess.cpp

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2025 Datadog, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include "safeAccess.h"
+
+SafeAccess::SafeFetch32 SafeAccess::_safeFetch32Func = nullptr;
+
+void SafeAccess::initSafeFetch(CodeCache* libjvm) {
+  // Hotspot JVM's safefetch implementation appears better, e.g. it actually returns errorValue,
+  // when the address is invalid. let's use it whenever possible.
+  // When the methods are not available, fallback to SafeAccess::load32() implementation.
+  _safeFetch32Func = (SafeFetch32)libjvm->findSymbol("SafeFetch32_impl");
+  if (_safeFetch32Func == nullptr && !WX_MEMORY) {
+    // jdk11 stub implementation other than Macosx/aarch64
+    void** entry = (void**)libjvm->findSymbol("_ZN12StubRoutines18_safefetch32_entryE");
+    if (entry != nullptr && *entry != nullptr) {
+      _safeFetch32Func = (SafeFetch32)*entry;
+    }
+  }
+  // Fallback
+  if (_safeFetch32Func == nullptr) {
+    _safeFetch32Func = (SafeFetch32)load32;
+  }
+}

ddprof-lib/src/main/cpp/safeAccess.h

@@ -18,6 +18,8 @@
 #define _SAFEACCESS_H
 
 #include "arch_dd.h"
+#include "codeCache.h"
+#include <cassert>
 #include <stdint.h>
 
 #ifdef __clang__
@@ -28,7 +30,18 @@
 #define NOADDRSANITIZE __attribute__((no_sanitize("address")))
 
 class SafeAccess {
+private:
+   typedef int (*SafeFetch32)(int* ptr, int errorValue);
+   static SafeFetch32 _safeFetch32Func;
+
 public:
+  static void initSafeFetch(CodeCache* libjvm);
+
+  static inline int safeFetch32(int* ptr, int errorValue) {
+    assert(_safeFetch32Func != nullptr);
+    return _safeFetch32Func(ptr, errorValue);
+  }
+
   NOINLINE NOADDRSANITIZE __attribute__((aligned(16))) static void *load(void **ptr) {
     return *ptr;
   }