diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java
index 60de1981f9827..7a2504efdd0fc 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java
@@ -43,7 +43,7 @@ private void scanForJDKJarHell(Path root) throws IOException {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
                 String entry = root.relativize(file).toString().replace('\\', '/');
-                if (entry.endsWith(".class")) {
+                if (entry.endsWith(".class") && entry.endsWith("module-info.class") == false) {
                     if (ext.getResource(entry) != null) {
                         detected.add(
                             entry
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java
index 7e4766ada6541..bffa011cb7be2 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java
@@ -20,12 +20,12 @@
 
 import org.apache.commons.io.output.NullOutputStream;
 import org.elasticsearch.gradle.JdkJarHellCheck;
-import org.elasticsearch.test.NamingConventionsCheck;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.file.FileCollection;
+import org.gradle.api.file.FileTree;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFile;
 import org.gradle.api.tasks.InputFiles;
@@ -47,6 +47,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 public class ThirdPartyAuditTask extends DefaultTask {
 
@@ -171,19 +172,38 @@ private void extractJars(FileCollection jars) {
         File jarExpandDir = getJarExpandDir();
         // We need to clean up to make sure old dependencies don't linger
         getProject().delete(jarExpandDir);
-        jars.forEach(jar ->
+
+        jars.forEach(jar -> {
+            FileTree jarFiles = getProject().zipTree(jar);
             getProject().copy(spec -> {
+                spec.from(jarFiles);
+                spec.into(jarExpandDir);
+                // exclude classes from multi release jars
+                spec.exclude("META-INF/versions/**");
+            });
+            // Deal with multi release jars:
+            // The order is important, we iterate here so we don't depend on the order in which Gradle executes the spec
+            // We extract multi release jar classes ( if these exist ) going from 9 - the first to support them, to the
+            // current `targetCompatibility` version.
+            // Each extract will overwrite the top level classes that existed before it, the result is that we end up
+            // with a single version of the class in `jarExpandDir`.
+            // This will be the closes version to `targetCompatibility`, the same class that would be loaded in a JVM
+            // that has `targetCompatibility` version.
+            // This means we only scan classes that would be loaded into `targetCompatibility`, and don't look at any
+            // pther version specific implementation of said classes.
+            IntStream.rangeClosed(
+                Integer.parseInt(JavaVersion.VERSION_1_9.getMajorVersion()),
+                Integer.parseInt(targetCompatibility.getMajorVersion())
+            ).forEach(majorVersion -> getProject().copy(spec -> {
                 spec.from(getProject().zipTree(jar));
                 spec.into(jarExpandDir);
-                // Exclude classes for multi release jars above target
-                for (int i = Integer.parseInt(targetCompatibility.getMajorVersion()) + 1;
-                     i <= Integer.parseInt(JavaVersion.VERSION_HIGHER.getMajorVersion());
-                     i++
-                ) {
-                  spec.exclude("META-INF/versions/" + i + "/**");
-                }
-            })
-        );
+                String metaInfPrefix = "META-INF/versions/" + majorVersion;
+                spec.include(metaInfPrefix + "/**");
+                // Drop the version specific prefix
+                spec.eachFile(details -> details.setPath(details.getPath().replace(metaInfPrefix, "")));
+                spec.setIncludeEmptyDirs(false);
+            }));
+        });
     }
 
     private void assertNoJarHell(Set<String> jdkJarHellClasses) {
@@ -276,9 +296,9 @@ private String formatClassList(Set<String> classList) {
     private Set<String> runJdkJarHellCheck() throws IOException {
         ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
         ExecResult execResult = getProject().javaexec(spec -> {
-            URL location = NamingConventionsCheck.class.getProtectionDomain().getCodeSource().getLocation();
+            URL location = JdkJarHellCheck.class.getProtectionDomain().getCodeSource().getLocation();
             if (location.getProtocol().equals("file") == false) {
-                throw new GradleException("Unexpected location for NamingConventionCheck class: " + location);
+                throw new GradleException("Unexpected location for JdkJarHellCheck class: " + location);
             }
             try {
                 spec.classpath(