8305209: JDWP exit error AGENT_ERROR_INVALID_THREAD(203): missing entry in running thread table

Reviewed-by: sspitsyn, lmesnik

Author: plummercj

src/jdk.jdwp.agent/share/native/libjdwp/threadControl.c

@@ -253,17 +253,24 @@ findThread(ThreadList *list, jthread thread)
          * Otherwise the thread should not be on the runningThreads.
          */
         if ( !gdata->jvmtiCallBacksCleared ) {
-            /* The thread better not be on runningThreads if the TLS lookup failed. */
+            /* The thread better not be on either list if the TLS lookup failed. */
             JDI_ASSERT(!nonTlsSearch(getEnv(), &runningThreads, thread));
+            JDI_ASSERT(!nonTlsSearch(getEnv(), &runningVThreads, thread));
         } else {
             /*
-             * Search the runningThreads list. The TLS lookup may have failed because the
-             * thread has terminated, but we never got the THREAD_END event.
+             * Search the runningThreads and runningVThreads lists. The TLS lookup may have
+             * failed because the thread has terminated, but we never got the THREAD_END event.
+             * The big comment above explains why this can happen.
              */
             if ( node == NULL ) {
                 if ( list == NULL || list == &runningThreads ) {
                     node = nonTlsSearch(getEnv(), &runningThreads, thread);
                 }
+                if ( node == NULL ) {
+                    if ( list == NULL || list == &runningVThreads ) {
+                        node = nonTlsSearch(getEnv(), &runningVThreads, thread);
+                    }
+                }
             }
         }
     }

test/jdk/com/sun/jdi/TestScaffold.java

@@ -710,6 +710,21 @@ public synchronized void waitForVMDisconnect() {
             } catch (InterruptedException e) {
             }
         }
+
+        // Make sure debuggee exits with no errors. Otherwise failures might go unnoticed.
+        Process p = vm.process();
+        try {
+            p.waitFor();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+        int res = p.exitValue();
+        // Some tests purposefully exit with an exception, which produces exitValue
+        // 1, so we have to allow it also.
+        if (res != 0 && res != 1) {
+            throw new RuntimeException("Non-zero debuggee exitValue: " + res);
+        }
+
         traceln("TS: waitForVMDisconnect: done");
     }
 

test/jdk/com/sun/jdi/ThreadMemoryLeakTest.java

@@ -58,14 +58,20 @@ public static void main(String[] args) throws InterruptedException {
         while (System.currentTimeMillis() - startTime < 100 * 1000) {
             iterations++;
             semaphore.acquire();
-            Executors.defaultThreadFactory().newThread(() -> {
+            TestScaffold.newThread(() -> {
                     adder.increment();
                     long sum = adder.sum();
                     if ((sum % 1000) == 0) {
                         System.out.println("Progress: " + sum);
                     }
                     try {
-                        Thread.sleep(50);
+                        String mainWrapper = System.getProperty("main.wrapper");
+                        // Virtual thread creation tends to overwhelm the debugger,
+                        // leading to high memory use for all the unprocessed events
+                        // that get queued up, so we need to slow it down a bit more
+                        // than we do for platform threads to avoid getting OOME.
+                        long timeToSleep = "Virtual".equals(mainWrapper) ? 100 : 50;
+                        Thread.sleep(timeToSleep);
                     }
                     catch (InterruptedException e) {
                         throw new RuntimeException(e);