8207793: [TESTBUG] runtime/Metaspace/FragmentMetaspace.java fails: heap needs to be increased

Reduce test time and allow OOM.

Reviewed-by: iklam, hseigel

Author: coleenp

test/hotspot/jtreg/runtime/Metaspace/FragmentMetaspace.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,7 @@
  * @library /runtime/testlibrary
  * @modules java.base/jdk.internal.misc
  * @modules java.compiler
- * @run main/othervm/timeout=200 -Xmx300m FragmentMetaspace
+ * @run main/othervm/timeout=200 -Xmx1g FragmentMetaspace
  */
 
 import java.io.IOException;
@@ -42,8 +42,8 @@ public class FragmentMetaspace {
     public static Class<?> c;
 
     public static void main(String... args) {
-        runGrowing(Long.valueOf(System.getProperty("time", "80000")),
-            Integer.valueOf(System.getProperty("iterations", "200")));
+        runGrowing(Long.valueOf(System.getProperty("time", "40000")),
+            Integer.valueOf(System.getProperty("iterations", "100")));
         // try to clean up and unload classes to decrease
         // class verification time in debug vm
         System.gc();
@@ -68,6 +68,9 @@ private static void runGrowing(long time, int iterations) {
                 gcl = null;
             } catch (IOException | InstantiationException | IllegalAccessException ex) {
                 throw new RuntimeException(ex);
+            } catch (OutOfMemoryError oome) {
+                System.out.println("javac failed with OOM; ignored.");
+                return;
             }
         }
     }

test/hotspot/jtreg/runtime/testlibrary/GeneratedClassLoader.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,6 +22,7 @@
  */
 
 import java.io.DataInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileWriter;
@@ -145,10 +146,17 @@ private GeneratedClass[] getGeneratedClass(int sizeFactor, int numClasses) throw
             pw.append(src);
             pw.flush();
         }
-        int exitcode = javac.run(null, null, null, file.getCanonicalPath());
+        ByteArrayOutputStream err = new ByteArrayOutputStream();
+        int exitcode = javac.run(null, null, err, file.getCanonicalPath());
         if (exitcode != 0) {
-            throw new RuntimeException("javac failure when compiling: " +
-                    file.getCanonicalPath());
+            // Print Error
+            System.err.print(err);
+            if (err.toString().contains("java.lang.OutOfMemoryError: Java heap space")) {
+              throw new OutOfMemoryError("javac failed with resources exhausted");
+            } else {
+              throw new RuntimeException("javac failure when compiling: " +
+                      file.getCanonicalPath());
+            }
         } else {
             if (deleteFiles) {
                 file.delete();