JacoCo Tutorial (Inside Repo)

examples/10_java_coverage/config/config.json

@@ -1,4 +1,7 @@
 {
+  "autograding" : {
+    "work_to_details": [ "*.csv" ]
+  },
   "resource_limits" : {
     "RLIMIT_CPU" : 60,
     "RLIMIT_NPROC" : 100,
@@ -12,21 +15,80 @@
     // instructor-provided JUnit test-based grading.
     {
       "type" : "Compilation",
-      "title" : "Java - Compilation",
+      "title" : "Class Compilation",
       "command" : "javac -cp submitty_junit.jar hw0/Factorial.java",
       "executable_name" : "hw0/Factorial.class",
       "points" : 2
     },
     {
       "type" : "Compilation",
-      "title" : "Java - Student and Instructor JUnit Test Compilation",
+      "title" : "Student and Instructor JUnit Test Compilation",
       "command" : "javac -cp submitty_junit.jar:. hw0/test/*Test.java",
       "executable_name" : "hw0/test/FactorialTest.class",
       "points" : 2
     },
 
+
+    // -----------------------------------------------------------------------
+    // COVERAGE USING JACOCO
+    // JaCoCo can do on-demand instrumentation
+    {
+      "title" : "JaCoCo - Running Student JUnit Tests in hw0/tests/",
+      "command" : "java -noverify -javaagent:submitty_jacocoagent.jar=destfile=coverage.exec -cp submitty_junit.jar:submitty_hamcrest.jar:submitty_junit/:. TestRunner hw0",
+      "points" : 4,
+      "validation" : [
+          { 
+              "method" : "MultipleJUnitTestGrader",
+              "actual_file" : "STDOUT.txt"
+          }
+      ]
+    },
+    {
+      "title" : "JaCoCo - Generating Coverage Report for Student Tests",
+      "command" : "java -jar submitty_jacococli.jar report coverage.exec --classfiles hw0 --csv jacoco_report.csv",
+      "points" : 6,
+      "validation" : [
+        {
+          "method" : "errorIfEmpty",
+          "actual_file" : "STDOUT.txt",
+          "description" : "JaCoCo report generation output",
+          "deduction" : 0.0
+        },
+        {
+          "method" : "JaCoCoCoverageReportGrader",
+          "actual_file" : "jacoco_report.csv",
+          "description" : "JaCoCo coverage report",
+          // specify the package & class to check (can omit to check all)
+         "package" : "hw0",
+          //"class" : "Factorial",
+          // which threshold(s) to apply
+          "instruction_coverage_threshold" : 90,
+          //"branch_coverage_threshold" : 90,
+          //"line_coverage_threshold" : 70,
+          //"complexity_coverage_threshold" : 80,
+          //"method_coverage_threshold" : 80,
+          "deduction" : 1.0
+        }
+      ]
+    },
+    {
+      "title" : "JaCoCo - Instructor JUnit Tests",
+      "command" : "java -noverify -cp submitty_junit.jar:submitty_hamcrest.jar:. org.junit.runner.JUnitCore hw0.test.FactorialTest",
+      "points" : 6,
+      "validation" : [
+        {
+          "method" : "JUnitTestGrader",
+          "actual_file" : "STDOUT.txt",
+          "num_tests" : 4
+        }
+      ]
+    },
+
+
+    // ------------------------------------------------------------------------------
+    // COVERAGE USING EMMA
     {
-      "title" : "Java - Instrumentation of Student Code",
+      "title" : "EMMA - Instrumentation of Student Code",
       "command" : "java -cp submitty_emma.jar emma instr -m overwrite -ip hw0",
       "points" : 0,
       "validation" : [ 
@@ -36,9 +98,8 @@
           }
       ]
     },
-
     {
-      "title" : "Java - Running Student JUnit Tests in hw0/tests/",
+      "title" : "EMMA - Running Student JUnit Tests in hw0/tests/",
       "command" : "java -noverify -cp submitty_junit.jar:submitty_hamcrest.jar:submitty_emma.jar:submitty_junit/:. TestRunner hw0",
       "points" : 4,
       "validation" : [
@@ -48,9 +109,8 @@
           }
       ]
     },
-
     {
-      "title" : "Java - Generating Coverage Report for Student Tests",
+      "title" : "EMMA - Generating Coverage Report for Student Tests",
       "command" : "java -cp submitty_emma.jar emma report -r txt -in coverage.em,coverage.ec -Dreport.txt.out.file=emma_report.txt",
       "points" : 6,
       "validation" : [
@@ -63,15 +123,15 @@
         {
           "method" : "EmmaCoverageReportGrader",
           "actual_file" : "emma_report.txt",
+          "description" : "EclEmma coverage report",
           "coverage_threshold" : 90,
           "deduction" : 1.0
         }
       ]
     },
-
     {
-      "title" : "Java - Instructor JUnit Tests",
-      "command" : "java -noverify -cp submitty_junit.jar:submitty_hamcrest.jar:submitty_emma.jar:. org.junit.runner.JUnitCore hw0.test.FactorialTest",
+      "title" : "EMMA - Instructor JUnit Tests",
+      "command" :  "java -noverify -cp submitty_junit.jar:submitty_hamcrest.jar:submitty_emma.jar:. org.junit.runner.JUnitCore hw0.test.FactorialTest",
       "points" : 6,
       "validation" : [
         {
@@ -81,5 +141,6 @@
         }
       ]
     }
+
   ]
 }

examples/10_java_coverage/config/provided_code/hw0/test/ALMHiddenFactorialTest.java

@@ -18,67 +18,66 @@
 
 public class ALMHiddenFactorialTest {
 
-    private static Factorial fac = null;
+	private static Factorial fac = null;
 
-    @BeforeClass
+	@BeforeClass
 	public static void setupBeforeTests() throws Exception {
-        fac = new Factorial();
-    }
+		fac = new Factorial();
+	}
 
-    /**
-     * Tests that Factorial throws an IllegalArgumentException
-     * for a negative number.
-     */
-    @Test(expected=IllegalArgumentException.class)
+	/**
+	 * Tests that Factorial throws an IllegalArgumentException
+	 * for a negative number.
+	 */
+	@Test(expected=IllegalArgumentException.class)
 	public void expectedIllegalArgumentException() {
-        fac.getFact(-1);
-    }
+		fac.getFact(-1);
+	}
 
-    /**
-     * Tests that Factorial throws no IllegalArgumentException
-     * for zero or for a positive number.
-     */
-    @Test
+	/**
+	 * Tests that Factorial throws no IllegalArgumentException
+	 * for zero or for a positive number.
+	 */
+	@Test
 	public void testThrowsIllegalArgumentException() {
-
-        // test 0
-        try {
-            fac.getFact(0);
-        } catch (IllegalArgumentException ex) {
-            fail("Threw IllegalArgumentException for 0 but 0 is nonnegative: "
+		// test 0
+		try {
+			fac.getFact(0);
+		} catch (IllegalArgumentException ex) {
+			fail("Threw IllegalArgumentException for 0 but 0 is nonnegative: "
 		 + ex);
-        }
-        // test 1
-        try {
-            fac.getFact(1);
-        } catch (IllegalArgumentException ex) {
-            fail("Threw IllegalArgumentException for 1 but 1 is nonnegative: "
+		}
+		// test 1
+		try {
+			fac.getFact(1);
+		} catch (IllegalArgumentException ex) {
+			fail("Threw IllegalArgumentException for 1 but 1 is nonnegative: "
 		 + ex);
-        }
-    }
+		}
+	}
 
-    /** Tests to see that Fibonacci returns the correct value for the base cases, n=0 and n=1 */
-    @Test
+	/** Tests to see that Fibonacci returns the correct value for the base cases, n=0 and n=1 */
+	@Test
 	public void testBaseCase() {
-        assertEquals("getFact(0)", 1, fac.getFact(0));
-        assertEquals("getFact(1)", 1, fac.getFact(1));
-    }
+		assertEquals("getFact(0)", 1, fac.getFact(0));
+		assertEquals("getFact(1)", 1, fac.getFact(1));
+	}
 
-    /** Tests inductive cases of the Factorial sequence */
-    @Test
-	public void testInductiveCase() {  
-	int[][] cases = new int[][] {
-	    { 2, 2 },
-	    { 3, 6 },
-	    { 4, 24 },
-	    { 5, 120 },
-	    { 6, 720 },
-	    { 7, 5040 }
-	};       
-	for (int i = 0; i < cases.length; i++) {
-	    assertEquals("getFact(" + cases[i][0] + ")",
-			 cases[i][1], fac.getFact(cases[i][0]));
+	/** Tests inductive cases of the Factorial sequence */
+	@Test
+	public void testInductiveCase() {
+		int[][] cases = new int[][] {
+			{ 2, 2 },
+			{ 3, 6 },
+			{ 4, 24 },
+			{ 5, 120 },
+			{ 6, 720 },
+			{ 7, 5040 }
+		};
+		for (int i = 0; i < cases.length; i++) {
+			assertEquals("getFact(" + cases[i][0] + ")",
+				cases[i][1], fac.getFact(cases[i][0]));
+		}
 	}
-    }
 
 }