diff --git a/examples/10_java_coverage/config/config.json b/examples/10_java_coverage/config/config.json index 87e92d4..08b13a4 100644 --- a/examples/10_java_coverage/config/config.json +++ b/examples/10_java_coverage/config/config.json @@ -1,4 +1,7 @@ { + "autograding" : { + "work_to_details": [ "*.csv" ] + }, "resource_limits" : { "RLIMIT_CPU" : 60, "RLIMIT_NPROC" : 100, @@ -12,21 +15,80 @@ // instructor-provided JUnit test-based grading. { "type" : "Compilation", - "title" : "Java - Compilation", + "title" : "Class Compilation", "command" : "javac -cp submitty_junit.jar hw0/Factorial.java", "executable_name" : "hw0/Factorial.class", "points" : 2 }, { "type" : "Compilation", - "title" : "Java - Student and Instructor JUnit Test Compilation", + "title" : "Student and Instructor JUnit Test Compilation", "command" : "javac -cp submitty_junit.jar:. hw0/test/*Test.java", "executable_name" : "hw0/test/FactorialTest.class", "points" : 2 }, + + // ----------------------------------------------------------------------- + // COVERAGE USING JACOCO + // JaCoCo can do on-demand instrumentation + { + "title" : "JaCoCo - Running Student JUnit Tests in hw0/tests/", + "command" : "java -noverify -javaagent:submitty_jacocoagent.jar=destfile=coverage.exec -cp submitty_junit.jar:submitty_hamcrest.jar:submitty_junit/:. TestRunner hw0", + "points" : 4, + "validation" : [ + { + "method" : "MultipleJUnitTestGrader", + "actual_file" : "STDOUT.txt" + } + ] + }, + { + "title" : "JaCoCo - Generating Coverage Report for Student Tests", + "command" : "java -jar submitty_jacococli.jar report coverage.exec --classfiles hw0 --csv jacoco_report.csv", + "points" : 6, + "validation" : [ + { + "method" : "errorIfEmpty", + "actual_file" : "STDOUT.txt", + "description" : "JaCoCo report generation output", + "deduction" : 0.0 + }, + { + "method" : "JaCoCoCoverageReportGrader", + "actual_file" : "jacoco_report.csv", + "description" : "JaCoCo coverage report", + // specify the package & class to check (can omit to check all) + "package" : "hw0", + //"class" : "Factorial", + // which threshold(s) to apply + "instruction_coverage_threshold" : 90, + //"branch_coverage_threshold" : 90, + //"line_coverage_threshold" : 70, + //"complexity_coverage_threshold" : 80, + //"method_coverage_threshold" : 80, + "deduction" : 1.0 + } + ] + }, + { + "title" : "JaCoCo - Instructor JUnit Tests", + "command" : "java -noverify -cp submitty_junit.jar:submitty_hamcrest.jar:. org.junit.runner.JUnitCore hw0.test.FactorialTest", + "points" : 6, + "validation" : [ + { + "method" : "JUnitTestGrader", + "actual_file" : "STDOUT.txt", + "num_tests" : 4 + } + ] + }, + + + // ------------------------------------------------------------------------------ + // COVERAGE USING EMMA { - "title" : "Java - Instrumentation of Student Code", + "title" : "EMMA - Instrumentation of Student Code", "command" : "java -cp submitty_emma.jar emma instr -m overwrite -ip hw0", "points" : 0, "validation" : [ @@ -36,9 +98,8 @@ } ] }, - { - "title" : "Java - Running Student JUnit Tests in hw0/tests/", + "title" : "EMMA - Running Student JUnit Tests in hw0/tests/", "command" : "java -noverify -cp submitty_junit.jar:submitty_hamcrest.jar:submitty_emma.jar:submitty_junit/:. TestRunner hw0", "points" : 4, "validation" : [ @@ -48,9 +109,8 @@ } ] }, - { - "title" : "Java - Generating Coverage Report for Student Tests", + "title" : "EMMA - Generating Coverage Report for Student Tests", "command" : "java -cp submitty_emma.jar emma report -r txt -in coverage.em,coverage.ec -Dreport.txt.out.file=emma_report.txt", "points" : 6, "validation" : [ @@ -63,15 +123,15 @@ { "method" : "EmmaCoverageReportGrader", "actual_file" : "emma_report.txt", + "description" : "EclEmma coverage report", "coverage_threshold" : 90, "deduction" : 1.0 } ] }, - { - "title" : "Java - Instructor JUnit Tests", - "command" : "java -noverify -cp submitty_junit.jar:submitty_hamcrest.jar:submitty_emma.jar:. org.junit.runner.JUnitCore hw0.test.FactorialTest", + "title" : "EMMA - Instructor JUnit Tests", + "command" : "java -noverify -cp submitty_junit.jar:submitty_hamcrest.jar:submitty_emma.jar:. org.junit.runner.JUnitCore hw0.test.FactorialTest", "points" : 6, "validation" : [ { @@ -81,5 +141,6 @@ } ] } + ] } diff --git a/examples/10_java_coverage/config/provided_code/hw0/test/ALMHiddenFactorialTest.java b/examples/10_java_coverage/config/provided_code/hw0/test/ALMHiddenFactorialTest.java index 2a4d8dc..01e4f1b 100644 --- a/examples/10_java_coverage/config/provided_code/hw0/test/ALMHiddenFactorialTest.java +++ b/examples/10_java_coverage/config/provided_code/hw0/test/ALMHiddenFactorialTest.java @@ -18,67 +18,66 @@ public class ALMHiddenFactorialTest { - private static Factorial fac = null; + private static Factorial fac = null; - @BeforeClass + @BeforeClass public static void setupBeforeTests() throws Exception { - fac = new Factorial(); - } + fac = new Factorial(); + } - /** - * Tests that Factorial throws an IllegalArgumentException - * for a negative number. - */ - @Test(expected=IllegalArgumentException.class) + /** + * Tests that Factorial throws an IllegalArgumentException + * for a negative number. + */ + @Test(expected=IllegalArgumentException.class) public void expectedIllegalArgumentException() { - fac.getFact(-1); - } + fac.getFact(-1); + } - /** - * Tests that Factorial throws no IllegalArgumentException - * for zero or for a positive number. - */ - @Test + /** + * Tests that Factorial throws no IllegalArgumentException + * for zero or for a positive number. + */ + @Test public void testThrowsIllegalArgumentException() { - - // test 0 - try { - fac.getFact(0); - } catch (IllegalArgumentException ex) { - fail("Threw IllegalArgumentException for 0 but 0 is nonnegative: " + // test 0 + try { + fac.getFact(0); + } catch (IllegalArgumentException ex) { + fail("Threw IllegalArgumentException for 0 but 0 is nonnegative: " + ex); - } - // test 1 - try { - fac.getFact(1); - } catch (IllegalArgumentException ex) { - fail("Threw IllegalArgumentException for 1 but 1 is nonnegative: " + } + // test 1 + try { + fac.getFact(1); + } catch (IllegalArgumentException ex) { + fail("Threw IllegalArgumentException for 1 but 1 is nonnegative: " + ex); - } - } + } + } - /** Tests to see that Fibonacci returns the correct value for the base cases, n=0 and n=1 */ - @Test + /** Tests to see that Fibonacci returns the correct value for the base cases, n=0 and n=1 */ + @Test public void testBaseCase() { - assertEquals("getFact(0)", 1, fac.getFact(0)); - assertEquals("getFact(1)", 1, fac.getFact(1)); - } + assertEquals("getFact(0)", 1, fac.getFact(0)); + assertEquals("getFact(1)", 1, fac.getFact(1)); + } - /** Tests inductive cases of the Factorial sequence */ - @Test - public void testInductiveCase() { - int[][] cases = new int[][] { - { 2, 2 }, - { 3, 6 }, - { 4, 24 }, - { 5, 120 }, - { 6, 720 }, - { 7, 5040 } - }; - for (int i = 0; i < cases.length; i++) { - assertEquals("getFact(" + cases[i][0] + ")", - cases[i][1], fac.getFact(cases[i][0])); + /** Tests inductive cases of the Factorial sequence */ + @Test + public void testInductiveCase() { + int[][] cases = new int[][] { + { 2, 2 }, + { 3, 6 }, + { 4, 24 }, + { 5, 120 }, + { 6, 720 }, + { 7, 5040 } + }; + for (int i = 0; i < cases.length; i++) { + assertEquals("getFact(" + cases[i][0] + ")", + cases[i][1], fac.getFact(cases[i][0])); + } } - } }