Skip to content

Commit b24d153

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
authored
Assert test run status in CI Vis instrumentation tests (#8316)
1 parent f7e62de commit b24d153

File tree

22 files changed

+533
-988
lines changed

22 files changed

+533
-988
lines changed

‎dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,21 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
1515
def runner = new JUnitCore()
1616

1717
def "test #testcaseName"() {
18-
runFeatures(features)
18+
runFeatures(features, success)
1919

2020
assertSpansData(testcaseName)
2121

2222
where:
23-
testcaseName | features
24-
"test-succeed" | ["org/example/cucumber/calculator/basic_arithmetic.feature"]
25-
"test-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"]
26-
"test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"]
27-
"test-multiple-features-${version()}" | [
23+
testcaseName | success | features
24+
"test-succeed" | true | ["org/example/cucumber/calculator/basic_arithmetic.feature"]
25+
"test-scenario-outline-${version()}" | true | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"]
26+
"test-failure" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"]
27+
"test-multiple-features-${version()}" | false | [
2828
"org/example/cucumber/calculator/basic_arithmetic.feature",
2929
"org/example/cucumber/calculator/basic_arithmetic_failed.feature"
3030
]
31-
"test-name-with-brackets" | ["org/example/cucumber/calculator/name_with_brackets.feature"]
32-
"test-empty-name-${version()}" | ["org/example/cucumber/calculator/empty_scenario_name.feature"]
31+
"test-name-with-brackets" | true | ["org/example/cucumber/calculator/name_with_brackets.feature"]
32+
"test-empty-name-${version()}" | true | ["org/example/cucumber/calculator/empty_scenario_name.feature"]
3333
}
3434

3535
def "test ITR #testcaseName"() {
@@ -56,17 +56,17 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
5656
givenFlakyRetryEnabled(true)
5757
givenFlakyTests(retriedTests)
5858

59-
runFeatures(features)
59+
runFeatures(features, success)
6060

6161
assertSpansData(testcaseName)
6262

6363
where:
64-
testcaseName | features | retriedTests
65-
"test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
66-
"test-retry-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
64+
testcaseName | success | features | retriedTests
65+
"test-failure" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
66+
"test-retry-failure" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
6767
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
6868
]
69-
"test-retry-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature"] | [
69+
"test-retry-scenario-outline-${version()}" | false | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature"] | [
7070
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature:Basic Arithmetic With Examples", "Many additions", null)
7171
]
7272
}
@@ -93,16 +93,28 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
9393
return CucumberTracingListener.FRAMEWORK_VERSION < "7" ? CucumberTracingListener.FRAMEWORK_VERSION : "latest"
9494
}
9595

96-
private void runFeatures(List<String> classpathFeatures) {
96+
private void runFeatures(List<String> classpathFeatures, boolean expectSuccess = true) {
9797
System.setProperty(Constants.GLUE_PROPERTY_NAME, "org.example.cucumber.calculator")
9898
System.setProperty(Constants.FILTER_TAGS_PROPERTY_NAME, "not @Disabled")
9999
System.setProperty(Constants.FEATURES_PROPERTY_NAME, classpathFeatures.stream()
100100
.map(f -> "classpath:" + f).
101101
collect(Collectors.joining(",")))
102102

103103
TestEventsHandlerHolder.start()
104-
runner.run(TestSucceedCucumber)
105-
TestEventsHandlerHolder.stop()
104+
try {
105+
def result = runner.run(TestSucceedCucumber)
106+
if (expectSuccess) {
107+
if (result.getFailureCount() > 0) {
108+
throw new AssertionError("Expected successful execution, got following failures: " + result.getFailures())
109+
}
110+
} else {
111+
if (result.getFailureCount() == 0) {
112+
throw new AssertionError("Expected a failed execution, got no failures")
113+
}
114+
}
115+
} finally {
116+
TestEventsHandlerHolder.stop()
117+
}
106118
}
107119

108120
@Override

‎dd-java-agent/instrumentation/junit-4.10/junit-4.13/src/test/groovy/JUnit413Test.groovy‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -19,32 +19,40 @@ class JUnit413Test extends CiVisibilityInstrumentationTest {
1919
def runner = new JUnitCore()
2020

2121
def "test #testcaseName"() {
22-
runTests(tests)
22+
runTests(tests, success)
2323

2424
assertSpansData(testcaseName)
2525

2626
where:
27-
testcaseName | tests
28-
"test-succeed-before-after" | [TestSucceedBeforeAfter]
29-
"test-succeed-before-class-after-class" | [TestSucceedBeforeClassAfterClass]
30-
"test-succeed-before-param-after-param" | [TestSucceedBeforeParamAfterParam]
31-
"test-failed-before-class" | [TestFailedBeforeClass]
32-
"test-failed-after-class" | [TestFailedAfterClass]
33-
"test-failed-before" | [TestFailedBefore]
34-
"test-failed-after" | [TestFailedAfter]
35-
"test-failed-before-param" | [TestFailedBeforeParam]
36-
"test-failed-after-param" | [TestFailedAfterParam]
27+
testcaseName | success | tests
28+
"test-succeed-before-after" | true | [TestSucceedBeforeAfter]
29+
"test-succeed-before-class-after-class" | true | [TestSucceedBeforeClassAfterClass]
30+
"test-succeed-before-param-after-param" | true | [TestSucceedBeforeParamAfterParam]
31+
"test-failed-before-class" | false | [TestFailedBeforeClass]
32+
"test-failed-after-class" | false | [TestFailedAfterClass]
33+
"test-failed-before" | false | [TestFailedBefore]
34+
"test-failed-after" | false | [TestFailedAfter]
35+
"test-failed-before-param" | false | [TestFailedBeforeParam]
36+
"test-failed-after-param" | false | [TestFailedAfterParam]
3737
}
3838

39-
private void runTests(Collection<Class<?>> tests) {
39+
private void runTests(Collection<Class<?>> tests, boolean expectSuccess = true) {
4040
TestEventsHandlerHolder.start()
4141
try {
4242
Class[] array = tests.toArray(new Class[0])
43-
runner.run(array)
44-
} catch (Throwable ignored) {
45-
// Ignored
43+
def result = runner.run(array)
44+
if (expectSuccess) {
45+
if (result.getFailureCount() > 0) {
46+
throw new AssertionError("Expected successful execution, got following failures: " + result.getFailures())
47+
}
48+
} else {
49+
if (result.getFailureCount() == 0) {
50+
throw new AssertionError("Expected a failed execution, got no failures")
51+
}
52+
}
53+
} finally {
54+
TestEventsHandlerHolder.stop()
4655
}
47-
TestEventsHandlerHolder.stop()
4856
}
4957

5058
@Override

‎dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,15 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
3636
givenFlakyRetryEnabled(true)
3737
givenFlakyTests(retriedTests)
3838

39-
runTests(tests)
39+
runTests(tests, success)
4040

4141
assertSpansData(testcaseName)
4242

4343
where:
44-
testcaseName | tests | retriedTests
45-
"test-failed" | [TestFailedMUnit] | []
46-
"test-retry-failed" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)]
47-
"test-failed-then-succeed" | [TestFailedThenSucceedMUnit] | [new TestIdentifier("org.example.TestFailedThenSucceedMUnit", "Calculator.add", null)]
44+
testcaseName | success | tests | retriedTests
45+
"test-failed" | false | [TestFailedMUnit] | []
46+
"test-retry-failed" | false | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)]
47+
"test-failed-then-succeed" | true | [TestFailedThenSucceedMUnit] | [new TestIdentifier("org.example.TestFailedThenSucceedMUnit", "Calculator.add", null)]
4848
}
4949

5050
def "test early flakiness detection #testcaseName"() {
@@ -79,15 +79,23 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
7979
"test-succeed-impacted" | [TestSucceedMUnit] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)])
8080
}
8181

82-
private void runTests(Collection<Class<?>> tests) {
82+
private void runTests(Collection<Class<?>> tests, boolean expectSuccess = true) {
8383
TestEventsHandlerHolder.start()
8484
try {
8585
Class[] array = tests.toArray(new Class[0])
86-
runner.run(array)
87-
} catch (Throwable ignored) {
88-
// Ignored
86+
def result = runner.run(array)
87+
if (expectSuccess) {
88+
if (result.getFailureCount() > 0) {
89+
throw new AssertionError("Expected successful execution, got following failures: " + result.getFailures())
90+
}
91+
} else {
92+
if (result.getFailureCount() == 0) {
93+
throw new AssertionError("Expected a failed execution, got no failures")
94+
}
95+
}
96+
} finally {
97+
TestEventsHandlerHolder.stop()
8998
}
90-
TestEventsHandlerHolder.stop()
9199
}
92100

93101
String version() {

0 commit comments

Comments
 (0)