Skip to content

Commit c737968

Browse files
committed
Add downstream test discrepency reporter
Example Run and Output: ``` $ python tests/report_downstream_test_diffs.py --playwright-js-path=../playwright --api=async MISSING, MISPELLED, OR MISNAMED: async/test_accessibility.py::test_autocomplete … async/test_workers.py::test_should_have_JSHandles_for_console_logs async/test_workers.py::test_should_report_console_logs async/test_workers.py::test_should_report_errors async/test_workers.py::test_should_report_network_activity async/test_workers.py::test_should_report_network_activity_on_worker_creation Missing: 1213, Found: 105 ``` Relates #138.
1 parent af97862 commit c737968

File tree

1 file changed

+85
-0
lines changed

1 file changed

+85
-0
lines changed
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
import argparse
2+
import json
3+
import os
4+
import re
5+
import subprocess
6+
import tempfile
7+
from collections import namedtuple
8+
9+
TestCase = namedtuple("TestCase", ["api", "file", "test"])
10+
11+
12+
def pytest_test_cases():
13+
p = subprocess.run(
14+
["pytest", "--browser", "chromium", "--collect-only", "-q"],
15+
stdout=subprocess.PIPE,
16+
)
17+
regex = (
18+
r"tests/(?P<api>a?sync)/test_(?P<file>.*)\.py::test_(?P<test>.*)\[chromium\]"
19+
)
20+
matches = re.finditer(regex, p.stdout.decode(), re.MULTILINE)
21+
for match in matches:
22+
yield TestCase(
23+
match.group("api"), match.group("file"), match.group("test"),
24+
)
25+
26+
27+
def jest_test_cases(playwright_js_path):
28+
env = os.environ.copy()
29+
env["REPORT_ONLY"] = "true"
30+
with tempfile.NamedTemporaryFile() as results:
31+
subprocess.run(
32+
["npx", "jest", "--json", "--outputFile", results.name],
33+
env=env,
34+
cwd=playwright_js_path,
35+
stderr=subprocess.DEVNULL,
36+
)
37+
38+
for test_suite in json.load(results)["testResults"]:
39+
regex = r"(.*/)?(?P<file>[^/]+)\.spec\.[jt]s$"
40+
file = re.match(regex, test_suite["name"]).group("file")
41+
for assertion in test_suite["assertionResults"]:
42+
yield TestCase("sync", normalized(file), normalized(assertion["title"]))
43+
yield TestCase(
44+
"async", normalized(file), normalized(assertion["title"])
45+
)
46+
47+
48+
def normalized(original):
49+
return re.sub(r"[^a-z_]", "_", original, flags=re.IGNORECASE)
50+
51+
52+
def main():
53+
parser = argparse.ArgumentParser()
54+
parser.add_argument(
55+
"--playwright-js-path",
56+
type=str,
57+
help="path to playwright JavaScript directory",
58+
required=True,
59+
)
60+
parser.add_argument(
61+
"--api",
62+
type=str,
63+
help="filter test cases based on API",
64+
choices=["sync", "async"],
65+
)
66+
args = parser.parse_args()
67+
68+
python_tests = set(pytest_test_cases())
69+
javascript_tests = set(jest_test_cases(args.playwright_js_path))
70+
71+
if args.api:
72+
javascript_tests = set([x for x in javascript_tests if x[0] == args.api])
73+
74+
missing = javascript_tests.difference(python_tests)
75+
found = javascript_tests.intersection(python_tests)
76+
77+
print("MISSING, MISPELLED, OR MISNAMED:")
78+
for (api, file, test) in sorted(missing):
79+
print(f"{api}/test_{file}.py::test_{test}")
80+
81+
print(f"\nMissing: {len(missing)}, Found: {len(found)}")
82+
83+
84+
if __name__ == "__main__":
85+
main()

0 commit comments

Comments
 (0)