Skip to content

Commit 5d883a9

Browse files
Merge branch 'main' into cf-576
2 parents 0f1a5e9 + 7942de0 commit 5d883a9

File tree

5 files changed

+59
-6
lines changed

5 files changed

+59
-6
lines changed

codeflash/discovery/pytest_new_process_discovery.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@ def pytest_collection_finish(self, session) -> None:
1616
collected_tests.extend(session.items)
1717
pytest_rootdir = session.config.rootdir
1818

19+
def pytest_collection_modifyitems(config, items):
20+
skip_benchmark = pytest.mark.skip(reason="Skipping benchmark tests")
21+
for item in items:
22+
if "benchmark" in item.fixturenames:
23+
item.add_marker(skip_benchmark)
1924

2025
def parse_pytest_collection_results(pytest_tests: list[Any]) -> list[dict[str, str]]:
2126
test_results = []
@@ -34,7 +39,7 @@ def parse_pytest_collection_results(pytest_tests: list[Any]) -> list[dict[str, s
3439

3540
try:
3641
exitcode = pytest.main(
37-
[tests_root, "-pno:logging", "--collect-only", "-m", "not skip"], plugins=[PytestCollectionPlugin()]
42+
[tests_root, "-p no:logging", "--collect-only", "-m", "not skip",], plugins=[PytestCollectionPlugin()]
3843
)
3944
except Exception as e: # noqa: BLE001
4045
print(f"Failed to collect tests: {e!s}") # noqa: T201

codeflash/verification/test_runner.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
if TYPE_CHECKING:
1717
from codeflash.models.models import TestFiles
1818

19+
BEHAVIORAL_BLOCKLISTED_PLUGINS = ["benchmark"]
20+
BENCHMARKING_BLOCKLISTED_PLUGINS = ["codspeed", "cov", "benchmark", "profiling"]
1921

2022
def execute_test_subprocess(
2123
cmd_list: list[str], cwd: Path, env: dict[str, str] | None, timeout: int = 600
@@ -87,16 +89,18 @@ def run_behavioral_tests(
8789
else:
8890
coverage_cmd.extend(shlex.split(pytest_cmd, posix=IS_POSIX)[1:])
8991

92+
blocklist_args = [f"-p no:{plugin}" for plugin in BEHAVIORAL_BLOCKLISTED_PLUGINS if plugin != "cov"]
9093
results = execute_test_subprocess(
91-
coverage_cmd + common_pytest_args + result_args + test_files, cwd=cwd, env=pytest_test_env, timeout=600
94+
coverage_cmd + common_pytest_args + blocklist_args + result_args + test_files, cwd=cwd, env=pytest_test_env, timeout=600
9295
)
9396
logger.debug(
9497
f"Result return code: {results.returncode}, "
9598
f"{'Result stderr:' + str(results.stderr) if results.stderr else ''}"
9699
)
97100
else:
101+
blocklist_args = [f"-p no:{plugin}" for plugin in BEHAVIORAL_BLOCKLISTED_PLUGINS]
98102
results = execute_test_subprocess(
99-
pytest_cmd_list + common_pytest_args + result_args + test_files,
103+
pytest_cmd_list + common_pytest_args + blocklist_args + result_args + test_files,
100104
cwd=cwd,
101105
env=pytest_test_env,
102106
timeout=600, # TODO: Make this dynamic
@@ -170,8 +174,10 @@ def run_benchmarking_tests(
170174
result_args = [f"--junitxml={result_file_path.as_posix()}", "-o", "junit_logging=all"]
171175
pytest_test_env = test_env.copy()
172176
pytest_test_env["PYTEST_PLUGINS"] = "codeflash.verification.pytest_plugin"
177+
blocklist_args = [f"-p no:{plugin}" for plugin in BENCHMARKING_BLOCKLISTED_PLUGINS]
178+
173179
results = execute_test_subprocess(
174-
pytest_cmd_list + pytest_args + result_args + test_files,
180+
pytest_cmd_list + pytest_args + blocklist_args + result_args + test_files,
175181
cwd=cwd,
176182
env=pytest_test_env,
177183
timeout=600, # TODO: Make this dynamic

docs/docs/how-codeflash-works.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ To optimize code, Codeflash first gathers all necessary context from the codebas
2525

2626
## Verification of correctness
2727

28-
![Verification](/img/verification.svg)
28+
![Verification](/img/codeflash_arch_diagram.gif)
2929

3030
The goal of correctness verification is to ensure that when the original code is replaced by the new code, there are no behavioral changes in the code and the rest of the system. This means the replacement should be completely safe.
3131

@@ -60,4 +60,4 @@ Codeflash implements several techniques to measure code performance accurately.
6060

6161
## Creating Pull Requests
6262

63-
Once an optimization passes all checks, Codeflash creates a pull request through the Codeflash GitHub app directly in your repository. The pull request includes the new code, the speedup percentage, an explanation of the optimization, test statistics including coverage, and the test content itself. You can review and merge the new code if it meets your standards. Feel free to modify the code as needed—we welcome your improvements!
63+
Once an optimization passes all checks, Codeflash creates a pull request through the Codeflash GitHub app directly in your repository. The pull request includes the new code, the speedup percentage, an explanation of the optimization, test statistics including coverage, and the test content itself. You can review and merge the new code if it meets your standards. Feel free to modify the code as needed—we welcome your improvements!
3.36 MB
Loading

tests/test_unit_test_discovery.py

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,48 @@ def test_unit_test_discovery_unittest():
3434
# assert len(tests) > 0
3535
# Unittest discovery within a pytest environment does not work
3636

37+
def test_benchmark_unit_test_discovery_pytest():
38+
with tempfile.TemporaryDirectory() as tmpdirname:
39+
# Create a dummy test file
40+
test_file_path = Path(tmpdirname) / "test_dummy.py"
41+
test_file_content = """
42+
from bubble_sort import sorter
43+
44+
def test_benchmark_sort(benchmark):
45+
benchmark(sorter, [5, 4, 3, 2, 1, 0])
46+
47+
def test_normal_test():
48+
assert sorter(list(reversed(range(100)))) == list(range(100))
49+
50+
def test_normal_test2():
51+
assert sorter(list(reversed(range(100)))) == list(range(100))"""
52+
test_file_path.write_text(test_file_content)
53+
path_obj_tempdirname = Path(tmpdirname)
54+
55+
# Create a file that the test file is testing
56+
code_file_path = path_obj_tempdirname / "bubble_sort.py"
57+
code_file_content = """
58+
def sorter(arr):
59+
return sorted(arr)"""
60+
code_file_path.write_text(code_file_content)
61+
62+
# Create a TestConfig with the temporary directory as the root
63+
test_config = TestConfig(
64+
tests_root=path_obj_tempdirname,
65+
project_root_path=path_obj_tempdirname,
66+
test_framework="pytest",
67+
tests_project_rootdir=path_obj_tempdirname.parent,
68+
)
69+
70+
# Discover tests
71+
tests = discover_unit_tests(test_config)
72+
assert len(tests) == 1
73+
assert 'bubble_sort.sorter' in tests
74+
assert len(tests['bubble_sort.sorter']) == 2
75+
functions = [test.tests_in_file.test_function for test in tests['bubble_sort.sorter']]
76+
assert 'test_normal_test' in functions
77+
assert 'test_normal_test2' in functions
78+
assert 'test_benchmark_sort' not in functions
3779

3880
def test_discover_tests_pytest_with_temp_dir_root():
3981
with tempfile.TemporaryDirectory() as tmpdirname:

0 commit comments

Comments
 (0)