diff --git a/codeflash/code_utils/instrument_existing_tests.py b/codeflash/code_utils/instrument_existing_tests.py index e691c2ed2..6eac52809 100644 --- a/codeflash/code_utils/instrument_existing_tests.py +++ b/codeflash/code_utils/instrument_existing_tests.py @@ -462,50 +462,53 @@ def create_wrapper_function(mode: TestingMode = TestingMode.BEHAVIOR) -> ast.Fun ), *( [ + ast.Assign( + targets=[ast.Name(id="test_stdout_tag", ctx=ast.Store())], + value=ast.JoinedStr( + values=[ + ast.FormattedValue(value=ast.Name(id="test_module_name", ctx=ast.Load()), conversion=-1), + ast.Constant(value=":"), + ast.FormattedValue( + value=ast.IfExp( + test=ast.Name(id="test_class_name", ctx=ast.Load()), + body=ast.BinOp( + left=ast.Name(id="test_class_name", ctx=ast.Load()), + op=ast.Add(), + right=ast.Constant(value="."), + ), + orelse=ast.Constant(value=""), + ), + conversion=-1, + ), + ast.FormattedValue(value=ast.Name(id="test_name", ctx=ast.Load()), conversion=-1), + ast.Constant(value=":"), + ast.FormattedValue(value=ast.Name(id="function_name", ctx=ast.Load()), conversion=-1), + ast.Constant(value=":"), + ast.FormattedValue(value=ast.Name(id="loop_index", ctx=ast.Load()), conversion=-1), + ast.Constant(value=":"), + ast.FormattedValue(value=ast.Name(id="invocation_id", ctx=ast.Load()), conversion=-1), + ] + ), + lineno=lineno + 9, + ), ast.Expr( value=ast.Call( func=ast.Name(id="print", ctx=ast.Load()), args=[ ast.JoinedStr( values=[ - ast.Constant(value="!######"), + ast.Constant(value="!$######"), ast.FormattedValue( - value=ast.Name(id="test_module_name", ctx=ast.Load()), conversion=-1 + value=ast.Name(id="test_stdout_tag", ctx=ast.Load()), conversion=-1 ), - ast.Constant(value=":"), - ast.FormattedValue( - value=ast.IfExp( - test=ast.Name(id="test_class_name", ctx=ast.Load()), - body=ast.BinOp( - left=ast.Name(id="test_class_name", ctx=ast.Load()), - op=ast.Add(), - right=ast.Constant(value="."), - ), - orelse=ast.Constant(value=""), - ), - conversion=-1, - ), - ast.FormattedValue(value=ast.Name(id="test_name", ctx=ast.Load()), conversion=-1), - ast.Constant(value=":"), - ast.FormattedValue( - value=ast.Name(id="function_name", ctx=ast.Load()), conversion=-1 - ), - ast.Constant(value=":"), - ast.FormattedValue(value=ast.Name(id="loop_index", ctx=ast.Load()), conversion=-1), - ast.Constant(value=":"), - ast.FormattedValue( - value=ast.Name(id="invocation_id", ctx=ast.Load()), conversion=-1 - ), - ast.Constant(value="######!"), + ast.Constant(value="######$!"), ] ) ], keywords=[], ) - ) + ), ] - if mode == TestingMode.BEHAVIOR - else [] ), ast.Assign( targets=[ast.Name(id="exception", ctx=ast.Store())], value=ast.Constant(value=None), lineno=lineno + 10 @@ -598,56 +601,30 @@ def create_wrapper_function(mode: TestingMode = TestingMode.BEHAVIOR) -> ast.Fun keywords=[], ) ), - *( - [ - ast.Expr( - value=ast.Call( - func=ast.Name(id="print", ctx=ast.Load()), - args=[ - ast.JoinedStr( - values=[ - ast.Constant(value="!######"), - ast.FormattedValue( - value=ast.Name(id="test_module_name", ctx=ast.Load()), conversion=-1 - ), - ast.Constant(value=":"), - ast.FormattedValue( - value=ast.IfExp( - test=ast.Name(id="test_class_name", ctx=ast.Load()), - body=ast.BinOp( - left=ast.Name(id="test_class_name", ctx=ast.Load()), - op=ast.Add(), - right=ast.Constant(value="."), - ), - orelse=ast.Constant(value=""), - ), - conversion=-1, - ), - ast.FormattedValue(value=ast.Name(id="test_name", ctx=ast.Load()), conversion=-1), - ast.Constant(value=":"), - ast.FormattedValue( - value=ast.Name(id="function_name", ctx=ast.Load()), conversion=-1 - ), - ast.Constant(value=":"), - ast.FormattedValue(value=ast.Name(id="loop_index", ctx=ast.Load()), conversion=-1), - ast.Constant(value=":"), - ast.FormattedValue( - value=ast.Name(id="invocation_id", ctx=ast.Load()), conversion=-1 - ), + ast.Expr( + value=ast.Call( + func=ast.Name(id="print", ctx=ast.Load()), + args=[ + ast.JoinedStr( + values=[ + ast.Constant(value="!######"), + ast.FormattedValue(value=ast.Name(id="test_stdout_tag", ctx=ast.Load()), conversion=-1), + *( + [ ast.Constant(value=":"), ast.FormattedValue( value=ast.Name(id="codeflash_duration", ctx=ast.Load()), conversion=-1 ), - ast.Constant(value="######!"), ] - ) - ], - keywords=[], + if mode == TestingMode.PERFORMANCE + else [] + ), + ast.Constant(value="######!"), + ] ) - ) - ] - if mode == TestingMode.PERFORMANCE - else [] + ], + keywords=[], + ) ), *( [ diff --git a/codeflash/verification/codeflash_capture.py b/codeflash/verification/codeflash_capture.py index 3d28a027e..45d046cf6 100644 --- a/codeflash/verification/codeflash_capture.py +++ b/codeflash/verification/codeflash_capture.py @@ -112,9 +112,8 @@ def wrapper(*args, **kwargs) -> None: # noqa: ANN002, ANN003 # Generate invocation id invocation_id = f"{line_id}_{codeflash_test_index}" - print( - f"!######{test_module_name}:{(test_class_name + '.' if test_class_name else '')}{test_name}:{function_name}:{loop_index}:{invocation_id}######!" - ) + test_stdout_tag = f"{test_module_name}:{(test_class_name + '.' if test_class_name else '')}{test_name}:{function_name}:{loop_index}:{invocation_id}" + print(f"!$######{test_stdout_tag}######$!") # Connect to sqlite codeflash_con = sqlite3.connect(f"{tmp_dir_path}_{codeflash_iteration}.sqlite") codeflash_cur = codeflash_con.cursor() @@ -131,6 +130,7 @@ def wrapper(*args, **kwargs) -> None: # noqa: ANN002, ANN003 exception = e finally: gc.enable() + print(f"!######{test_stdout_tag}######!") # Capture instance state after initialization if hasattr(args[0], "__dict__"): diff --git a/codeflash/verification/parse_test_output.py b/codeflash/verification/parse_test_output.py index b9060f4bc..ed9ebf1c8 100644 --- a/codeflash/verification/parse_test_output.py +++ b/codeflash/verification/parse_test_output.py @@ -36,8 +36,8 @@ def parse_func(file_path: Path) -> XMLParser: return parse(file_path, xml_parser) -matches_re = re.compile(r"!######(.*?):(.*?)([^\.:]*?):(.*?):(.*?):(.*?)######!") -cleaner_re = re.compile(r"!######.*?######!|-+\s*Captured\s+(Log|Out)\s*-+\n?") +matches_re_start = re.compile(r"!\$######(.*?):(.*?)([^\.:]*?):(.*?):(.*?):(.*?)######\$!\n") +matches_re_end = re.compile(r"!######(.*?):(.*?)([^\.:]*?):(.*?):(.*?):(.*?)######!") def parse_test_return_values_bin(file_location: Path, test_files: TestFiles, test_config: TestConfig) -> TestResults: @@ -265,12 +265,16 @@ def parse_test_xml( timed_out = True sys_stdout = testcase.system_out or "" - matches = matches_re.findall(sys_stdout) - - if sys_stdout: - sys_stdout = cleaner_re.sub("", sys_stdout).strip() - - if not matches or not len(matches): + begin_matches = list(matches_re_start.finditer(sys_stdout)) + end_matches = {} + for match in matches_re_end.finditer(sys_stdout): + groups = match.groups() + if len(groups[5].split(":")) > 1: + iteration_id = groups[5].split(":")[0] + groups = groups[:5] + (iteration_id,) + end_matches[groups] = match + + if not begin_matches or not begin_matches: test_results.add( FunctionTestInvocation( loop_index=loop_index, @@ -288,26 +292,36 @@ def parse_test_xml( test_type=test_type, return_value=None, timed_out=timed_out, - stdout=sys_stdout, + stdout="", ) ) else: - for match in matches: - split_val = match[5].split(":") - if len(split_val) > 1: - iteration_id = split_val[0] - runtime = int(split_val[1]) + for match_index, match in enumerate(begin_matches): + groups = match.groups() + end_match = end_matches.get(groups) + iteration_id, runtime = groups[5], None + if end_match: + stdout = sys_stdout[match.end() : end_match.start()] + split_val = end_match.groups()[5].split(":") + if len(split_val) > 1: + iteration_id = split_val[0] + runtime = int(split_val[1]) + else: + iteration_id, runtime = split_val[0], None + elif match_index == len(begin_matches) - 1: + stdout = sys_stdout[match.end() :] else: - iteration_id, runtime = split_val[0], None + stdout = sys_stdout[match.end() : begin_matches[match_index + 1].start()] + test_results.add( FunctionTestInvocation( - loop_index=int(match[4]), + loop_index=int(groups[4]), id=InvocationId( - test_module_path=match[0], - test_class_name=None if match[1] == "" else match[1][:-1], - test_function_name=match[2], - function_getting_tested=match[3], + test_module_path=groups[0], + test_class_name=None if groups[1] == "" else groups[1][:-1], + test_function_name=groups[2], + function_getting_tested=groups[3], iteration_id=iteration_id, ), file_name=test_file_path, @@ -317,7 +331,7 @@ def parse_test_xml( test_type=test_type, return_value=None, timed_out=timed_out, - stdout=sys_stdout, + stdout=stdout, ) ) diff --git a/tests/test_instrument_all_and_run.py b/tests/test_instrument_all_and_run.py index 5bc942fdd..7e1a20f49 100644 --- a/tests/test_instrument_all_and_run.py +++ b/tests/test_instrument_all_and_run.py @@ -25,7 +25,8 @@ codeflash_wrap.index[test_id] = 0 codeflash_test_index = codeflash_wrap.index[test_id] invocation_id = f'{{line_id}}_{{codeflash_test_index}}' - print(f"!######{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}######!") + test_stdout_tag = f"{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}" + print(f"!$######{{test_stdout_tag}}######$!") exception = None gc.disable() try: @@ -36,6 +37,7 @@ codeflash_duration = time.perf_counter_ns() - counter exception = e gc.enable() + print(f"!######{{test_stdout_tag}}######!") pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value) codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call')) codeflash_con.commit() @@ -170,9 +172,8 @@ def test_sort(): out_str = """codeflash stdout: Sorting list result: [0, 1, 2, 3, 4, 5] - -codeflash stdout: Sorting list -result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]""" +""" + assert test_results[0].stdout == out_str assert out_str == test_results[0].stdout assert test_results[0].id.function_getting_tested == "sorter" assert test_results[0].id.iteration_id == "1_0" @@ -185,7 +186,10 @@ def test_sort(): assert test_results[0].runtime > 0 assert test_results[0].did_pass assert test_results[0].return_value == ([0, 1, 2, 3, 4, 5],) - assert out_str == test_results[1].stdout.strip() + out_str = """codeflash stdout: Sorting list +result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] +""" + assert out_str == test_results[1].stdout assert test_results[1].id.function_getting_tested == "sorter" assert test_results[1].id.iteration_id == "4_0" @@ -197,6 +201,10 @@ def test_sort(): ) assert test_results[1].runtime > 0 assert test_results[1].did_pass + out_str = """codeflash stdout: Sorting list +result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] +""" + assert test_results[1].stdout == out_str results2, _ = func_optimizer.run_and_parse_tests( testing_type=TestingMode.BEHAVIOR, test_env=test_env, @@ -208,10 +216,8 @@ def test_sort(): ) out_str = """codeflash stdout: Sorting list result: [0, 1, 2, 3, 4, 5] - -codeflash stdout: Sorting list -result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]""" - assert out_str == results2[0].stdout.strip() +""" + assert out_str == results2[0].stdout assert compare_test_results(test_results, results2) finally: fto_path.write_text(original_code, "utf-8") @@ -234,7 +240,8 @@ def test_sort(): output = sort_class.sorter(input) assert output == [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]""" - expected = """import gc + expected = ( + """import gc import os import sqlite3 import time @@ -244,40 +251,9 @@ def test_sort(): from code_to_optimize.bubble_sort_method import BubbleSorter -def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, function_name, line_id, loop_index, codeflash_cur, codeflash_con, *args, **kwargs): - test_id = f'{{test_module_name}}:{{test_class_name}}:{{test_name}}:{{line_id}}:{{loop_index}}' - if not hasattr(codeflash_wrap, 'index'): - codeflash_wrap.index = {{}} - if test_id in codeflash_wrap.index: - codeflash_wrap.index[test_id] += 1 - else: - codeflash_wrap.index[test_id] = 0 - codeflash_test_index = codeflash_wrap.index[test_id] - invocation_id = f'{{line_id}}_{{codeflash_test_index}}' - """ - if sys.version_info < (3, 12): - expected += """print(f"!######{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}######!")""" - else: - expected += """print(f'!######{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}######!')""" - expected += """ - exception = None - gc.disable() - try: - counter = time.perf_counter_ns() - return_value = wrapped(*args, **kwargs) - codeflash_duration = time.perf_counter_ns() - counter - except Exception as e: - codeflash_duration = time.perf_counter_ns() - counter - exception = e - gc.enable() - pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value) - codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call')) - codeflash_con.commit() - if exception: - raise exception - return return_value """ - expected += """ + + codeflash_wrap_string + + """ def test_sort(): codeflash_loop_index = int(os.environ['CODEFLASH_LOOP_INDEX']) codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] @@ -294,6 +270,7 @@ def test_sort(): assert output == [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] codeflash_con.close() """ + ) fto_path = (Path(__file__).parent.resolve() / "../code_to_optimize/bubble_sort_method.py").resolve() original_code = fto_path.read_text("utf-8") fto = FunctionToOptimize( @@ -379,7 +356,7 @@ def test_sort(): assert test_results[1].runtime > 0 assert test_results[1].did_pass assert test_results[1].return_value == ([0, 1, 2, 3, 4, 5],) - out_str = """codeflash stdout : BubbleSorter.sorter() called\n\n\ncodeflash stdout : BubbleSorter.sorter() called""" + out_str = """codeflash stdout : BubbleSorter.sorter() called\n""" assert test_results[1].stdout == out_str assert compare_test_results(test_results, test_results) assert test_results[2].id.function_getting_tested == "BubbleSorter.__init__" @@ -397,6 +374,7 @@ def test_sort(): ) assert test_results[3].runtime > 0 assert test_results[3].did_pass + assert test_results[3].stdout == """codeflash stdout : BubbleSorter.sorter() called\n""" results2, _ = func_optimizer.run_and_parse_tests( testing_type=TestingMode.BEHAVIOR, diff --git a/tests/test_instrument_tests.py b/tests/test_instrument_tests.py index 44661912a..a117c2205 100644 --- a/tests/test_instrument_tests.py +++ b/tests/test_instrument_tests.py @@ -37,7 +37,8 @@ codeflash_wrap.index[test_id] = 0 codeflash_test_index = codeflash_wrap.index[test_id] invocation_id = f'{{line_id}}_{{codeflash_test_index}}' - print(f"!######{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}######!") + test_stdout_tag = f"{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}" + print(f"!$######{{test_stdout_tag}}######$!") exception = None gc.disable() try: @@ -48,6 +49,7 @@ codeflash_duration = time.perf_counter_ns() - counter exception = e gc.enable() + print(f"!######{{test_stdout_tag}}######!") pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value) codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call')) codeflash_con.commit() @@ -66,6 +68,8 @@ codeflash_wrap.index[test_id] = 0 codeflash_test_index = codeflash_wrap.index[test_id] invocation_id = f'{{line_id}}_{{codeflash_test_index}}' + test_stdout_tag = f"{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}" + print(f"!$######{{test_stdout_tag}}######$!") exception = None gc.disable() try: @@ -76,7 +80,7 @@ codeflash_duration = time.perf_counter_ns() - counter exception = e gc.enable() - print(f"!######{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}:{{codeflash_duration}}######!") + print(f"!######{{test_stdout_tag}}:{{codeflash_duration}}######!") if exception: raise exception return return_value @@ -126,10 +130,12 @@ def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, functi invocation_id = f'{{line_id}}_{{codeflash_test_index}}' """ if sys.version_info < (3, 12): - expected += """print(f"!######{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}######!")""" + expected += """test_stdout_tag = f"{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}" + """ else: - expected += """print(f'!######{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}######!')""" - expected += """ + expected += """test_stdout_tag = f'{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}' + """ + expected += """print(f'!$######{{test_stdout_tag}}######$!') exception = None gc.disable() try: @@ -140,6 +146,7 @@ def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, functi codeflash_duration = time.perf_counter_ns() - counter exception = e gc.enable() + print(f'!######{{test_stdout_tag}}######!') pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value) codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call')) codeflash_con.commit() @@ -227,10 +234,12 @@ def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, functi invocation_id = f'{{line_id}}_{{codeflash_test_index}}' """ if sys.version_info < (3, 12): - expected += """print(f"!######{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}######!")""" + expected += """test_stdout_tag = f"{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}" + """ else: - expected += """print(f'!######{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}######!')""" - expected += """ + expected += """test_stdout_tag = f'{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}' + """ + expected += """print(f'!$######{{test_stdout_tag}}######$!') exception = None gc.disable() try: @@ -241,6 +250,7 @@ def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, functi codeflash_duration = time.perf_counter_ns() - counter exception = e gc.enable() + print(f'!######{{test_stdout_tag}}######!') pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value) codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call')) codeflash_con.commit() @@ -287,10 +297,12 @@ def test_prepare_image_for_yolo(): def test_perfinjector_bubble_sort_results() -> None: computed_fn_opt = False code = """from code_to_optimize.bubble_sort import sorter +import datetime def test_sort(): input = [5, 4, 3, 2, 1, 0] + print(datetime.datetime.now().isoformat()) output = sorter(input) assert output == [0, 1, 2, 3, 4, 5] @@ -299,7 +311,8 @@ def test_sort(): assert output == [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]""" expected = ( - """import gc + """import datetime +import gc import os import sqlite3 import time @@ -319,17 +332,19 @@ def test_sort(): codeflash_cur = codeflash_con.cursor() codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') input = [5, 4, 3, 2, 1, 0] - output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '1', codeflash_loop_index, codeflash_cur, codeflash_con, input) + print(datetime.datetime.now().isoformat()) + output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '2', codeflash_loop_index, codeflash_cur, codeflash_con, input) assert output == [0, 1, 2, 3, 4, 5] input = [5.0, 4.0, 3.0, 2.0, 1.0, 0.0] - output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '4', codeflash_loop_index, codeflash_cur, codeflash_con, input) + output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '5', codeflash_loop_index, codeflash_cur, codeflash_con, input) assert output == [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] codeflash_con.close() """ ) expected_perfonly = ( - """import gc + """import datetime +import gc import os import time @@ -342,10 +357,11 @@ def test_sort(): def test_sort(): codeflash_loop_index = int(os.environ['CODEFLASH_LOOP_INDEX']) input = [5, 4, 3, 2, 1, 0] - output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '1', codeflash_loop_index, input) + print(datetime.datetime.now().isoformat()) + output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '2', codeflash_loop_index, input) assert output == [0, 1, 2, 3, 4, 5] input = [5.0, 4.0, 3.0, 2.0, 1.0, 0.0] - output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '4', codeflash_loop_index, input) + output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '5', codeflash_loop_index, input) assert output == [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] """ ) @@ -370,7 +386,7 @@ def test_sort(): os.chdir(run_cwd) success, new_test = inject_profiling_into_existing_test( test_path, - [CodePosition(6, 13), CodePosition(10, 13)], + [CodePosition(8, 14), CodePosition(12, 14)], func, project_root_path, "pytest", @@ -386,7 +402,7 @@ def test_sort(): success, new_perf_test = inject_profiling_into_existing_test( test_path, - [CodePosition(6, 13), CodePosition(10, 13)], + [CodePosition(8, 14), CodePosition(12, 14)], func, project_root_path, "pytest", @@ -436,7 +452,7 @@ def test_sort(): testing_time=0.1, ) assert test_results[0].id.function_getting_tested == "sorter" - assert test_results[0].id.iteration_id == "1_0" + assert test_results[0].id.iteration_id == "2_0" assert test_results[0].id.test_class_name is None assert test_results[0].id.test_function_name == "test_sort" assert ( @@ -448,7 +464,7 @@ def test_sort(): assert test_results[0].return_value == ([0, 1, 2, 3, 4, 5],) assert test_results[1].id.function_getting_tested == "sorter" - assert test_results[1].id.iteration_id == "4_0" + assert test_results[1].id.iteration_id == "5_0" assert test_results[1].id.test_class_name is None assert test_results[1].id.test_function_name == "test_sort" assert ( @@ -471,7 +487,7 @@ def test_sort(): testing_time=0.1, ) assert test_results_perf[0].id.function_getting_tested == "sorter" - assert test_results_perf[0].id.iteration_id == "1_0" + assert test_results_perf[0].id.iteration_id == "2_0" assert test_results_perf[0].id.test_class_name is None assert test_results_perf[0].id.test_function_name == "test_sort" assert ( @@ -481,9 +497,15 @@ def test_sort(): assert test_results_perf[0].runtime > 0 assert test_results_perf[0].did_pass assert test_results_perf[0].return_value is None + assert ( + test_results_perf[0].stdout + == """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5] +""" + ) assert test_results_perf[1].id.function_getting_tested == "sorter" - assert test_results_perf[1].id.iteration_id == "4_0" + assert test_results_perf[1].id.iteration_id == "5_0" assert test_results_perf[1].id.test_class_name is None assert test_results_perf[1].id.test_function_name == "test_sort" assert ( @@ -492,12 +514,11 @@ def test_sort(): ) assert test_results_perf[1].runtime > 0 assert test_results_perf[1].did_pass - out_str = """codeflash stdout: Sorting list -result: [0, 1, 2, 3, 4, 5] -codeflash stdout: Sorting list -result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]""" - assert out_str == test_results_perf[1].stdout + out_str = """codeflash stdout: Sorting list +result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] +""" + assert test_results_perf[1].stdout == out_str ctx_result = func_optimizer.get_code_optimization_context() code_context: CodeOptimizationContext = ctx_result.unwrap() original_helper_code: dict[Path, str] = {} @@ -507,8 +528,7 @@ def test_sort(): helper_code = f.read() original_helper_code[helper_function_path] = helper_code computed_fn_opt = True - line_profiler_output_file = add_decorator_imports( - func_optimizer.function_to_optimize, code_context) + line_profiler_output_file = add_decorator_imports(func_optimizer.function_to_optimize, code_context) line_profile_results, _ = func_optimizer.run_and_parse_tests( testing_type=TestingMode.LINE_PROFILE, test_env=test_env, @@ -517,14 +537,16 @@ def test_sort(): pytest_min_loops=1, pytest_max_loops=1, testing_time=0.1, - line_profiler_output_file = line_profiler_output_file + line_profiler_output_file=line_profiler_output_file, ) tmp_lpr = list(line_profile_results["timings"].keys()) - assert len(tmp_lpr) == 1 and line_profile_results["timings"][tmp_lpr[0]][0][1]==2 + assert len(tmp_lpr) == 1 and line_profile_results["timings"][tmp_lpr[0]][0][1] == 2 finally: if computed_fn_opt: func_optimizer.write_code_and_helpers( - func_optimizer.function_to_optimize_source_code, original_helper_code, func_optimizer.function_to_optimize.file_path + func_optimizer.function_to_optimize_source_code, + original_helper_code, + func_optimizer.function_to_optimize.file_path, ) test_path.unlink(missing_ok=True) test_path_perf.unlink(missing_ok=True) @@ -682,6 +704,12 @@ def test_sort_parametrized(input, expected_output): ) assert test_results[0].runtime > 0 assert test_results[0].did_pass + assert ( + test_results[0].stdout + == """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5] +""" + ) assert test_results[1].id.function_getting_tested == "sorter" assert test_results[1].id.iteration_id == "0_1" @@ -693,6 +721,12 @@ def test_sort_parametrized(input, expected_output): ) assert test_results[1].runtime > 0 assert test_results[1].did_pass + assert ( + test_results[1].stdout + == """codeflash stdout: Sorting list +result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] +""" + ) assert test_results[2].id.function_getting_tested == "sorter" assert test_results[2].id.iteration_id == "0_2" @@ -738,7 +772,8 @@ def test_sort_parametrized(input, expected_output): assert test_results_perf[1].did_pass out_str = """codeflash stdout: Sorting list -result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]""" +result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] +""" assert out_str == test_results_perf[1].stdout assert test_results_perf[2].id.function_getting_tested == "sorter" @@ -760,8 +795,7 @@ def test_sort_parametrized(input, expected_output): helper_code = f.read() original_helper_code[helper_function_path] = helper_code computed_fn_opt = True - line_profiler_output_file = add_decorator_imports( - func_optimizer.function_to_optimize, code_context) + line_profiler_output_file = add_decorator_imports(func_optimizer.function_to_optimize, code_context) line_profile_results, _ = func_optimizer.run_and_parse_tests( testing_type=TestingMode.LINE_PROFILE, test_env=test_env, @@ -770,14 +804,16 @@ def test_sort_parametrized(input, expected_output): pytest_min_loops=1, pytest_max_loops=1, testing_time=0.1, - line_profiler_output_file = line_profiler_output_file + line_profiler_output_file=line_profiler_output_file, ) tmp_lpr = list(line_profile_results["timings"].keys()) - assert len(tmp_lpr) == 1 and line_profile_results["timings"][tmp_lpr[0]][0][1]==3 + assert len(tmp_lpr) == 1 and line_profile_results["timings"][tmp_lpr[0]][0][1] == 3 finally: if computed_fn_opt: func_optimizer.write_code_and_helpers( - func_optimizer.function_to_optimize_source_code, original_helper_code, func_optimizer.function_to_optimize.file_path + func_optimizer.function_to_optimize_source_code, + original_helper_code, + func_optimizer.function_to_optimize.file_path, ) test_path.unlink(missing_ok=True) test_path_perf.unlink(missing_ok=True) @@ -954,7 +990,10 @@ def test_sort_parametrized_loop(input, expected_output): assert test_results[0].runtime > 0 assert test_results[0].did_pass assert test_results[0].return_value == ([0, 1, 2, 3, 4, 5],) - + out_str = """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5] +""" + assert test_results[0].stdout == out_str assert test_results[1].id.function_getting_tested == "sorter" assert test_results[1].id.iteration_id == "0_0_1" assert test_results[1].id.test_class_name is None @@ -965,6 +1004,7 @@ def test_sort_parametrized_loop(input, expected_output): ) assert test_results[1].runtime > 0 assert test_results[1].did_pass + assert test_results[1].stdout == out_str assert test_results[2].id.function_getting_tested == "sorter" assert test_results[2].id.iteration_id == "0_0_2" @@ -976,6 +1016,10 @@ def test_sort_parametrized_loop(input, expected_output): ) assert test_results[2].runtime > 0 assert test_results[2].did_pass + out_str = """codeflash stdout: Sorting list +result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] +""" + assert test_results[2].stdout == out_str assert test_results[3].id.function_getting_tested == "sorter" assert test_results[3].id.iteration_id == "0_0_3" @@ -988,6 +1032,8 @@ def test_sort_parametrized_loop(input, expected_output): assert test_results[3].runtime > 0 assert test_results[3].did_pass + assert test_results[3].stdout == out_str + assert test_results[4].id.function_getting_tested == "sorter" assert test_results[4].id.iteration_id == "0_0_4" assert test_results[4].id.test_class_name is None @@ -998,6 +1044,10 @@ def test_sort_parametrized_loop(input, expected_output): ) assert test_results[4].runtime > 0 assert test_results[4].did_pass + out_str = """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49] +""" + assert test_results[4].stdout == out_str assert test_results[5].id.function_getting_tested == "sorter" assert test_results[5].id.iteration_id == "0_0_5" @@ -1009,6 +1059,7 @@ def test_sort_parametrized_loop(input, expected_output): ) assert test_results[5].runtime > 0 assert test_results[5].did_pass + assert test_results[5].stdout == out_str test_results, _ = func_optimizer.run_and_parse_tests( testing_type=TestingMode.PERFORMANCE, @@ -1099,8 +1150,7 @@ def test_sort_parametrized_loop(input, expected_output): helper_code = f.read() original_helper_code[helper_function_path] = helper_code computed_fn_opt = True - line_profiler_output_file = add_decorator_imports( - func_optimizer.function_to_optimize, code_context) + line_profiler_output_file = add_decorator_imports(func_optimizer.function_to_optimize, code_context) line_profile_results, _ = func_optimizer.run_and_parse_tests( testing_type=TestingMode.LINE_PROFILE, test_env=test_env, @@ -1109,14 +1159,16 @@ def test_sort_parametrized_loop(input, expected_output): pytest_min_loops=1, pytest_max_loops=1, testing_time=0.1, - line_profiler_output_file = line_profiler_output_file + line_profiler_output_file=line_profiler_output_file, ) tmp_lpr = list(line_profile_results["timings"].keys()) - assert len(tmp_lpr) == 1 and line_profile_results["timings"][tmp_lpr[0]][0][1]==6 + assert len(tmp_lpr) == 1 and line_profile_results["timings"][tmp_lpr[0]][0][1] == 6 finally: if computed_fn_opt: func_optimizer.write_code_and_helpers( - func_optimizer.function_to_optimize_source_code, original_helper_code, func_optimizer.function_to_optimize.file_path + func_optimizer.function_to_optimize_source_code, + original_helper_code, + func_optimizer.function_to_optimize.file_path, ) test_path.unlink(missing_ok=True) test_path_behavior.unlink(missing_ok=True) @@ -1335,13 +1387,9 @@ def test_sort(): assert test_results[0].return_value is None out_str = """codeflash stdout: Sorting list result: [0, 1, 2, 3, 4, 5] +""" -codeflash stdout: Sorting list -result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] - -codeflash stdout: Sorting list -result: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49]""" - assert test_results[1].stdout == out_str + assert test_results[0].stdout == out_str assert test_results[1].id.function_getting_tested == "sorter" assert test_results[1].id.iteration_id == "2_2_1" assert test_results[1].id.test_class_name is None @@ -1352,6 +1400,10 @@ def test_sort(): ) assert test_results[1].runtime > 0 assert test_results[1].did_pass + out_str2 = """codeflash stdout: Sorting list +result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] +""" + assert test_results[1].stdout == out_str2 assert test_results[2].id.function_getting_tested == "sorter" assert test_results[2].id.iteration_id == "2_2_2" @@ -1363,6 +1415,10 @@ def test_sort(): ) assert test_results[2].runtime > 0 assert test_results[2].did_pass + out_str3 = """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49] +""" + assert test_results[2].stdout == out_str3 ctx_result = func_optimizer.get_code_optimization_context() code_context: CodeOptimizationContext = ctx_result.unwrap() original_helper_code: dict[Path, str] = {} @@ -1372,8 +1428,7 @@ def test_sort(): helper_code = f.read() original_helper_code[helper_function_path] = helper_code computed_fn_opt = True - line_profiler_output_file = add_decorator_imports( - func_optimizer.function_to_optimize, code_context) + line_profiler_output_file = add_decorator_imports(func_optimizer.function_to_optimize, code_context) line_profile_results, _ = func_optimizer.run_and_parse_tests( testing_type=TestingMode.LINE_PROFILE, test_env=test_env, @@ -1382,14 +1437,16 @@ def test_sort(): pytest_min_loops=1, pytest_max_loops=1, testing_time=0.1, - line_profiler_output_file = line_profiler_output_file + line_profiler_output_file=line_profiler_output_file, ) tmp_lpr = list(line_profile_results["timings"].keys()) - assert len(tmp_lpr) == 1 and line_profile_results["timings"][tmp_lpr[0]][0][1]==3 + assert len(tmp_lpr) == 1 and line_profile_results["timings"][tmp_lpr[0]][0][1] == 3 finally: if computed_fn_opt is True: func_optimizer.write_code_and_helpers( - func_optimizer.function_to_optimize_source_code, original_helper_code, func_optimizer.function_to_optimize.file_path + func_optimizer.function_to_optimize_source_code, + original_helper_code, + func_optimizer.function_to_optimize.file_path, ) test_path.unlink(missing_ok=True) test_path_perf.unlink(missing_ok=True) @@ -1594,6 +1651,10 @@ def test_sort(self): assert test_results[0].runtime > 0 assert test_results[0].did_pass assert test_results[0].return_value == ([0, 1, 2, 3, 4, 5],) + out_str = """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5] +""" + assert test_results[0].stdout == out_str assert test_results[1].id.function_getting_tested == "sorter" assert test_results[1].id.iteration_id == "4_0" @@ -1605,6 +1666,10 @@ def test_sort(self): ) assert test_results[1].runtime > 0 assert test_results[1].did_pass + out_str = """codeflash stdout: Sorting list +result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] +""" + assert test_results[1].stdout == out_str assert test_results[2].id.function_getting_tested == "sorter" assert test_results[2].id.iteration_id == "7_0" @@ -1657,6 +1722,10 @@ def test_sort(self): ) assert test_results[2].runtime > 0 assert test_results[2].did_pass + out_str = """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49] +""" + assert test_results[2].stdout == out_str finally: test_path.unlink(missing_ok=True) test_path_behavior.unlink(missing_ok=True) @@ -1841,6 +1910,10 @@ def test_sort(self, input, expected_output): assert test_results[0].runtime > 0 assert test_results[0].did_pass assert test_results[0].return_value == ([0, 1, 2, 3, 4, 5],) + out_str = """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5] +""" + assert test_results[0].stdout == out_str assert test_results[1].id.function_getting_tested == "sorter" assert test_results[1].id.iteration_id == "0_1" @@ -1852,6 +1925,10 @@ def test_sort(self, input, expected_output): ) assert test_results[1].runtime > 0 assert test_results[1].did_pass + out_str = """codeflash stdout: Sorting list +result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] +""" + assert test_results[1].stdout == out_str assert test_results[2].id.function_getting_tested == "sorter" assert test_results[2].id.iteration_id == "0_2" @@ -1863,6 +1940,10 @@ def test_sort(self, input, expected_output): ) assert test_results[2].runtime > 0 assert test_results[2].did_pass + out_str = """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49] +""" + assert test_results[2].stdout == out_str test_results, coverage_data = func_optimizer.run_and_parse_tests( testing_type=TestingMode.PERFORMANCE, @@ -2092,6 +2173,10 @@ def test_sort(self): assert test_results[0].runtime > 0 assert test_results[0].did_pass assert test_results[0].return_value == ([0, 1, 2, 3, 4, 5],) + out_str = """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5] +""" + assert test_results[0].stdout == out_str assert test_results[1].id.function_getting_tested == "sorter" assert test_results[1].id.iteration_id == "2_2_1" @@ -2103,6 +2188,10 @@ def test_sort(self): ) assert test_results[1].runtime > 0 assert test_results[1].did_pass + out_str = """codeflash stdout: Sorting list +result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] +""" + assert test_results[1].stdout == out_str assert test_results[2].id.function_getting_tested == "sorter" assert test_results[2].id.iteration_id == "2_2_2" @@ -2114,6 +2203,10 @@ def test_sort(self): ) assert test_results[2].runtime > 0 assert test_results[2].did_pass + out_str = """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49] +""" + assert test_results[2].stdout == out_str test_results, coverage_data = func_optimizer.run_and_parse_tests( test_env=test_env, @@ -2341,6 +2434,10 @@ def test_sort(self, input, expected_output): assert test_results[0].runtime > 0 assert test_results[0].did_pass assert test_results[0].return_value == ([0, 1, 2, 3, 4, 5],) + out_str = """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5] +""" + assert test_results[0].stdout == out_str assert test_results[1].id.function_getting_tested == "sorter" assert test_results[1].id.iteration_id == "0_0_1" @@ -2352,6 +2449,10 @@ def test_sort(self, input, expected_output): ) assert test_results[1].runtime > 0 assert test_results[1].did_pass + out_str = """codeflash stdout: Sorting list +result: [0, 1, 2, 3, 4, 5] +""" + assert test_results[1].stdout == out_str assert test_results[2].id.function_getting_tested == "sorter" assert test_results[2].id.iteration_id == "0_0_2" @@ -2363,6 +2464,10 @@ def test_sort(self, input, expected_output): ) assert test_results[2].runtime > 0 assert test_results[2].did_pass + out_str = """codeflash stdout: Sorting list +result: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] +""" + assert test_results[2].stdout == out_str assert test_results[3].id.function_getting_tested == "sorter" assert test_results[3].id.iteration_id == "0_0_3" @@ -2821,7 +2926,8 @@ def test_code_replacement10() -> None: assert code_context.testgen_context_code == get_code_output """ - expected = """import gc + expected = ( + """import gc import os import sqlite3 import time @@ -2831,39 +2937,9 @@ def test_code_replacement10() -> None: from codeflash.optimization.optimizer import Optimizer -def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, function_name, line_id, loop_index, codeflash_cur, codeflash_con, *args, **kwargs): - test_id = f'{{test_module_name}}:{{test_class_name}}:{{test_name}}:{{line_id}}:{{loop_index}}' - if not hasattr(codeflash_wrap, 'index'): - codeflash_wrap.index = {{}} - if test_id in codeflash_wrap.index: - codeflash_wrap.index[test_id] += 1 - else: - codeflash_wrap.index[test_id] = 0 - codeflash_test_index = codeflash_wrap.index[test_id] - invocation_id = f'{{line_id}}_{{codeflash_test_index}}' """ - if sys.version_info < (3, 12): - expected += """ print(f"!######{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}######!")""" - else: - expected += """ print(f'!######{{test_module_name}}:{{(test_class_name + '.' if test_class_name else '')}}{{test_name}}:{{function_name}}:{{loop_index}}:{{invocation_id}}######!')""" - expected += """ - exception = None - gc.disable() - try: - counter = time.perf_counter_ns() - return_value = wrapped(*args, **kwargs) - codeflash_duration = time.perf_counter_ns() - counter - except Exception as e: - codeflash_duration = time.perf_counter_ns() - counter - exception = e - gc.enable() - pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value) - codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call')) - codeflash_con.commit() - if exception: - raise exception - return return_value - + + codeflash_wrap_string + + """ def test_code_replacement10() -> None: codeflash_loop_index = int(os.environ['CODEFLASH_LOOP_INDEX']) codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] @@ -2882,6 +2958,7 @@ def test_code_replacement10() -> None: assert code_context.testgen_context_code == get_code_output codeflash_con.close() """ + ) with tempfile.NamedTemporaryFile(mode="w") as f: f.write(code) @@ -2899,7 +2976,7 @@ def test_code_replacement10() -> None: ) os.chdir(original_cwd) assert success - assert new_test == expected.format( + assert new_test.replace('"', "'") == expected.replace('"', "'").format( module_path=Path(f.name).name, tmp_dir_path=get_run_tmp_file(Path("test_return_values")) ) diff --git a/tests/test_instrumentation_run_results_aiservice.py b/tests/test_instrumentation_run_results_aiservice.py index c1a759681..78d9973f1 100644 --- a/tests/test_instrumentation_run_results_aiservice.py +++ b/tests/test_instrumentation_run_results_aiservice.py @@ -6,7 +6,6 @@ from pathlib import Path import isort - from code_to_optimize.bubble_sort_method import BubbleSorter from codeflash.code_utils.code_utils import get_run_tmp_file from codeflash.discovery.functions_to_optimize import FunctionToOptimize @@ -16,17 +15,18 @@ from codeflash.verification.instrument_codeflash_capture import instrument_codeflash_capture # Used by aiservice instrumentation -behavior_logging_code = """from __future__ import annotations +behavior_logging_code = """ +from __future__ import annotations import gc import inspect import os import time +import dill as pickle from pathlib import Path from typing import Any, Callable, Optional -import dill as pickle def codeflash_wrap( wrapped: Callable[..., Any], @@ -48,8 +48,9 @@ def codeflash_wrap( codeflash_wrap.index[test_id] = 0 codeflash_test_index = codeflash_wrap.index[test_id] invocation_id = f"{line_id}_{codeflash_test_index}" + test_stdout_tag = f"{test_module_name}:{(test_class_name + '.' if test_class_name else '')}{test_name}:{function_name}:{loop_index}:{invocation_id}" print( - f"!######{test_module_name}:{(test_class_name + '.' if test_class_name else '')}{test_name}:{function_name}:{loop_index}:{invocation_id}######!" + f"!$######{test_stdout_tag}######$!" ) exception = None gc.disable() @@ -61,6 +62,7 @@ def codeflash_wrap( codeflash_duration = time.perf_counter_ns() - counter exception = e gc.enable() + print(f"!######{test_stdout_tag}######!") iteration = os.environ["CODEFLASH_TEST_ITERATION"] with Path( "{codeflash_run_tmp_dir_client_side}", f"test_return_values_{iteration}.bin" @@ -178,7 +180,7 @@ def test_single_element_list(): testing_time=0.1, ) assert test_results[0].id.function_getting_tested == "sorter" - assert test_results[0].stdout == "codeflash stdout : BubbleSorter.sorter() called" + assert test_results[0].stdout == "codeflash stdout : BubbleSorter.sorter() called\n" assert test_results[0].id.test_function_name == "test_single_element_list" assert test_results[0].did_pass assert test_results[0].return_value[1]["arr"] == [42] @@ -222,7 +224,7 @@ def sorter(self, arr): test_results, test_results_mutated_attr ) # Without codeflash capture, the init state was not verified, and the results are verified as correct even with the attribute mutated - assert test_results_mutated_attr[0].stdout == "codeflash stdout : BubbleSorter.sorter() called" + assert test_results_mutated_attr[0].stdout == "codeflash stdout : BubbleSorter.sorter() called\n" finally: fto_path.write_text(original_code, "utf-8") test_path.unlink(missing_ok=True) @@ -319,13 +321,13 @@ def test_single_element_list(): testing_time=0.1, ) # Verify instance_state result, which checks instance state right after __init__, using codeflash_capture + + # Verify function_to_optimize result assert test_results[0].id.function_getting_tested == "BubbleSorter.__init__" assert test_results[0].id.test_function_name == "test_single_element_list" assert test_results[0].did_pass assert test_results[0].return_value[0] == {"x": 0} - assert test_results[0].stdout == "codeflash stdout : BubbleSorter.sorter() called" - - # Verify function_to_optimize result + assert test_results[0].stdout == "" assert test_results[1].id.function_getting_tested == "sorter" assert test_results[1].id.test_function_name == "test_single_element_list" assert test_results[1].did_pass @@ -336,6 +338,12 @@ def test_single_element_list(): # Check function return value assert test_results[1].return_value[2] == [1, 2, 3] + assert ( + test_results[1].stdout + == """codeflash stdout : BubbleSorter.sorter() called +""" + ) + # Replace with optimized code that mutated instance attribute optimized_code_mutated_attr = """ import sys @@ -393,7 +401,7 @@ def sorter(self, arr): assert test_results_mutated_attr[0].id.function_getting_tested == "BubbleSorter.__init__" assert test_results_mutated_attr[0].return_value[0] == {"x": 1} assert test_results_mutated_attr[0].verification_type == VerificationType.INIT_STATE_FTO - assert test_results_mutated_attr[0].stdout == "codeflash stdout : BubbleSorter.sorter() called" + assert test_results_mutated_attr[0].stdout == "" assert not compare_test_results( test_results, test_results_mutated_attr ) # The test should fail because the instance attribute was mutated @@ -445,7 +453,7 @@ def sorter(self, arr): assert test_results_new_attr[0].id.function_getting_tested == "BubbleSorter.__init__" assert test_results_new_attr[0].return_value[0] == {"x": 0, "y": 2} assert test_results_new_attr[0].verification_type == VerificationType.INIT_STATE_FTO - assert test_results_new_attr[0].stdout == "codeflash stdout : BubbleSorter.sorter() called" + assert test_results_new_attr[0].stdout == "" # assert test_results_new_attr[1].return_value[1]["self"].x == 0 TODO: add self as input # assert test_results_new_attr[1].return_value[1]["self"].y == 2 TODO: add self as input assert compare_test_results( diff --git a/tests/test_test_runner.py b/tests/test_test_runner.py index 0e80b76e0..5dc6df678 100644 --- a/tests/test_test_runner.py +++ b/tests/test_test_runner.py @@ -1,5 +1,3 @@ -import re - import os import tempfile from pathlib import Path @@ -145,5 +143,5 @@ def test_sort(): test_xml_file_path=result_file, test_files=test_files, test_config=config, run_result=process ) match = ImportErrorPattern.search(process.stdout).group() - assert match=="ModuleNotFoundError: No module named 'torch'" + assert match == "ModuleNotFoundError: No module named 'torch'" result_file.unlink(missing_ok=True)