diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8c75369a128..61752526aa2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1460,6 +1460,11 @@ repos: entry: ./scripts/format_test_list.py language: script files: tests/integration/test_lists/.*\.txt$ + - id: waive list check + name: Checks for duplicated test items in waives.txt + entry: ./scripts/check_test_list.py --check-duplicate-waives + language: script + pass_filenames: false - id: DCO check name: Checks the commit message for a developer certificate of origin signature entry: ./scripts/dco_check.py diff --git a/scripts/check_test_list.py b/scripts/check_test_list.py old mode 100644 new mode 100755 index 080eefb8341..c7b5357d25f --- a/scripts/check_test_list.py +++ b/scripts/check_test_list.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """ This script is used to verify test lists for L0, QA, and waives file. @@ -110,14 +111,14 @@ def verify_qa_test_lists(llm_src): f.write(f"{cleaned_line}\n") -def verify_waive_list(llm_src, args): +def check_waive_duplicates(llm_src): + """Check for duplicate entries in waives.txt and write report.""" waives_list_path = f"{llm_src}/tests/integration/test_lists/waives.txt" dup_cases_record = f"{llm_src}/dup_cases.txt" - non_existent_cases_record = f"{llm_src}/nonexits_cases.json" - # Remove prefix and markers in wavies.txt - dedup_lines = { - } # Track all occurrences: processed_line -> [(line_no, original_line), ...] - processed_lines = set() + + # Track all occurrences: processed_line -> [(line_no, original_line), ...] + dedup_lines = {} + with open(waives_list_path, "r") as f: lines = f.readlines() @@ -128,10 +129,6 @@ def verify_waive_list(llm_src, args): if not line: continue - # Skip Perf tests due to they are dynamically generated - if "perf/test_perf.py" in line: - continue - # Check for SKIP marker in waives.txt and split by the first occurrence line = line.split(" SKIP", 1)[0].strip() @@ -141,6 +138,41 @@ def verify_waive_list(llm_src, args): else: dedup_lines[line] = [(line_no, original_line)] + # Write duplicate report after processing all lines + for processed_line, occurrences in dedup_lines.items(): + if len(occurrences) > 1: + with open(dup_cases_record, "a") as f: + f.write( + f"Duplicate waive records found for '{processed_line}' ({len(occurrences)} occurrences):\n" + ) + for i, (line_no, original_line) in enumerate(occurrences, 1): + f.write( + f" Occurrence {i} at line {line_no}: '{original_line}'\n" + ) + f.write(f"\n") + + +def verify_waive_list(llm_src, args): + waives_list_path = f"{llm_src}/tests/integration/test_lists/waives.txt" + non_existent_cases_record = f"{llm_src}/nonexits_cases.json" + + processed_lines = set() + with open(waives_list_path, "r") as f: + lines = f.readlines() + + for line in lines: + line = line.strip() + + if not line: + continue + + # Skip Perf tests due to they are dynamically generated + if "perf/test_perf.py" in line: + continue + + # Check for SKIP marker in waives.txt and split by the first occurrence + line = line.split(" SKIP", 1)[0].strip() + # If the line starts with 'full:', process it if line.startswith("full:"): line = line.split("/", 1)[1].lstrip("/") @@ -173,19 +205,6 @@ def verify_waive_list(llm_src, args): processed_lines.add(line) - # Write duplicate report after processing all lines - for processed_line, occurrences in dedup_lines.items(): - if len(occurrences) > 1: - with open(dup_cases_record, "a") as f: - f.write( - f"Duplicate waive records found for '{processed_line}' ({len(occurrences)} occurrences):\n" - ) - for i, (line_no, original_line) in enumerate(occurrences, 1): - f.write( - f" Occurrence {i} at line {line_no}: '{original_line}'\n" - ) - f.write(f"\n") - # Write the processed lines to a tmp file tmp_waives_file = f"{llm_src}/processed_waive_list.txt" with open(tmp_waives_file, "w") as f: @@ -210,11 +229,19 @@ def main(): parser.add_argument("--waive", action="store_true", help="Enable test list verification for waive file.") + parser.add_argument( + "--check-duplicate-waives", + action="store_true", + help="Enable duplicate check in waives.txt (fails if duplicates found)." + ) args = parser.parse_args() script_dir = os.path.dirname(os.path.realpath(__file__)) llm_src = os.path.abspath(os.path.join(script_dir, "../")) - install_python_dependencies(llm_src) + # Only skip installing dependencies if ONLY --check-duplicates is used + if args.l0 or args.qa or args.waive: + install_python_dependencies(llm_src) + pass_flag = True # Verify L0 test lists if args.l0: @@ -243,6 +270,12 @@ def main(): print("-----------Skipping waive list verification.-----------", flush=True) + # Check for duplicates in waives.txt if requested + if args.check_duplicate_waives: + print("-----------Checking for duplicates in waives.txt...-----------", + flush=True) + check_waive_duplicates(llm_src) + invalid_json_file = os.path.join(llm_src, "invalid_tests.json") if os.path.isfile(invalid_json_file) and os.path.getsize( invalid_json_file) > 0: @@ -261,7 +294,8 @@ def main(): print( "Duplicate test names found in waives.txt, please delete one or combine them first!!!\n" ) - # pass_flag = False + if args.check_duplicate_waives: + pass_flag = False non_existent_cases_file = os.path.join(llm_src, "nonexits_cases.json") if os.path.isfile(non_existent_cases_file) and os.path.getsize( diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 3fd599358f3..6fa27c3171d 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -395,8 +395,6 @@ accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[True] SKIP disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_bf16_empty_batch[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/5601682) accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False] SKIP (https://nvbugs/5655584) accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_chunked_prefill SKIP (https://nvbugs/5608930) -accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[False] SKIP (https://nvbugspro.nvidia.com/bug/5651854) -test_e2e.py::test_ptp_quickstart_multimodal_chunked_prefill[phi4-multimodal-instruct-fp4-multimodals/Phi-4-multimodal-instruct-FP4-0.8-image] SKIP (https://nvbugs/5568836) test_e2e.py::test_ptp_quickstart_multimodal_chunked_prefill[phi4-multimodal-instruct-fp4-multimodals/Phi-4-multimodal-instruct-FP4-0.8-image] SKIP (https://nvbugs/5568836) test_e2e.py::test_ptp_quickstart_multimodal_kv_cache_reuse[phi4-multimodal-instruct-fp4-multimodals/Phi-4-multimodal-instruct-FP4-0.8-image] SKIP (https://nvbugs/5568836) test_e2e.py::test_ptp_quickstart_multimodal_multiturn[phi4-multimodal-instruct-fp4-multimodals/Phi-4-multimodal-instruct-FP4] SKIP (https://nvbugs/5568836)