From f4a4a4440bad3f6e81e0bbba8dbccc42968b9fc3 Mon Sep 17 00:00:00 2001 From: Maciej Perkowski Date: Mon, 24 Jul 2023 13:24:51 +0200 Subject: [PATCH 1/5] scripts: Add --no-detailed-test-id arg to test_plan.py script An option --no-detailed-test-id was added to twister to help align names for test outside of zephyr tree. This commit add this arg to test_plan.py script which is then propagated to twister. Signed-off-by: Maciej Perkowski --- scripts/ci/test_plan.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/ci/test_plan.py b/scripts/ci/test_plan.py index 8f2f24b08256b..d6ea6fe26d243 100755 --- a/scripts/ci/test_plan.py +++ b/scripts/ci/test_plan.py @@ -86,7 +86,7 @@ def __repr__(self): return "".format(self.name) class Filters: - def __init__(self, modified_files, pull_request=False, platforms=[]): + def __init__(self, modified_files, pull_request=False, platforms=[], detailed_test_id=True): self.modified_files = modified_files self.twister_options = [] self.full_twister = False @@ -95,6 +95,7 @@ def __init__(self, modified_files, pull_request=False, platforms=[]): self.pull_request = pull_request self.platforms = platforms self.default_run = False + self.detailed_test_id = detailed_test_id def process(self): self.find_modules() @@ -112,6 +113,8 @@ def process(self): def get_plan(self, options, integration=False): fname = "_test_plan_partial.json" cmd = ["scripts/twister", "-c"] + options + ["--save-tests", fname ] + if not self.detailed_test_id: + cmd += ["--no-detailed-test-id"] if integration: cmd.append("--integration") @@ -353,6 +356,13 @@ def parse_args(): help="Number of tests per builder") parser.add_argument('-n', '--default-matrix', default=10, type=int, help="Number of tests per builder") + parser.add_argument('--detailed-test-id', action='store_true', + help="Include paths to tests' locations in tests' names.") + parser.add_argument("--no-detailed-test-id", dest='detailed_test_id', action="store_false", + help="Don't put paths into tests' names.") + + # Include paths in names by default. + parser.set_defaults(detailed_test_id=True) return parser.parse_args() @@ -375,8 +385,7 @@ def parse_args(): print("\n".join(files)) print("=========") - - f = Filters(files, args.pull_request, args.platform) + f = Filters(files, args.pull_request, args.platform, args.detailed_test_id) f.process() # remove dupes and filtered cases From 8253a3437fd86c64f2e3d31749539f0f8aea35d7 Mon Sep 17 00:00:00 2001 From: Maciej Perkowski Date: Mon, 31 Jul 2023 13:05:35 +0200 Subject: [PATCH 2/5] scripts: Allow test_plan.py to work with other than zephyr repos The test_plan.py script has a path to repository to be scanned for changes hard coded to zephyr. This patch separates zephyr path from such repository's path and adds an arg to pass repo to scan Signed-off-by: Maciej Perkowski --- scripts/ci/test_plan.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/ci/test_plan.py b/scripts/ci/test_plan.py index d6ea6fe26d243..3b0e755923af9 100755 --- a/scripts/ci/test_plan.py +++ b/scripts/ci/test_plan.py @@ -19,10 +19,10 @@ if "ZEPHYR_BASE" not in os.environ: exit("$ZEPHYR_BASE environment variable undefined.") -repository_path = Path(os.environ['ZEPHYR_BASE']) +zephyr_base = Path(os.environ['ZEPHYR_BASE']) logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) -sys.path.append(os.path.join(repository_path, 'scripts')) +sys.path.append(os.path.join(zephyr_base, 'scripts')) import list_boards def _get_match_fn(globs, regexes): @@ -112,7 +112,7 @@ def process(self): def get_plan(self, options, integration=False): fname = "_test_plan_partial.json" - cmd = ["scripts/twister", "-c"] + options + ["--save-tests", fname ] + cmd = [f"{zephyr_base}/scripts/twister", "-c"] + options + ["--save-tests", fname ] if not self.detailed_test_id: cmd += ["--no-detailed-test-id"] if integration: @@ -131,7 +131,7 @@ def find_modules(self): if 'west.yml' in self.modified_files: print(f"Manifest file 'west.yml' changed") print("=========") - old_manifest_content = repo.git.show(f"{args.commits[:-2]}:west.yml") + old_manifest_content = repo_to_scan.git.show(f"{args.commits[:-2]}:west.yml") with open("west_old.yml", "w") as manifest: manifest.write(old_manifest_content) old_manifest = Manifest.from_file("west_old.yml") @@ -212,8 +212,12 @@ def find_boards(self): if p and p.groups(): boards.add(p.group(1)) - # Limit search to $ZEPHYR_BASE since this is where the changed files are - lb_args = argparse.Namespace(**{ 'arch_roots': [repository_path], 'board_roots': [repository_path] }) + roots = [zephyr_base] + if repository_path != zephyr_base: + roots.append(repository_path) + + # Look for boards in monitored repositories + lb_args = argparse.Namespace(**{ 'arch_roots': roots, 'board_roots': roots}) known_boards = list_boards.find_boards(lb_args) for b in boards: name_re = re.compile(b) @@ -268,7 +272,7 @@ def find_tests(self): def find_tags(self): - tag_cfg_file = os.path.join(repository_path, 'scripts', 'ci', 'tags.yaml') + tag_cfg_file = os.path.join(zephyr_base, 'scripts', 'ci', 'tags.yaml') with open(tag_cfg_file, 'r') as ymlfile: tags_config = yaml.safe_load(ymlfile) @@ -360,6 +364,8 @@ def parse_args(): help="Include paths to tests' locations in tests' names.") parser.add_argument("--no-detailed-test-id", dest='detailed_test_id', action="store_false", help="Don't put paths into tests' names.") + parser.add_argument('-r', '--repo-to-scan', default=None, + help="Repo to scan") # Include paths in names by default. parser.set_defaults(detailed_test_id=True) @@ -372,9 +378,12 @@ def parse_args(): args = parse_args() files = [] errors = 0 + repository_path = zephyr_base + if args.repo_to_scan: + repository_path = Path(args.repo_to_scan) if args.commits: - repo = Repo(repository_path) - commit = repo.git.diff("--name-only", args.commits) + repo_to_scan = Repo(repository_path) + commit = repo_to_scan.git.diff("--name-only", args.commits) files = commit.split("\n") elif args.modified_files: with open(args.modified_files, "r") as fp: From cf5a9d93c750f5fe386e53177c5032f07348084c Mon Sep 17 00:00:00 2001 From: Maciej Perkowski Date: Tue, 25 Jul 2023 16:45:55 +0200 Subject: [PATCH 3/5] scripts: Allow using alternative ignore-patters in test_plan.py Expand test_plan.py args with --ignore-path. This allows to provide an alternative lists of patterns for the script. Signed-off-by: Maciej Perkowski --- scripts/ci/test_plan.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/ci/test_plan.py b/scripts/ci/test_plan.py index 3b0e755923af9..f93eeefb2d4ad 100755 --- a/scripts/ci/test_plan.py +++ b/scripts/ci/test_plan.py @@ -86,7 +86,7 @@ def __repr__(self): return "".format(self.name) class Filters: - def __init__(self, modified_files, pull_request=False, platforms=[], detailed_test_id=True): + def __init__(self, modified_files, pull_request=False, platforms=[], detailed_test_id=True, ignore_path=None): self.modified_files = modified_files self.twister_options = [] self.full_twister = False @@ -96,6 +96,9 @@ def __init__(self, modified_files, pull_request=False, platforms=[], detailed_te self.platforms = platforms self.default_run = False self.detailed_test_id = detailed_test_id + self.ignore_path = f"{zephyr_base}/scripts/ci/twister_ignore.txt" + if ignore_path: + self.ignore_path = ignore_path def process(self): self.find_modules() @@ -310,7 +313,7 @@ def find_tags(self): logging.info(f'Potential tag based filters: {exclude_tags}') def find_excludes(self, skip=[]): - with open("scripts/ci/twister_ignore.txt", "r") as twister_ignore: + with open(self.ignore_path, "r") as twister_ignore: ignores = twister_ignore.read().splitlines() ignores = filter(lambda x: not x.startswith("#"), ignores) @@ -366,6 +369,8 @@ def parse_args(): help="Don't put paths into tests' names.") parser.add_argument('-r', '--repo-to-scan', default=None, help="Repo to scan") + parser.add_argument('--ignore-path', default=None, + help="Path to a text file with patterns of files to be matched against changed files") # Include paths in names by default. parser.set_defaults(detailed_test_id=True) @@ -394,7 +399,7 @@ def parse_args(): print("\n".join(files)) print("=========") - f = Filters(files, args.pull_request, args.platform, args.detailed_test_id) + f = Filters(files, args.pull_request, args.platform, args.detailed_test_id, args.ignore_path) f.process() # remove dupes and filtered cases From 5ce17ac135bbe79aacb3aa155009cb0c0aa38f99 Mon Sep 17 00:00:00 2001 From: Maciej Perkowski Date: Wed, 2 Aug 2023 14:36:03 +0200 Subject: [PATCH 4/5] scripts: Add arg to set_plan.py for alternative tag relation list Add --alt-tag arg for test_plan.py script. User can use it and point to an alternative file with tag-directories relations. If so, such file will be used instead of the default one. Signed-off-by: Maciej Perkowski --- scripts/ci/test_plan.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/ci/test_plan.py b/scripts/ci/test_plan.py index f93eeefb2d4ad..df15749665a90 100755 --- a/scripts/ci/test_plan.py +++ b/scripts/ci/test_plan.py @@ -86,7 +86,7 @@ def __repr__(self): return "".format(self.name) class Filters: - def __init__(self, modified_files, pull_request=False, platforms=[], detailed_test_id=True, ignore_path=None): + def __init__(self, modified_files, pull_request=False, platforms=[], detailed_test_id=True, ignore_path=None, alt_tags=None): self.modified_files = modified_files self.twister_options = [] self.full_twister = False @@ -96,6 +96,9 @@ def __init__(self, modified_files, pull_request=False, platforms=[], detailed_te self.platforms = platforms self.default_run = False self.detailed_test_id = detailed_test_id + self.tag_cfg_file = os.path.join(zephyr_base, 'scripts', 'ci', 'tags.yaml') + if alt_tags: + self.tag_cfg_file = alt_tags self.ignore_path = f"{zephyr_base}/scripts/ci/twister_ignore.txt" if ignore_path: self.ignore_path = ignore_path @@ -275,8 +278,7 @@ def find_tests(self): def find_tags(self): - tag_cfg_file = os.path.join(zephyr_base, 'scripts', 'ci', 'tags.yaml') - with open(tag_cfg_file, 'r') as ymlfile: + with open(self.tag_cfg_file, 'r') as ymlfile: tags_config = yaml.safe_load(ymlfile) tags = {} @@ -371,6 +373,8 @@ def parse_args(): help="Repo to scan") parser.add_argument('--ignore-path', default=None, help="Path to a text file with patterns of files to be matched against changed files") + parser.add_argument('--alt-tags', default=None, + help="Path to a file describing relations between directories and tags") # Include paths in names by default. parser.set_defaults(detailed_test_id=True) @@ -399,7 +403,7 @@ def parse_args(): print("\n".join(files)) print("=========") - f = Filters(files, args.pull_request, args.platform, args.detailed_test_id, args.ignore_path) + f = Filters(files, args.pull_request, args.platform, args.detailed_test_id, args.ignore_path, args.alt_tags) f.process() # remove dupes and filtered cases From d1adc373a00f2dc45b73eb6473c08f427599c528 Mon Sep 17 00:00:00 2001 From: Maciej Perkowski Date: Wed, 2 Aug 2023 16:43:54 +0200 Subject: [PATCH 5/5] scripts: Add arg to test_plan.py for alternative test locations The arg --testsuite-root was copied from twister. When it is used for test_plan.py it will be propagated to twister calls. This allows to make alternative test locations (e.g. from another repo) to work with test_plan.py Signed-off-by: Maciej Perkowski --- scripts/ci/test_plan.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/ci/test_plan.py b/scripts/ci/test_plan.py index df15749665a90..a5df4c7f8b11c 100755 --- a/scripts/ci/test_plan.py +++ b/scripts/ci/test_plan.py @@ -86,8 +86,9 @@ def __repr__(self): return "".format(self.name) class Filters: - def __init__(self, modified_files, pull_request=False, platforms=[], detailed_test_id=True, ignore_path=None, alt_tags=None): + def __init__(self, modified_files, pull_request=False, platforms=[], detailed_test_id=True, ignore_path=None, alt_tags=None, testsuite_root=None): self.modified_files = modified_files + self.testsuite_root = testsuite_root self.twister_options = [] self.full_twister = False self.all_tests = [] @@ -116,11 +117,14 @@ def process(self): else: self.find_excludes() - def get_plan(self, options, integration=False): + def get_plan(self, options, integration=False, use_testsuite_root=True): fname = "_test_plan_partial.json" cmd = [f"{zephyr_base}/scripts/twister", "-c"] + options + ["--save-tests", fname ] if not self.detailed_test_id: cmd += ["--no-detailed-test-id"] + if self.testsuite_root and use_testsuite_root: + for root in self.testsuite_root: + cmd+=["-T", root] if integration: cmd.append("--integration") @@ -274,7 +278,7 @@ def find_tests(self): _options.extend(["-p", platform]) else: _options.append("--all") - self.get_plan(_options) + self.get_plan(_options, use_testsuite_root=False) def find_tags(self): @@ -375,6 +379,12 @@ def parse_args(): help="Path to a text file with patterns of files to be matched against changed files") parser.add_argument('--alt-tags', default=None, help="Path to a file describing relations between directories and tags") + parser.add_argument( + "-T", "--testsuite-root", action="append", default=[], + help="Base directory to recursively search for test cases. All " + "testcase.yaml files under here will be processed. May be " + "called multiple times. Defaults to the 'samples/' and " + "'tests/' directories at the base of the Zephyr tree.") # Include paths in names by default. parser.set_defaults(detailed_test_id=True) @@ -403,7 +413,7 @@ def parse_args(): print("\n".join(files)) print("=========") - f = Filters(files, args.pull_request, args.platform, args.detailed_test_id, args.ignore_path, args.alt_tags) + f = Filters(files, args.pull_request, args.platform, args.detailed_test_id, args.ignore_path, args.alt_tags, args.testsuite_root) f.process() # remove dupes and filtered cases