diff --git a/.github/labeler.yml b/.github/labeler.yml index 25380e0b6db8a..b49c3b54bf094 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -124,7 +124,7 @@ - "scripts/kconfig/**/*" - "Kconfig" - "Kconfig.zephyr" -"area: Sanitycheck": +"area: Twister": - "scripts/twister" - "scripts/pylib/twister/**/*" "area: Modules": diff --git a/.github/workflows/sanitycheck_tests.yml b/.github/workflows/twister_tests.yml similarity index 87% rename from .github/workflows/sanitycheck_tests.yml rename to .github/workflows/twister_tests.yml index 62ead116d5083..77be06806f001 100644 --- a/.github/workflows/sanitycheck_tests.yml +++ b/.github/workflows/twister_tests.yml @@ -1,7 +1,7 @@ # Copyright (c) 2020 Intel Corporation. # SPDX-License-Identifier: Apache-2.0 -name: Zephyr Sanitycheck TestSuite +name: Twsiter TestSuite on: push: @@ -9,13 +9,13 @@ on: - 'scripts/pylib/twister/**' - 'scripts/twister' - 'scripts/tests/twister/**' - - '.github/workflows/sanitycheck_tests.yml' + - '.github/workflows/twister_tests.yml' pull_request: paths: - 'scripts/pylib/twister/**' - 'scripts/twister' - 'scripts/tests/twister/**' - - '.github/workflows/sanitycheck_tests.yml' + - '.github/workflows/twister_tests.yml' jobs: build: @@ -47,5 +47,5 @@ jobs: ZEPHYR_BASE: ./ ZEPHYR_TOOLCHAIN_VARIANT: zephyr run: | - echo "Run Sanitycheck tests" + echo "Run twister tests" PYTHONPATH=./scripts/tests pytest ./scripts/tests/twister diff --git a/doc/contribute/index.rst b/doc/contribute/index.rst index f27373301d937..b41fde1f520c2 100644 --- a/doc/contribute/index.rst +++ b/doc/contribute/index.rst @@ -221,7 +221,7 @@ every Pull Request (PR) in order to verify several aspects of the PR: * Git commit formatting * Coding Style -* Sanity Check builds for multiple architectures and boards +* Twister builds for multiple architectures and boards * Documentation build to verify any doc changes CI is run on the ``shippable`` cloud service and it uses the same tools diff --git a/doc/contribute/project_roles.rst b/doc/contribute/project_roles.rst index ae697437295f1..5cba6fc5586d9 100644 --- a/doc/contribute/project_roles.rst +++ b/doc/contribute/project_roles.rst @@ -228,7 +228,7 @@ Merge Criteria * License * Checkpatch (Coding Style) * Pylint - * Sanitycheck + Other Unit tests + * Integration Tests (Via twister) on emulation/simulation platforms * Simulated Bluetooth Tests * Planned diff --git a/doc/development_process/dev_env_and_tools.rst b/doc/development_process/dev_env_and_tools.rst index 08c993160ab09..1b2cc06a0828f 100644 --- a/doc/development_process/dev_env_and_tools.rst +++ b/doc/development_process/dev_env_and_tools.rst @@ -198,9 +198,9 @@ Closing Stale Issues and Pull Requests Continuous Integration *********************** -All changes submitted to GitHub are subject to sanity tests that are run on +All changes submitted to GitHub are subject to tests that are run on emulated platforms and architectures to identify breakage and regressions that -can be immediately identified. Sanity testing additionally performs build tests +can be immediately identified. Testing using Twister additionally performs build tests of all boards and platforms. Documentation changes are also verified through review and build testing to verify doc generation will be successful. diff --git a/doc/guides/coverage.rst b/doc/guides/coverage.rst index c54ef0be3c7f1..9c97dc994f927 100644 --- a/doc/guides/coverage.rst +++ b/doc/guides/coverage.rst @@ -129,8 +129,8 @@ You may postprocess these with your preferred tools. For example: Alternatively, you can use gcovr (at least version 4.2). -Sanitycheck coverage reports -**************************** +Coverage reports using Twister +****************************** Zephyr's :ref:`twister script ` can automatically generate a coverage report from the tests which were executed. diff --git a/doc/guides/modules.rst b/doc/guides/modules.rst index 3ff16566b5a26..cac1aa307af60 100644 --- a/doc/guides/modules.rst +++ b/doc/guides/modules.rst @@ -536,8 +536,8 @@ requires the following folder structure: -Sanitycheck -=========== +Twister (Test Runner) +===================== To execute both tests and samples available in modules, the Zephyr test runner (twister) should be pointed to the directories containing those samples and diff --git a/doc/guides/test/ztest.rst b/doc/guides/test/ztest.rst index 3544c33dd3cfc..4b917ad30ad8d 100644 --- a/doc/guides/test/ztest.rst +++ b/doc/guides/test/ztest.rst @@ -99,7 +99,7 @@ feature. The ``twister`` script can parse the testcases in all test projects or a subset of them, and can generate reports on a granular level, i.e. if cases have passed or failed or if they were blocked or skipped. -Sanitycheck parses the source files looking for test case names, so you +Twister parses the source files looking for test case names, so you can list all kernel test cases, for example, by entering:: twister --list-tests -T tests/kernel diff --git a/scripts/pylib/twister/scl.py b/scripts/pylib/twister/scl.py index 78ca9d68a3532..7c1a430e83d9e 100644 --- a/scripts/pylib/twister/scl.py +++ b/scripts/pylib/twister/scl.py @@ -1,7 +1,7 @@ #! /usr/bin/python # # SPDX-License-Identifier: Apache-2.0 -# Zephyr's Sanity Check library +# Zephyr's Twister library # # pylint: disable=unused-import # diff --git a/scripts/pylib/twister/twisterlib.py b/scripts/pylib/twister/twisterlib.py index 0477780eb9140..127a5e2603a34 100644 --- a/scripts/pylib/twister/twisterlib.py +++ b/scripts/pylib/twister/twisterlib.py @@ -340,24 +340,24 @@ def __iter__(self): return iter(self._entries.values()) -class SanityCheckException(Exception): +class TwisterException(Exception): pass -class SanityRuntimeError(SanityCheckException): +class TwisterRuntimeError(TwisterException): pass -class ConfigurationError(SanityCheckException): +class ConfigurationError(TwisterException): def __init__(self, cfile, message): - SanityCheckException.__init__(self, cfile + ": " + message) + TwisterException.__init__(self, cfile + ": " + message) -class BuildError(SanityCheckException): +class BuildError(TwisterException): pass -class ExecutionError(SanityCheckException): +class ExecutionError(TwisterException): pass @@ -1219,7 +1219,7 @@ def __init__(self, filename, extra_sections): try: if magic != b'\x7fELF': - raise SanityRuntimeError("%s is not an ELF binary" % filename) + raise TwisterRuntimeError("%s is not an ELF binary" % filename) except Exception as e: print(str(e)) sys.exit(2) @@ -1234,7 +1234,7 @@ def __init__(self, filename, extra_sections): "utf-8").strip() try: if is_xip_output.endswith("no symbols"): - raise SanityRuntimeError("%s has no symbol information" % filename) + raise TwisterRuntimeError("%s has no symbol information" % filename) except Exception as e: print(str(e)) sys.exit(2) @@ -1330,12 +1330,12 @@ def _calculate_sizes(self): -class SanityConfigParser: +class TwisterConfigParser: """Class to read test case files with semantic checking """ def __init__(self, filename, schema): - """Instantiate a new SanityConfigParser object + """Instantiate a new TwisterConfigParser object @param filename Source .yaml file to read """ @@ -1496,7 +1496,7 @@ def __init__(self): self.filter_data = dict() def load(self, platform_file): - scp = SanityConfigParser(platform_file, self.platform_schema) + scp = TwisterConfigParser(platform_file, self.platform_schema) scp.load() data = scp.data @@ -1605,7 +1605,7 @@ def get_unique(testcase_root, workdir, name): unique = os.path.normpath(os.path.join(relative_tc_root, workdir, name)) check = name.split(".") if len(check) < 2: - raise SanityCheckException(f"""bad test name '{name}' in {testcase_root}/{workdir}. \ + raise TwisterException(f"""bad test name '{name}' in {testcase_root}/{workdir}. \ Tests should reference the category and subsystem with a dot as a separator. """ ) @@ -1683,7 +1683,7 @@ def scan_path(self, path): _subcases, warnings = self.scan_file(filename) if warnings: logger.error("%s: %s" % (filename, warnings)) - raise SanityRuntimeError("%s: %s" % (filename, warnings)) + raise TwisterRuntimeError("%s: %s" % (filename, warnings)) if _subcases: subcases += _subcases except ValueError as e: @@ -2812,7 +2812,7 @@ def get_toolchain(): try: if not toolchain: - raise SanityRuntimeError("E: Variable ZEPHYR_TOOLCHAIN_VARIANT is not defined") + raise TwisterRuntimeError("E: Variable ZEPHYR_TOOLCHAIN_VARIANT is not defined") except Exception as e: print(str(e)) sys.exit(2) @@ -2839,7 +2839,7 @@ def add_testcases(self, testcase_filter=[]): tc_path = os.path.join(dirpath, filename) try: - parsed_data = SanityConfigParser(tc_path, self.tc_schema) + parsed_data = TwisterConfigParser(tc_path, self.tc_schema) parsed_data.load() tc_path = os.path.dirname(tc_path) @@ -3239,7 +3239,7 @@ def discard_report(self, filename): try: if not self.discards: - raise SanityRuntimeError("apply_filters() hasn't been run!") + raise TwisterRuntimeError("apply_filters() hasn't been run!") except Exception as e: logger.error(str(e)) sys.exit(2) diff --git a/scripts/tests/twister/README.md b/scripts/tests/twister/README.md index 0f5e677d2d564..c19d70d9a60ae 100644 --- a/scripts/tests/twister/README.md +++ b/scripts/tests/twister/README.md @@ -1,8 +1,8 @@ -# Sanitycheck Testing +# Twister Testing Running the tests require the environment variable ZEPHYR_BASE to be set. -Sanitycheck Testsuite are located in $ZEPHYR_BASE/scripts/tests directory with all the data files in $ZEPHYR_BASE/scripts/test_data directory. +Twister Testsuite are located in $ZEPHYR_BASE/scripts/tests directory with all the data files in $ZEPHYR_BASE/scripts/test_data directory. ## Dependencies @@ -20,7 +20,7 @@ The testcases can be executed from the root directory using pytest $ZEPHYR_BASE/scripts/tests/twister ``` -## Sanitycheck Coverage +## Twister Coverage The coverage for all the tests can be run using the command below. This will collect all the tests available. diff --git a/scripts/tests/twister/test_testinstance.py b/scripts/tests/twister/test_testinstance.py index e2aaff2b0b698..15c9a2e9eb540 100644 --- a/scripts/tests/twister/test_testinstance.py +++ b/scripts/tests/twister/test_testinstance.py @@ -13,7 +13,7 @@ ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister")) -from twisterlib import TestInstance, BuildError, TestCase, SanityCheckException +from twisterlib import TestInstance, BuildError, TestCase, TwisterException TESTDATA_1 = [ @@ -108,7 +108,7 @@ def test_get_unique(testcase_root, workdir, name, expected): def test_get_unique_exception(testcase_root, workdir, name, exception): '''Test to check if tests reference the category and subsystem with a dot as a separator''' - with pytest.raises(SanityCheckException): + with pytest.raises(TwisterException): unique = TestCase(testcase_root, workdir, name) assert unique == exception diff --git a/scripts/tests/twister/test_testsuite_class.py b/scripts/tests/twister/test_testsuite_class.py index 8213db7705e23..37b60ae1417ed 100755 --- a/scripts/tests/twister/test_testsuite_class.py +++ b/scripts/tests/twister/test_testsuite_class.py @@ -43,7 +43,7 @@ def test_testsuite_add_testcases(class_testsuite): @pytest.mark.parametrize("board_root_dir", [("board_config_file_not_exist"), ("board_config")]) def test_add_configurations(test_data, class_testsuite, board_root_dir): - """ Testing add_configurations function of TestSuite class in Sanitycheck + """ Testing add_configurations function of TestSuite class in Twister Test : Asserting on default platforms list """ class_testsuite.board_roots = os.path.abspath(test_data + board_root_dir) @@ -56,7 +56,7 @@ def test_add_configurations(test_data, class_testsuite, board_root_dir): assert sorted(suite.default_platforms) != sorted(['demo_board_1']) def test_get_all_testcases(class_testsuite, all_testcases_dict): - """ Testing get_all_testcases function of TestSuite class in Sanitycheck """ + """ Testing get_all_testcases function of TestSuite class in Twister """ class_testsuite.testcases = all_testcases_dict expected_tests = ['sample_test.app', 'test_a.check_1.1a', 'test_a.check_1.1c', 'test_a.check_1.2a', 'test_a.check_1.2b', 'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a', 'test_a.check_1.unit_1b', 'test_a.check_2.1a', 'test_a.check_2.1c', 'test_a.check_2.2a', 'test_a.check_2.2b', 'test_a.check_2.Unit_1c', 'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b', 'test_b.check_1', 'test_b.check_2', 'test_c.check_1', 'test_c.check_2'] @@ -64,7 +64,7 @@ def test_get_all_testcases(class_testsuite, all_testcases_dict): assert sorted(class_testsuite.get_all_tests()) == sorted(expected_tests) def test_get_toolchain(class_testsuite, monkeypatch, capsys): - """ Testing get_toolchain function of TestSuite class in Sanitycheck + """ Testing get_toolchain function of TestSuite class in Twister Test 1 : Test toolchain returned by get_toolchain function is same as in the environment. Test 2 : Monkeypatch to delete the ZEPHYR_TOOLCHAIN_VARIANT env var and check if appropriate error is raised""" @@ -79,7 +79,7 @@ def test_get_toolchain(class_testsuite, monkeypatch, capsys): assert out == "E: Variable ZEPHYR_TOOLCHAIN_VARIANT is not defined\n" def test_get_platforms(class_testsuite, platforms_list): - """ Testing get_platforms function of TestSuite class in Sanitycheck """ + """ Testing get_platforms function of TestSuite class in Twister """ class_testsuite.platforms = platforms_list platform = class_testsuite.get_platform("demo_board_1") assert isinstance(platform, Platform) @@ -87,7 +87,7 @@ def test_get_platforms(class_testsuite, platforms_list): def test_load_from_file(test_data, class_testsuite, platforms_list, all_testcases_dict, caplog, tmpdir_factory): - """ Testing load_from_file function of TestSuite class in Sanitycheck """ + """ Testing load_from_file function of TestSuite class in Twister """ # Scenario 1 : Validating the error raised if file to load from doesn't exist with pytest.raises(SystemExit): class_testsuite.load_from_file(test_data +"twister_test.csv") @@ -160,7 +160,7 @@ def test_load_from_file(test_data, class_testsuite, TESTDATA_PART1) def test_apply_filters_part1(class_testsuite, all_testcases_dict, platforms_list, tc_attribute, tc_value, plat_attribute, plat_value, expected_discards): - """ Testing apply_filters function of TestSuite class in Sanitycheck + """ Testing apply_filters function of TestSuite class in Twister Part 1: Response of apply_filters function (discard dictionary) have appropriate values according to the filters """ @@ -233,7 +233,7 @@ def test_apply_filters_part1(class_testsuite, all_testcases_dict, platforms_list @pytest.mark.parametrize("extra_filter, extra_filter_value, expected_discards", TESTDATA_PART2) def test_apply_filters_part2(class_testsuite, all_testcases_dict, platforms_list, extra_filter, extra_filter_value, expected_discards): - """ Testing apply_filters function of TestSuite class in Sanitycheck + """ Testing apply_filters function of TestSuite class in Twister Part 2 : Response of apply_filters function (discard dictionary) have appropriate values according to the filters """ @@ -265,7 +265,7 @@ def test_apply_filters_part2(class_testsuite, all_testcases_dict, TESTDATA_PART3) def test_apply_filters_part3(class_testsuite, all_testcases_dict, platforms_list, tc_min_flash, plat_flash, tc_min_ram, plat_ram): - """ Testing apply_filters function of TestSuite class in Sanitycheck + """ Testing apply_filters function of TestSuite class in Twister Part 3 : Testing edge cases for ram and flash values of platforms & testcases """ class_testsuite.platforms = platforms_list @@ -282,7 +282,7 @@ def test_apply_filters_part3(class_testsuite, all_testcases_dict, platforms_list assert not discards def test_add_instances(test_data, class_testsuite, all_testcases_dict, platforms_list): - """ Testing add_instances() function of TestSuite class in Sanitycheck + """ Testing add_instances() function of TestSuite class in Twister Test 1: instances dictionary keys have expected values (Platform Name + Testcase Name) Test 2: Values of 'instances' dictionary in Testsuite class are an instance of 'TestInstance' class diff --git a/scripts/tests/twister/test_twister.py b/scripts/tests/twister/test_twister.py index ca89f6be5d8a8..058b59d3ad745 100644 --- a/scripts/tests/twister/test_twister.py +++ b/scripts/tests/twister/test_twister.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: Apache-2.0 """ -This test file contains foundational testcases for Sanitycheck tool +This test file contains foundational testcases for Twister tool """ import os @@ -14,7 +14,7 @@ sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister")) import scl -from twisterlib import SanityConfigParser +from twisterlib import TwisterConfigParser def test_yamlload(): """ Test to check if loading the non-existent files raises the errors """ @@ -30,7 +30,7 @@ def test_correct_schema(filename, schema, test_data): """ Test to validate the testcase schema""" filename = test_data + filename schema = scl.yaml_load(ZEPHYR_BASE +'/scripts/schemas/twister//' + schema) - data = SanityConfigParser(filename, schema) + data = TwisterConfigParser(filename, schema) data.load() assert data diff --git a/scripts/twister b/scripts/twister index d03a3f3d1ef6f..12b486aaeeb26 100755 --- a/scripts/twister +++ b/scripts/twister @@ -612,7 +612,7 @@ structure in the main Zephyr tree: boards///""") serial.add_argument("--device-serial-pty", help="""Script for controlling pseudoterminal. - Sanitycheck believes that it interacts with a terminal + Twister believes that it interacts with a terminal when it actually interacts with the script. E.g "twister --device-testing