From 7798f473d204200fb7a98df10585c6ac067d20f9 Mon Sep 17 00:00:00 2001 From: alpharush <0xalpharush@protonmail.com> Date: Thu, 5 Jan 2023 07:03:37 -0600 Subject: [PATCH] make it easier to run specific tests and document --- CONTRIBUTING.md | 9 ++++++++- tests/test_ast_parsing.py | 30 +++++++++++++++--------------- tests/test_detectors.py | 2 +- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a30595ec..3cfdb7d3f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,10 @@ For each new detector, at least one regression tests must be present. - If updating an existing detector, identify the respective json artifacts and then delete them, or run `python ./tests/test_detectors.py --overwrite` instead. - Run `pytest ./tests/test_detectors.py` and check that everything worked. -To see the tests coverage, run `pytest tests/test_detectors.py --cov=slither/detectors --cov-branch --cov-report html` +To see the tests coverage, run `pytest tests/test_detectors.py --cov=slither/detectors --cov-branch --cov-report html`. +To run tests for a specific detector, run `pytest tests/test_detectors.py -k ReentrancyReadBeforeWritten` (the detector's class name is the argument). +To run tests for a specific version, run `pytest tests/test_detectors.py -k 0.7.6`. +The id's of tests can be inspected using ``pytest tests/test_detectors.py --collect-only`. ### Parser tests - Create a test in `tests/ast-parsing` @@ -73,6 +76,10 @@ To see the tests coverage, run `pytest tests/test_detectors.py --cov=slither/d - Run `pytest ./tests/test_ast_parsing.py` and check that everything worked. To see the tests coverage, run `pytest tests/test_ast_parsing.py --cov=slither/solc_parsing --cov-branch --cov-report html` +To run tests for a specific test case, run `pytest tests/test_ast_parsing.py -k user_defined_value_type` (the file name is the argument). +To run tests for a specific version, run `pytest tests/test_ast_parsing.py -k 0.8.12`. +To run tests for a specific compiler json format, run `pytest tests/test_ast_parsing.py -k legacy` (can be legacy or compact). +The id's of tests can be inspected using ``pytest tests/test_ast_parsing.py --collect-only`. ### Synchronization with crytic-compile By default, `slither` follows either the latest version of crytic-compile in pip, or `crytic-compile@master` (look for dependencies in [`setup.py`](./setup.py). If crytic-compile development comes with breaking changes, the process to update `slither` is: diff --git a/tests/test_ast_parsing.py b/tests/test_ast_parsing.py index e96a129b8..23783137b 100644 --- a/tests/test_ast_parsing.py +++ b/tests/test_ast_parsing.py @@ -433,18 +433,19 @@ except OSError: pass -@pytest.mark.parametrize("test_item", ALL_TESTS, ids=lambda x: x.test_file) -def test_parsing(test_item: Test): - flavors = ["compact"] - if not test_item.disable_legacy: - flavors += ["legacy"] - for version, flavor in test_item.versions_with_flavors: - test_file = os.path.join( - TEST_ROOT, "compile", f"{test_item.test_file}-{version}-{flavor}.zip" - ) - expected_file = os.path.join( - TEST_ROOT, "expected", f"{test_item.test_file}-{version}-{flavor}.json" - ) +def pytest_generate_tests(metafunc): + test_cases = [] + for test_item in ALL_TESTS: + for version, flavor in test_item.versions_with_flavors: + test_cases.append((test_item.test_file, version, flavor)) + metafunc.parametrize("test_file, version, flavor", test_cases) + + +class TestASTParsing: + # pylint: disable=no-self-use + def test_parsing(self, test_file, version, flavor): + test_file = os.path.join(TEST_ROOT, "compile", f"{test_file}-{version}-{flavor}.zip") + expected_file = os.path.join(TEST_ROOT, "expected", f"{test_file}-{version}-{flavor}.json") cc = load_from_zip(test_file)[0] @@ -465,19 +466,18 @@ def test_parsing(test_item: Test): raise diff = DeepDiff(expected, actual, ignore_order=True, verbose_level=2, view="tree") - if diff: for change in diff.get("values_changed", []): path_list = re.findall(r"\['(.*?)'\]", change.path()) path = "_".join(path_list) with open( - f"test_artifacts/{test_item.test_file}_{path}_expected.dot", + f"test_artifacts/{test_file}_{path}_expected.dot", "w", encoding="utf8", ) as f: f.write(change.t1) with open( - f"test_artifacts/{test_item.test_file}_{version}_{flavor}_{path}_actual.dot", + f"test_artifacts/{test_file}_{version}_{flavor}_{path}_actual.dot", "w", encoding="utf8", ) as f: diff --git a/tests/test_detectors.py b/tests/test_detectors.py index a5ecaae2a..a45369fcd 100644 --- a/tests/test_detectors.py +++ b/tests/test_detectors.py @@ -52,7 +52,7 @@ def set_solc(test_item: Test): # pylint: disable=too-many-lines def id_test(test_item: Test): - return f"{test_item.detector}: {test_item.solc_ver}/{test_item.test_file}" + return f"{test_item.detector.__name__}-{test_item.solc_ver}-{test_item.test_file}" ALL_TEST_OBJECTS = [