From f4eab883a7ad3d116dbdb476df53877ea257c2d7 Mon Sep 17 00:00:00 2001 From: Josselin Date: Tue, 10 Nov 2020 10:23:24 +0100 Subject: [PATCH 1/3] Fix parsing tests --- CONTRIBUTING.md | 1 + slither/solc_parsing/slitherSolc.py | 1 - tests/test_ast_parsing.py | 88 +++++++++++++++++++++++++---- 3 files changed, 77 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef025a624..fd8cd91ca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,6 +23,7 @@ Some pull request guidelines: ## Development Environment Instructions for installing a development version of Slither can be found in our [wiki](https://github.com/crytic/slither/wiki/Developer-installation). +To run the unit tests, you need `deepdiff` installed (`pip install deepdiff`). ## Linters Several linters and security checkers are run on the PRs. diff --git a/slither/solc_parsing/slitherSolc.py b/slither/solc_parsing/slitherSolc.py index 30a03b4ab..d9e39dccb 100644 --- a/slither/solc_parsing/slitherSolc.py +++ b/slither/solc_parsing/slitherSolc.py @@ -129,7 +129,6 @@ class SlitherSolc: return for contract_data in data_loaded[self.get_children()]: - assert contract_data[self.get_key()] in [ "ContractDefinition", "PragmaDirective", diff --git a/tests/test_ast_parsing.py b/tests/test_ast_parsing.py index 0c8b7b88b..962705be1 100644 --- a/tests/test_ast_parsing.py +++ b/tests/test_ast_parsing.py @@ -2,6 +2,7 @@ import errno import json import os import subprocess +import sys from collections import namedtuple from distutils.version import StrictVersion from typing import List, Dict @@ -80,6 +81,8 @@ XFAIL = [ "for_0.7.0_legacy", "for_0.7.1_legacy", "for_0.7.2_legacy", + "for_0.7.3_legacy", + "for_0.7.4_legacy", "function_0.6.0_legacy", "function_0.6.1_legacy", "function_0.6.2_legacy", @@ -98,6 +101,10 @@ XFAIL = [ "function_0.7.1_compact", "function_0.7.2_legacy", "function_0.7.2_compact", + "function_0.7.3_legacy", + "function_0.7.3_compact", + "function_0.7.4_legacy", + "function_0.7.4_compact", "import_0.4.0_legacy", "import_0.4.1_legacy", "import_0.4.2_legacy", @@ -208,6 +215,10 @@ XFAIL = [ "import_0.7.1_compact", "import_0.7.2_legacy", "import_0.7.2_compact", + "import_0.7.3_legacy", + "import_0.7.3_compact", + "import_0.7.4_legacy", + "import_0.7.4_compact", "indexrangeaccess_0.6.1_legacy", "indexrangeaccess_0.6.2_legacy", "indexrangeaccess_0.6.3_legacy", @@ -223,12 +234,18 @@ XFAIL = [ "indexrangeaccess_0.7.0_legacy", "indexrangeaccess_0.7.1_legacy", "indexrangeaccess_0.7.2_legacy", + "indexrangeaccess_0.7.3_legacy", + "indexrangeaccess_0.7.4_legacy", "literal_0.7.0_legacy", "literal_0.7.0_compact", "literal_0.7.1_legacy", "literal_0.7.1_compact", "literal_0.7.2_legacy", "literal_0.7.2_compact", + "literal_0.7.3_legacy", + "literal_0.7.3_compact", + "literal_0.7.4_legacy", + "literal_0.7.4_compact", "memberaccess_0.6.8_legacy", "memberaccess_0.6.9_legacy", "memberaccess_0.6.10_legacy", @@ -253,6 +270,8 @@ XFAIL = [ "struct_0.7.0_legacy", "struct_0.7.1_legacy", "struct_0.7.2_legacy", + "struct_0.7.3_legacy", + "struct_0.7.4_legacy", "trycatch_0.6.0_legacy", "trycatch_0.6.1_legacy", "trycatch_0.6.2_legacy", @@ -269,6 +288,8 @@ XFAIL = [ "trycatch_0.7.0_legacy", "trycatch_0.7.1_legacy", "trycatch_0.7.2_legacy", + "trycatch_0.7.3_legacy", + "trycatch_0.7.4_legacy", "variable_0.6.5_legacy", "variable_0.6.5_compact", "variable_0.6.6_legacy", @@ -365,6 +386,8 @@ XFAIL = [ "variabledeclaration_0.7.0_legacy", "variabledeclaration_0.7.1_legacy", "variabledeclaration_0.7.2_legacy", + "variabledeclaration_0.7.3_legacy", + "variabledeclaration_0.7.4_legacy", ] @@ -488,6 +511,14 @@ def generate_output(sl: Slither) -> Dict[str, Dict[str, str]]: ALL_TESTS = get_all_test() +def set_solc(test_item: Item): + # hacky hack hack to pick the solc version we want + env = dict(os.environ) + env["SOLC_VERSION"] = test_item.solc_ver + os.environ.clear() + os.environ.update(env) + + @pytest.mark.parametrize("test_item", ALL_TESTS, ids=id_test) def test_parsing(test_item: Item): flavor = "legacy" if test_item.is_legacy else "compact" @@ -499,11 +530,7 @@ def test_parsing(test_item: Item): if id_test(test_item) in XFAIL: pytest.xfail("this test needs to be fixed") - # hacky hack hack to pick the solc version we want - env = dict(os.environ) - env["SOLC_VERSION"] = test_item.solc_ver - os.environ.clear() - os.environ.update(env) + set_solc(test_item) sl = Slither( test_file, @@ -518,14 +545,51 @@ def test_parsing(test_item: Item): with open(expected_file, "r") as f: expected = json.load(f) except OSError as e: - if e.errno != errno.ENOENT: - raise - - # if the expected output doesn't exist, make it - with open(expected_file, "w") as f: - json.dump(actual, f, indent=" ") - expected = actual + pytest.xfail("the file for this test was not generated") + raise diff = DeepDiff(expected, actual, ignore_order=True, verbose_level=2) assert not diff, diff.pretty() + + +def _generate_test(test_item: Item, skip_existing=False): + flavor = "legacy" if test_item.is_legacy else "compact" + test_file = os.path.join(TEST_ROOT, f"{test_item.test_id}-{test_item.base_ver}.sol") + expected_file = os.path.join( + TEST_ROOT, "expected", f"{test_item.test_id}-{test_item.solc_ver}-{flavor}.json" + ) + + if skip_existing: + if os.path.isfile(expected_file): + return + + if id_test(test_item) in XFAIL: + return + + set_solc(test_item) + + sl = Slither( + test_file, + solc_force_legacy_json=test_item.is_legacy, + disallow_partial=True, + skip_analyze=True, + ) + + actual = generate_output(sl) + + with open(expected_file, "w") as f: + json.dump(actual, f, ident=" ") + + +if __name__ == "__main__": + if len(sys.argv) != 2 or sys.argv[1] not in ["--generate", "--overwrite"]: + print("To generate the missing json artifacts run\n\tpython tests/test_ast_parsing.py --generate") + print("To re-generate all the json artifacts run\n\tpython tests/test_ast_parsing.py --overwrite") + print("\tThis will overwrite the previous json files") + elif sys.argv[1] == "--generate": + for test in ALL_TESTS: + _generate_test(test, skip_existing=True) + elif sys.argv[1] == "--overwrite": + for test in ALL_TESTS: + _generate_test(test) From bc960ae54ef063e85d9d465248c06b720bbf60e4 Mon Sep 17 00:00:00 2001 From: Josselin Date: Tue, 10 Nov 2020 10:37:19 +0100 Subject: [PATCH 2/3] Fix pylint on tests/test_ast_parsing.py --- tests/test_ast_parsing.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_ast_parsing.py b/tests/test_ast_parsing.py index 26d36b5bd..4b8434788 100644 --- a/tests/test_ast_parsing.py +++ b/tests/test_ast_parsing.py @@ -1,4 +1,3 @@ -import errno import json import os import subprocess @@ -536,7 +535,7 @@ def test_parsing(test_item: Item): try: with open(expected_file, "r") as f: expected = json.load(f) - except OSError as e: + except OSError: pytest.xfail("the file for this test was not generated") raise @@ -580,8 +579,8 @@ if __name__ == "__main__": print("To re-generate all the json artifacts run\n\tpython tests/test_ast_parsing.py --overwrite") print("\tThis will overwrite the previous json files") elif sys.argv[1] == "--generate": - for test in ALL_TESTS: - _generate_test(test, skip_existing=True) + for next_test in ALL_TESTS: + _generate_test(next_test, skip_existing=True) elif sys.argv[1] == "--overwrite": - for test in ALL_TESTS: - _generate_test(test) + for next_test in ALL_TESTS: + _generate_test(next_test) From 202f5069e4b636581f4202fb63fc349d2864a703 Mon Sep 17 00:00:00 2001 From: Josselin Date: Tue, 10 Nov 2020 10:37:56 +0100 Subject: [PATCH 3/3] Fix linter on tests/test_ast_paring.py --- tests/test_ast_parsing.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_ast_parsing.py b/tests/test_ast_parsing.py index 4b8434788..cddaf8673 100644 --- a/tests/test_ast_parsing.py +++ b/tests/test_ast_parsing.py @@ -575,8 +575,12 @@ def _generate_test(test_item: Item, skip_existing=False): if __name__ == "__main__": if len(sys.argv) != 2 or sys.argv[1] not in ["--generate", "--overwrite"]: - print("To generate the missing json artifacts run\n\tpython tests/test_ast_parsing.py --generate") - print("To re-generate all the json artifacts run\n\tpython tests/test_ast_parsing.py --overwrite") + print( + "To generate the missing json artifacts run\n\tpython tests/test_ast_parsing.py --generate" + ) + print( + "To re-generate all the json artifacts run\n\tpython tests/test_ast_parsing.py --overwrite" + ) print("\tThis will overwrite the previous json files") elif sys.argv[1] == "--generate": for next_test in ALL_TESTS: