import json import os import re import subprocess import sys from collections import namedtuple from distutils.version import StrictVersion from typing import List, Dict import pytest from crytic_compile import CryticCompile, save_to_zip from crytic_compile.utils.zip import load_from_zip from deepdiff import DeepDiff from slither import Slither from slither.printers.guidance.echidna import Echidna # these solc versions only support legacy ast format LEGACY_SOLC_VERS = [f"0.4.{v}" for v in range(12)] SLITHER_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) TEST_ROOT = os.path.join(SLITHER_ROOT, "tests", "ast-parsing") ALL_04 = range(0, 27) ALL_05 = range(0, 18) ALL_06 = range(0, 13) ALL_07 = range(0, 7) ALL_08 = range(0, 7) # these are tests that are currently failing right now XFAIL = ( [f"emit_0.4.{ver}_legacy" for ver in ALL_04] + [f"emit_0.4.{ver}_compact" for ver in range(12, 27)] + [f"function_0.6.{ver}_legacy" for ver in ALL_06] + [f"function_0.7.{ver}_legacy" for ver in ALL_07] + [f"function_0.7.{ver}_compact" for ver in range(1, 7)] + [f"import_0.4.{ver}_legacy" for ver in ALL_04] + [f"import_0.4.{ver}_compact" for ver in range(12, 27)] + [f"import_0.5.{ver}_legacy" for ver in ALL_05] + [f"import_0.5.{ver}_compact" for ver in ALL_05] + [f"import_0.6.{ver}_legacy" for ver in ALL_06] + [f"import_0.6.{ver}_compact" for ver in ALL_06] + [f"import_0.7.{ver}_legacy" for ver in ALL_07] + [f"import_0.7.{ver}_compact" for ver in ALL_07] + [f"import_0.8.{ver}_compact" for ver in ALL_08] + [f"indexrangeaccess_0.6.{ver}_legacy" for ver in range(1, 13)] + [f"indexrangeaccess_0.7.{ver}_legacy" for ver in ALL_07] + [f"literal_0.7.{ver}_legacy" for ver in ALL_07] + [f"literal_0.8.{ver}_legacy" for ver in ALL_08] + [f"literal_0.7.{ver}_compact" for ver in ALL_07] + [f"literal_0.8.{ver}_compact" for ver in ALL_08] + [f"memberaccess_0.6.{ver}_legacy" for ver in range(8, 13)] + [f"memberaccess_0.7.{ver}_legacy" for ver in range(0, 3)] + [f"struct_0.6.{ver}_legacy" for ver in ALL_06] + [f"struct_0.7.{ver}_legacy" for ver in ALL_07] + [f"struct_0.8.{ver}_legacy" for ver in ALL_08] + [f"trycatch_0.6.{ver}_legacy" for ver in ALL_06] + [f"trycatch_0.7.{ver}_legacy" for ver in ALL_07] + [f"variable_0.6.{ver}_legacy" for ver in range(5, 23)] + [f"variable_0.6.{ver}_compact" for ver in range(5, 23)] + [f"variable_0.7.{ver}_legacy" for ver in range(0, 2)] + [f"variable_0.7.{ver}_compact" for ver in range(0, 2)] + [f"variabledeclaration_0.4.{ver}_legacy" for ver in ALL_04] + [f"variabledeclaration_0.5.{ver}_legacy" for ver in ALL_05] + [f"variabledeclaration_0.6.{ver}_legacy" for ver in ALL_06] + [f"variabledeclaration_0.7.{ver}_legacy" for ver in ALL_07] + [f"variabledeclaration_0.8.{ver}_legacy" for ver in ALL_08] + [f"variabledeclaration_0.4.{ver}_compact" for ver in range(12, 27)] + [f"top-level_0.7.{ver}_legacy" for ver in ALL_07] + [f"top-level_0.7.{ver}_compact" for ver in ALL_07] + [f"top-level_0.8.{ver}_legacy" for ver in ALL_08] + [f"top-level_0.8.{ver}_compact" for ver in ALL_08] + [f"top-level-import_0.7.{ver}_legacy" for ver in range(1, 7)] + [f"top-level-import_0.7.{ver}_compact" for ver in range(1, 7)] + [f"top-level-import_0.8.{ver}_compact" for ver in ALL_08] ) def get_solc_versions() -> List[str]: """ get a list of all the supported versions of solidity, sorted from earliest to latest :return: ascending list of versions, for example ["0.4.0", "0.4.1", ...] """ result = subprocess.run(["solc-select", "versions"], stdout=subprocess.PIPE, check=True) solc_versions = result.stdout.decode("utf-8").split("\n") # there's an extra newline so just remove all empty strings solc_versions = [version.split(" ")[0] for version in solc_versions if version != ""] solc_versions = sorted(solc_versions, key=lambda x: list(map(int, x.split(".")))) return solc_versions def get_tests(solc_versions) -> Dict[str, List[str]]: """ parse the list of testcases on disk :param solc_versions: the list of valid solidity versions :return: a dictionary of test id to list of base solidity versions supported """ slither_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) test_dir = os.path.join(slither_root, "tests", "ast-parsing") tests: Dict[str, List[str]] = {} for name in os.listdir(test_dir): if not name.endswith(".sol"): continue test_name, test_ver = name[:-4].rsplit("-", 1) if test_name not in tests: tests[test_name] = [] tests[test_name].append(test_ver) for key in tests: if len(tests[key]) > 1: tests[key] = sorted(tests[key], key=StrictVersion) # validate tests for test, vers in tests.items(): if len(vers) == 1: if vers[0] != "all": raise Exception("only one test found but not called all", test) else: for ver in vers: if ver not in solc_versions: raise Exception("base version not found", test, ver) return tests Item = namedtuple( "TestItem", [ "test_id", "base_ver", "solc_ver", "is_legacy", ], ) def get_all_test() -> List[Item]: """ generate a list of testcases by testing each test id with every solidity version for both legacy and compact ast :return: the testcases """ solc_versions = get_solc_versions() tests = get_tests(solc_versions) ret = [] for test, base_vers in tests.items(): print(f"generating testcases id={test} vers={base_vers}") base_ver_idx = 0 for solc_ver in solc_versions: # if it's time to move to the next base version, do it now if base_ver_idx + 1 < len(base_vers) and base_vers[base_ver_idx + 1] == solc_ver: base_ver_idx += 1 for legacy_json in [True, False]: if not legacy_json and solc_ver in LEGACY_SOLC_VERS: continue if legacy_json and solc_ver > "0.8": continue ret.append( Item( test_id=test, base_ver=base_vers[base_ver_idx], solc_ver=solc_ver, is_legacy=legacy_json, ) ) return ret def id_test(test_item: Item): flavor = "legacy" if test_item.is_legacy else "compact" return f"{test_item.test_id}_{test_item.solc_ver}_{flavor}" def generate_output(sl: Slither) -> Dict[str, Dict[str, str]]: output = {} for contract in sl.contracts: output[contract.name] = {} for func_or_modifier in contract.functions + contract.modifiers: output[contract.name][ func_or_modifier.full_name ] = func_or_modifier.slithir_cfg_to_dot_str(skip_expressions=True) return output ALL_TESTS = get_all_test() # create the output folder if needed try: os.mkdir("test_artifacts") except OSError: pass def set_solc(test_item: Item): subprocess.run(["solc-select", "use", test_item.solc_ver], stdout=subprocess.PIPE, check=True) @pytest.mark.parametrize("test_item", ALL_TESTS, ids=id_test) def test_parsing(test_item: Item): flavor = "legacy" if test_item.is_legacy else "compact" test_file = os.path.join( TEST_ROOT, "compile", f"{test_item.test_id}-{test_item.solc_ver}-{flavor}.zip" ) expected_file = os.path.join( TEST_ROOT, "expected", f"{test_item.test_id}-{test_item.solc_ver}-{flavor}.json" ) if id_test(test_item) in XFAIL: pytest.xfail("this test needs to be fixed") # set_solc(test_item) cc = load_from_zip(test_file)[0] sl = Slither( cc, solc_force_legacy_json=test_item.is_legacy, disallow_partial=True, skip_analyze=True, ) actual = generate_output(sl) try: with open(expected_file, "r") as f: expected = json.load(f) except OSError: pytest.xfail("the file for this test was not generated") raise diff = DeepDiff(expected, actual, ignore_order=True, verbose_level=2, view="tree") if diff: for change in diff.get("values_changed", []): path_list = re.findall(r"\['(.*?)'\]", change.path()) path = "_".join(path_list) with open(f"test_artifacts/{id_test(test_item)}_{path}_expected.dot", "w") as f: f.write(change.t1) with open(f"test_artifacts/{id_test(test_item)}_{path}_actual.dot", "w") as f: f.write(change.t2) assert not diff, diff.pretty() sl = Slither(cc, solc_force_legacy_json=test_item.is_legacy, disallow_partial=True) sl.register_printer(Echidna) sl.run_printers() def _generate_test(test_item: Item, skip_existing=False): flavor = "legacy" if test_item.is_legacy else "compact" test_file = os.path.join( TEST_ROOT, "compile", f"{test_item.test_id}-{test_item.solc_ver}-{flavor}.zip" ) expected_file = os.path.join( TEST_ROOT, "expected", f"{test_item.test_id}-{test_item.solc_ver}-{flavor}.json" ) if expected_file in XFAIL: return if skip_existing: if os.path.isfile(expected_file): return if id_test(test_item) in XFAIL: return # set_solc(test_item) try: sl = Slither( test_file, solc_force_legacy_json=test_item.is_legacy, disallow_partial=True, skip_analyze=True, ) # pylint: disable=broad-except except Exception: print(test_item) print(f"{expected_file} failed") return actual = generate_output(sl) print(f"Generate {expected_file}") with open(expected_file, "w") as f: json.dump(actual, f, indent=" ") def _generate_compile(test_item: Item, skip_existing=False): flavor = "legacy" if test_item.is_legacy else "compact" test_file = os.path.join(TEST_ROOT, f"{test_item.test_id}-{test_item.base_ver}.sol") expected_file = os.path.join( TEST_ROOT, "compile", f"{test_item.test_id}-{test_item.solc_ver}-{flavor}.zip" ) if skip_existing: if os.path.isfile(expected_file): return set_solc(test_item) cc = CryticCompile(test_file, solc_force_legacy_json=test_item.is_legacy) print(f"Compiled to {expected_file}") save_to_zip([cc], expected_file) if __name__ == "__main__": if len(sys.argv) != 2 or sys.argv[1] not in ["--generate", "--overwrite", "--compile"]: print( "To generate the missing json artifacts run\n\tpython tests/test_ast_parsing.py --generate" ) print( "To re-generate all the json artifacts run\n\tpython tests/test_ast_parsing.py --overwrite" ) print("To compile json artifacts run\n\tpython tests/test_ast_parsing.py --compile") print("\tThis will overwrite the previous json files") elif sys.argv[1] == "--generate": for next_test in ALL_TESTS: _generate_test(next_test, skip_existing=True) elif sys.argv[1] == "--overwrite": for next_test in ALL_TESTS: _generate_test(next_test) elif sys.argv[1] == "--compile": for next_test in ALL_TESTS: _generate_compile(next_test, skip_existing=True)