mirror of https://github.com/crytic/slither
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
338 lines
11 KiB
338 lines
11 KiB
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from collections import namedtuple
|
|
from distutils.version import StrictVersion
|
|
from typing import List, Dict
|
|
|
|
import pytest
|
|
from crytic_compile import CryticCompile, save_to_zip
|
|
from crytic_compile.utils.zip import load_from_zip
|
|
from deepdiff import DeepDiff
|
|
|
|
from slither import Slither
|
|
from slither.printers.guidance.echidna import Echidna
|
|
|
|
# these solc versions only support legacy ast format
|
|
|
|
LEGACY_SOLC_VERS = [f"0.4.{v}" for v in range(12)]
|
|
|
|
SLITHER_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
TEST_ROOT = os.path.join(SLITHER_ROOT, "tests", "ast-parsing")
|
|
|
|
ALL_04 = range(0, 27)
|
|
ALL_05 = range(0, 18)
|
|
ALL_06 = range(0, 13)
|
|
ALL_07 = range(0, 7)
|
|
ALL_08 = range(0, 7)
|
|
|
|
# these are tests that are currently failing right now
|
|
XFAIL = (
|
|
[f"emit_0.4.{ver}_legacy" for ver in ALL_04]
|
|
+ [f"emit_0.4.{ver}_compact" for ver in range(12, 27)]
|
|
+ [f"function_0.6.{ver}_legacy" for ver in ALL_06]
|
|
+ [f"function_0.7.{ver}_legacy" for ver in ALL_07]
|
|
+ [f"function_0.7.{ver}_compact" for ver in range(1, 7)]
|
|
+ [f"import_0.4.{ver}_legacy" for ver in ALL_04]
|
|
+ [f"import_0.4.{ver}_compact" for ver in range(12, 27)]
|
|
+ [f"import_0.5.{ver}_legacy" for ver in ALL_05]
|
|
+ [f"import_0.5.{ver}_compact" for ver in ALL_05]
|
|
+ [f"import_0.6.{ver}_legacy" for ver in ALL_06]
|
|
+ [f"import_0.6.{ver}_compact" for ver in ALL_06]
|
|
+ [f"import_0.7.{ver}_legacy" for ver in ALL_07]
|
|
+ [f"import_0.7.{ver}_compact" for ver in ALL_07]
|
|
+ [f"import_0.8.{ver}_compact" for ver in ALL_08]
|
|
+ [f"indexrangeaccess_0.6.{ver}_legacy" for ver in range(1, 13)]
|
|
+ [f"indexrangeaccess_0.7.{ver}_legacy" for ver in ALL_07]
|
|
+ [f"literal_0.7.{ver}_legacy" for ver in ALL_07]
|
|
+ [f"literal_0.8.{ver}_legacy" for ver in ALL_08]
|
|
+ [f"literal_0.7.{ver}_compact" for ver in ALL_07]
|
|
+ [f"literal_0.8.{ver}_compact" for ver in ALL_08]
|
|
+ [f"memberaccess_0.6.{ver}_legacy" for ver in range(8, 13)]
|
|
+ [f"memberaccess_0.7.{ver}_legacy" for ver in range(0, 3)]
|
|
+ [f"struct_0.6.{ver}_legacy" for ver in ALL_06]
|
|
+ [f"struct_0.7.{ver}_legacy" for ver in ALL_07]
|
|
+ [f"struct_0.8.{ver}_legacy" for ver in ALL_08]
|
|
+ [f"trycatch_0.6.{ver}_legacy" for ver in ALL_06]
|
|
+ [f"trycatch_0.7.{ver}_legacy" for ver in ALL_07]
|
|
+ [f"variable_0.6.{ver}_legacy" for ver in range(5, 23)]
|
|
+ [f"variable_0.6.{ver}_compact" for ver in range(5, 23)]
|
|
+ [f"variable_0.7.{ver}_legacy" for ver in range(0, 2)]
|
|
+ [f"variable_0.7.{ver}_compact" for ver in range(0, 2)]
|
|
+ [f"variabledeclaration_0.4.{ver}_legacy" for ver in ALL_04]
|
|
+ [f"variabledeclaration_0.5.{ver}_legacy" for ver in ALL_05]
|
|
+ [f"variabledeclaration_0.6.{ver}_legacy" for ver in ALL_06]
|
|
+ [f"variabledeclaration_0.7.{ver}_legacy" for ver in ALL_07]
|
|
+ [f"variabledeclaration_0.8.{ver}_legacy" for ver in ALL_08]
|
|
+ [f"variabledeclaration_0.4.{ver}_compact" for ver in range(12, 27)]
|
|
+ [f"top-level_0.7.{ver}_legacy" for ver in ALL_07]
|
|
+ [f"top-level_0.7.{ver}_compact" for ver in ALL_07]
|
|
+ [f"top-level_0.8.{ver}_legacy" for ver in ALL_08]
|
|
+ [f"top-level_0.8.{ver}_compact" for ver in ALL_08]
|
|
+ [f"top-level-import_0.7.{ver}_legacy" for ver in range(1, 7)]
|
|
+ [f"top-level-import_0.7.{ver}_compact" for ver in range(1, 7)]
|
|
+ [f"top-level-import_0.8.{ver}_compact" for ver in ALL_08]
|
|
)
|
|
|
|
|
|
def get_solc_versions() -> List[str]:
|
|
"""
|
|
get a list of all the supported versions of solidity, sorted from earliest to latest
|
|
:return: ascending list of versions, for example ["0.4.0", "0.4.1", ...]
|
|
"""
|
|
result = subprocess.run(["solc-select", "versions"], stdout=subprocess.PIPE, check=True)
|
|
solc_versions = result.stdout.decode("utf-8").split("\n")
|
|
|
|
# there's an extra newline so just remove all empty strings
|
|
solc_versions = [version.split(" ")[0] for version in solc_versions if version != ""]
|
|
|
|
solc_versions = sorted(solc_versions, key=lambda x: list(map(int, x.split("."))))
|
|
return solc_versions
|
|
|
|
|
|
def get_tests(solc_versions) -> Dict[str, List[str]]:
|
|
"""
|
|
parse the list of testcases on disk
|
|
:param solc_versions: the list of valid solidity versions
|
|
:return: a dictionary of test id to list of base solidity versions supported
|
|
"""
|
|
slither_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
test_dir = os.path.join(slither_root, "tests", "ast-parsing")
|
|
|
|
tests: Dict[str, List[str]] = {}
|
|
|
|
for name in os.listdir(test_dir):
|
|
if not name.endswith(".sol"):
|
|
continue
|
|
|
|
test_name, test_ver = name[:-4].rsplit("-", 1)
|
|
|
|
if test_name not in tests:
|
|
tests[test_name] = []
|
|
|
|
tests[test_name].append(test_ver)
|
|
|
|
for key in tests:
|
|
if len(tests[key]) > 1:
|
|
tests[key] = sorted(tests[key], key=StrictVersion)
|
|
|
|
# validate tests
|
|
for test, vers in tests.items():
|
|
if len(vers) == 1:
|
|
if vers[0] != "all":
|
|
raise Exception("only one test found but not called all", test)
|
|
else:
|
|
for ver in vers:
|
|
if ver not in solc_versions:
|
|
raise Exception("base version not found", test, ver)
|
|
|
|
return tests
|
|
|
|
|
|
Item = namedtuple(
|
|
"TestItem",
|
|
[
|
|
"test_id",
|
|
"base_ver",
|
|
"solc_ver",
|
|
"is_legacy",
|
|
],
|
|
)
|
|
|
|
|
|
def get_all_test() -> List[Item]:
|
|
"""
|
|
generate a list of testcases by testing each test id with every solidity version for both legacy and compact ast
|
|
:return: the testcases
|
|
"""
|
|
solc_versions = get_solc_versions()
|
|
tests = get_tests(solc_versions)
|
|
|
|
ret = []
|
|
|
|
for test, base_vers in tests.items():
|
|
print(f"generating testcases id={test} vers={base_vers}")
|
|
|
|
base_ver_idx = 0
|
|
|
|
for solc_ver in solc_versions:
|
|
|
|
# if it's time to move to the next base version, do it now
|
|
if base_ver_idx + 1 < len(base_vers) and base_vers[base_ver_idx + 1] == solc_ver:
|
|
base_ver_idx += 1
|
|
|
|
for legacy_json in [True, False]:
|
|
if not legacy_json and solc_ver in LEGACY_SOLC_VERS:
|
|
continue
|
|
|
|
if legacy_json and solc_ver > "0.8":
|
|
continue
|
|
|
|
ret.append(
|
|
Item(
|
|
test_id=test,
|
|
base_ver=base_vers[base_ver_idx],
|
|
solc_ver=solc_ver,
|
|
is_legacy=legacy_json,
|
|
)
|
|
)
|
|
return ret
|
|
|
|
|
|
def id_test(test_item: Item):
|
|
flavor = "legacy" if test_item.is_legacy else "compact"
|
|
return f"{test_item.test_id}_{test_item.solc_ver}_{flavor}"
|
|
|
|
|
|
def generate_output(sl: Slither) -> Dict[str, Dict[str, str]]:
|
|
output = {}
|
|
for contract in sl.contracts:
|
|
output[contract.name] = {}
|
|
|
|
for func_or_modifier in contract.functions + contract.modifiers:
|
|
output[contract.name][
|
|
func_or_modifier.full_name
|
|
] = func_or_modifier.slithir_cfg_to_dot_str(skip_expressions=True)
|
|
|
|
return output
|
|
|
|
|
|
ALL_TESTS = get_all_test()
|
|
|
|
# create the output folder if needed
|
|
try:
|
|
os.mkdir("test_artifacts")
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def set_solc(test_item: Item):
|
|
subprocess.run(["solc-select", "use", test_item.solc_ver], stdout=subprocess.PIPE, check=True)
|
|
|
|
|
|
@pytest.mark.parametrize("test_item", ALL_TESTS, ids=id_test)
|
|
def test_parsing(test_item: Item):
|
|
flavor = "legacy" if test_item.is_legacy else "compact"
|
|
test_file = os.path.join(
|
|
TEST_ROOT, "compile", f"{test_item.test_id}-{test_item.solc_ver}-{flavor}.zip"
|
|
)
|
|
expected_file = os.path.join(
|
|
TEST_ROOT, "expected", f"{test_item.test_id}-{test_item.solc_ver}-{flavor}.json"
|
|
)
|
|
|
|
if id_test(test_item) in XFAIL:
|
|
pytest.xfail("this test needs to be fixed")
|
|
|
|
# set_solc(test_item)
|
|
|
|
cc = load_from_zip(test_file)[0]
|
|
|
|
sl = Slither(
|
|
cc,
|
|
solc_force_legacy_json=test_item.is_legacy,
|
|
disallow_partial=True,
|
|
skip_analyze=True,
|
|
)
|
|
|
|
actual = generate_output(sl)
|
|
|
|
try:
|
|
with open(expected_file, "r") as f:
|
|
expected = json.load(f)
|
|
except OSError:
|
|
pytest.xfail("the file for this test was not generated")
|
|
raise
|
|
|
|
diff = DeepDiff(expected, actual, ignore_order=True, verbose_level=2, view="tree")
|
|
|
|
if diff:
|
|
for change in diff.get("values_changed", []):
|
|
path_list = re.findall(r"\['(.*?)'\]", change.path())
|
|
path = "_".join(path_list)
|
|
with open(f"test_artifacts/{id_test(test_item)}_{path}_expected.dot", "w") as f:
|
|
f.write(change.t1)
|
|
with open(f"test_artifacts/{id_test(test_item)}_{path}_actual.dot", "w") as f:
|
|
f.write(change.t2)
|
|
|
|
assert not diff, diff.pretty()
|
|
|
|
sl = Slither(cc, solc_force_legacy_json=test_item.is_legacy, disallow_partial=True)
|
|
sl.register_printer(Echidna)
|
|
sl.run_printers()
|
|
|
|
|
|
def _generate_test(test_item: Item, skip_existing=False):
|
|
flavor = "legacy" if test_item.is_legacy else "compact"
|
|
test_file = os.path.join(
|
|
TEST_ROOT, "compile", f"{test_item.test_id}-{test_item.solc_ver}-{flavor}.zip"
|
|
)
|
|
expected_file = os.path.join(
|
|
TEST_ROOT, "expected", f"{test_item.test_id}-{test_item.solc_ver}-{flavor}.json"
|
|
)
|
|
|
|
if expected_file in XFAIL:
|
|
return
|
|
if skip_existing:
|
|
if os.path.isfile(expected_file):
|
|
return
|
|
if id_test(test_item) in XFAIL:
|
|
return
|
|
# set_solc(test_item)
|
|
try:
|
|
sl = Slither(
|
|
test_file,
|
|
solc_force_legacy_json=test_item.is_legacy,
|
|
disallow_partial=True,
|
|
skip_analyze=True,
|
|
)
|
|
# pylint: disable=broad-except
|
|
except Exception:
|
|
print(test_item)
|
|
print(f"{expected_file} failed")
|
|
return
|
|
|
|
actual = generate_output(sl)
|
|
print(f"Generate {expected_file}")
|
|
with open(expected_file, "w") as f:
|
|
json.dump(actual, f, indent=" ")
|
|
|
|
|
|
def _generate_compile(test_item: Item, skip_existing=False):
|
|
flavor = "legacy" if test_item.is_legacy else "compact"
|
|
test_file = os.path.join(TEST_ROOT, f"{test_item.test_id}-{test_item.base_ver}.sol")
|
|
expected_file = os.path.join(
|
|
TEST_ROOT, "compile", f"{test_item.test_id}-{test_item.solc_ver}-{flavor}.zip"
|
|
)
|
|
|
|
if skip_existing:
|
|
if os.path.isfile(expected_file):
|
|
return
|
|
|
|
set_solc(test_item)
|
|
|
|
cc = CryticCompile(test_file, solc_force_legacy_json=test_item.is_legacy)
|
|
print(f"Compiled to {expected_file}")
|
|
save_to_zip([cc], expected_file)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) != 2 or sys.argv[1] not in ["--generate", "--overwrite", "--compile"]:
|
|
print(
|
|
"To generate the missing json artifacts run\n\tpython tests/test_ast_parsing.py --generate"
|
|
)
|
|
print(
|
|
"To re-generate all the json artifacts run\n\tpython tests/test_ast_parsing.py --overwrite"
|
|
)
|
|
print("To compile json artifacts run\n\tpython tests/test_ast_parsing.py --compile")
|
|
print("\tThis will overwrite the previous json files")
|
|
elif sys.argv[1] == "--generate":
|
|
for next_test in ALL_TESTS:
|
|
_generate_test(next_test, skip_existing=True)
|
|
elif sys.argv[1] == "--overwrite":
|
|
for next_test in ALL_TESTS:
|
|
_generate_test(next_test)
|
|
elif sys.argv[1] == "--compile":
|
|
for next_test in ALL_TESTS:
|
|
_generate_compile(next_test, skip_existing=True)
|
|
|