import json import os import pathlib import sys from pprint import pprint from typing import Type, Optional, List import pytest from deepdiff import DeepDiff # pip install deepdiff from slither import Slither from slither.detectors.abstract_detector import AbstractDetector from slither.detectors import all_detectors class Test: # pylint: disable=too-few-public-methods def __init__( self, detector: Type[AbstractDetector], test_file: str, solc_ver: str, additional_files: Optional[List[str]] = None, ): """ :param detector: :param test_file: :param solc_ver: :param additional_files: If the test changes additional files, list them here to allow the test to update the source mapping """ self.detector = detector self.test_file = test_file self.expected_result = test_file + "." + solc_ver + "." + detector.__name__ + ".json" self.solc_ver = solc_ver if additional_files is None: self.additional_files = [] else: self.additional_files = additional_files def set_solc(test_item: Test): # hacky hack hack to pick the solc version we want env = dict(os.environ) env["SOLC_VERSION"] = test_item.solc_ver os.environ.clear() os.environ.update(env) def id_test(test_item: Test): return f"{test_item.detector}: {test_item.test_file}" ALL_TESTS = [ Test( all_detectors.BooleanEquality, "tests/detectors/boolean-constant-equality/boolean-constant-equality.sol", "0.4.25", ), Test( all_detectors.BooleanConstantMisuse, "tests/detectors/boolean-constant-misuse/boolean-constant-misuse.sol", "0.6.0", ), Test( all_detectors.UncheckedLowLevel, "tests/detectors/unchecked-lowlevel/unchecked_lowlevel.sol", "0.4.25", ), Test( all_detectors.UncheckedLowLevel, "tests/detectors/unchecked-lowlevel/unchecked_lowlevel-0.5.1.sol", "0.5.1", ), Test( all_detectors.UncheckedLowLevel, "tests/detectors/unchecked-lowlevel/unchecked_lowlevel-0.5.1.sol", "0.5.1", ), Test( all_detectors.UnindexedERC20EventParameters, "tests/detectors/erc20-indexed/erc20_indexed.sol", "0.4.25", ), Test( all_detectors.IncorrectERC20InterfaceDetection, "tests/detectors/erc20-interface/incorrect_erc20_interface.sol", "0.4.25", ), Test( all_detectors.IncorrectERC721InterfaceDetection, "tests/detectors/erc721-interface/incorrect_erc721_interface.sol", "0.4.25", ), Test( all_detectors.UninitializedStateVarsDetection, "tests/detectors/uninitialized-state/uninitialized.sol", "0.4.25", ), Test( all_detectors.UninitializedStateVarsDetection, "tests/detectors/uninitialized-state/uninitialized-0.5.1.sol", "0.5.1", ), Test(all_detectors.Backdoor, "tests/detectors/backdoor/backdoor.sol", "0.4.25"), Test(all_detectors.Backdoor, "tests/detectors/backdoor/backdoor.sol", "0.5.1"), Test(all_detectors.Suicidal, "tests/detectors/backdoor/backdoor.sol", "0.4.25"), Test(all_detectors.Suicidal, "tests/detectors/backdoor/backdoor.sol", "0.5.1"), Test( all_detectors.ConstantPragma, "tests/detectors/pragma/pragma.0.4.24.sol", "0.4.25", ["tests/detectors/pragma/pragma.0.4.23.sol"], ), Test(all_detectors.IncorrectSolc, "tests/detectors/solc-version/old_solc.sol", "0.4.21"), Test( all_detectors.IncorrectSolc, "tests/detectors/solc-version/solc_version_incorrect.sol", "0.4.25", ), Test( all_detectors.IncorrectSolc, "tests/detectors/solc-version/solc_version_incorrect_05.sol", "0.5.7", ), Test(all_detectors.ReentrancyEth, "tests/detectors/reentrancy-eth/reentrancy.sol", "0.4.25"), Test( all_detectors.ReentrancyEth, "tests/detectors/reentrancy-eth/reentrancy_indirect.sol", "0.4.25", ), Test( all_detectors.ReentrancyEth, "tests/detectors/reentrancy-eth/reentrancy-0.5.1.sol", "0.5.1" ), Test( all_detectors.UninitializedStorageVars, "tests/detectors/uninitialized-storage/uninitialized_storage_pointer.sol", "0.4.25", ), Test(all_detectors.TxOrigin, "tests/detectors/tx-origin/tx_origin.sol", "0.4.25"), Test(all_detectors.TxOrigin, "tests/detectors/tx-origin/tx_origin-0.5.1.sol", "0.5.1"), Test(all_detectors.UnusedStateVars, "tests/detectors/unused-state/unused_state.sol", "0.4.25"), Test(all_detectors.UnusedStateVars, "tests/detectors/unused-state/unused_state.sol", "0.5.1"), Test(all_detectors.LockedEther, "tests/detectors/locked-ether/locked_ether.sol", "0.4.25"), Test(all_detectors.LockedEther, "tests/detectors/locked-ether/locked_ether-0.5.1.sol", "0.5.1"), Test( all_detectors.ArbitrarySend, "tests/detectors/arbitrary-send/arbitrary_send.sol", "0.4.25" ), Test( all_detectors.ArbitrarySend, "tests/detectors/arbitrary-send/arbitrary_send-0.5.1.sol", "0.5.1", ), Test(all_detectors.Assembly, "tests/detectors/assembly/inline_assembly_contract.sol", "0.4.25"), Test(all_detectors.Assembly, "tests/detectors/assembly/inline_assembly_library.sol", "0.4.25"), Test( all_detectors.Assembly, "tests/detectors/assembly/inline_assembly_contract-0.5.1.sol", "0.5.1", ), Test( all_detectors.Assembly, "tests/detectors/assembly/inline_assembly_library-0.5.1.sol", "0.5.1", ), Test( all_detectors.LowLevelCalls, "tests/detectors/low-level-calls/low_level_calls.sol", "0.4.25" ), Test( all_detectors.LowLevelCalls, "tests/detectors/low-level-calls/low_level_calls.sol", "0.5.1" ), Test( all_detectors.ConstCandidateStateVars, "tests/detectors/constable-states/const_state_variables.sol", "0.4.25", ), Test( all_detectors.ConstCandidateStateVars, "tests/detectors/constable-states/const_state_variables.sol", "0.5.1", ), Test( all_detectors.ExternalFunction, "tests/detectors/external-function/external_function.sol", "0.4.25", ), Test( all_detectors.ExternalFunction, "tests/detectors/external-function/external_function_2.sol", "0.4.25", ), Test( all_detectors.ExternalFunction, "tests/detectors/external-function/external_function.sol", "0.5.1", ), Test( all_detectors.ExternalFunction, "tests/detectors/external-function/external_function_2.sol", "0.5.1", ), Test( all_detectors.NamingConvention, "tests/detectors/naming-convention/naming_convention.sol", "0.4.25", ), Test( all_detectors.NamingConvention, "tests/detectors/naming-convention/naming_convention.sol", "0.5.1", ), Test( all_detectors.ControlledDelegateCall, "tests/detectors/controlled-delegatecall/controlled_delegatecall.sol", "0.4.25", ), Test( all_detectors.ControlledDelegateCall, "tests/detectors/controlled-delegatecall/controlled_delegatecall.sol", "0.5.1", ), Test( all_detectors.UninitializedLocalVars, "tests/detectors/uninitialized-local/uninitialized_local_variable.sol", "0.4.25", ), Test(all_detectors.ConstantFunctionsAsm, "tests/detectors/constant/constant.sol", "0.4.25"), Test(all_detectors.ConstantFunctionsState, "tests/detectors/constant/constant.sol", "0.4.25"), Test( all_detectors.ConstantFunctionsAsm, "tests/detectors/constant/constant-0.5.1.sol", "0.5.1" ), Test( all_detectors.ConstantFunctionsState, "tests/detectors/constant/constant-0.5.1.sol", "0.5.1" ), Test( all_detectors.UnusedReturnValues, "tests/detectors/unused-return/unused_return.sol", "0.4.25", ), Test( all_detectors.UnusedReturnValues, "tests/detectors/unused-return/unused_return.sol", "0.5.1" ), Test( all_detectors.ShadowingAbstractDetection, "tests/detectors/shadowing-abstract/shadowing_abstract.sol", "0.4.25", ), Test( all_detectors.StateShadowing, "tests/detectors/shadowing-state/shadowing_state_variable.sol", "0.4.25", ), Test(all_detectors.Timestamp, "tests/detectors/timestamp/timestamp.sol", "0.4.25"), Test(all_detectors.Timestamp, "tests/detectors/timestamp/timestamp.sol", "0.5.1"), Test( all_detectors.MultipleCallsInLoop, "tests/detectors/calls-loop/multiple_calls_in_loop.sol", "0.4.25", ), Test( all_detectors.BuiltinSymbolShadowing, "tests/detectors/shadowing-builtin/shadowing_builtin_symbols.sol", "0.4.25", ), Test( all_detectors.LocalShadowing, "tests/detectors/shadowing-local/shadowing_local_variable.sol", "0.4.25", ), Test( all_detectors.RightToLeftOverride, "tests/detectors/rtlo/right_to_left_override.sol", "0.4.25", ), Test(all_detectors.VoidConstructor, "tests/detectors/void-cst/void-cst.sol", "0.5.1"), Test( all_detectors.UncheckedSend, "tests/detectors/unchecked-send/unchecked_send-0.5.1.sol", "0.5.1", ), Test( all_detectors.ReentrancyEvent, "tests/detectors/reentrancy-events/reentrancy-0.5.1-events.sol", "0.5.1", ), Test( all_detectors.IncorrectStrictEquality, "tests/detectors/incorrect-equality/incorrect_equality.sol", "0.5.1", ), Test( all_detectors.TooManyDigits, "tests/detectors/too-many-digits/too_many_digits.sol", "0.5.1" ), Test( all_detectors.UnprotectedUpgradeable, "tests/detectors/unprotected-upgrade/Buggy.sol", "0.6.12", ), Test( all_detectors.UnprotectedUpgradeable, "tests/detectors/unprotected-upgrade/Fixed.sol", "0.6.12", ), Test( all_detectors.NamingConvention, "tests/detectors/naming-convention/naming_convention_ignore.sol", "0.4.25", ), Test( all_detectors.ABIEncoderV2Array, "tests/detectors/abiencoderv2-array/storage_ABIEncoderV2_array.sol", "0.4.25", ), Test( all_detectors.ArrayByReference, "tests/detectors/array-by-reference/array_by_reference.sol", "0.4.25", ), Test( all_detectors.AssertStateChange, "tests/detectors/assert-state-change/assert_state_change.sol", "0.5.8", ), Test( all_detectors.ArrayLengthAssignment, "tests/detectors/controlled-array-length/array_length_assignment.sol", "0.4.25", ), Test( all_detectors.CostlyOperationsInLoop, "tests/detectors/costly-loop/multiple_costly_operations_in_loop.sol", "0.4.25", ), Test( all_detectors.FunctionInitializedState, "tests/detectors/function-init-state/function_init_state_variables.sol", "0.4.25", ), Test( all_detectors.MappingDeletionDetection, "tests/detectors/mapping-deletion/MappingDeletion.sol", "0.4.25", ), Test( all_detectors.UnimplementedFunctionDetection, "tests/detectors/missing-inheritance/unimplemented_interfaces.sol", "0.5.12", ), Test( all_detectors.PublicMappingNested, "tests/detectors/public-mappings-nested/public_mappings_nested.sol", "0.4.25", ), Test( all_detectors.RedundantStatements, "tests/detectors/redundant-statements/redundant_statements.sol", "0.4.25", ), Test( all_detectors.ReusedBaseConstructor, "tests/detectors/reused-constructor/reused_base_constructor.sol", "0.4.25", ), Test( all_detectors.StorageSignedIntegerArray, "tests/detectors/storage-array/storage_signed_integer_array.sol", "0.5.8", ), Test( all_detectors.UnimplementedFunctionDetection, "tests/detectors/unimplemented-functions/unimplemented.sol", "0.4.25", ), Test( all_detectors.UninitializedFunctionPtrsConstructor, "tests/detectors/uninitialized-fptr-cst/uninitialized_function_ptr_constructor.sol", "0.5.8", ), Test(all_detectors.BadPRNG, "tests/detectors/weak-prng/bad_prng.sol", "0.4.25"), ] GENERIC_PATH = "/GENERIC_PATH" @pytest.mark.parametrize("test_item", ALL_TESTS, ids=id_test) def test_detector(test_item: Test): set_solc(test_item) sl = Slither(test_item.test_file) sl.register_detector(test_item.detector) results = sl.run_detectors() with open(test_item.expected_result, encoding="utf8") as f: expected_result = json.load(f) results_as_string = json.dumps(results) current_path = str(pathlib.Path(pathlib.Path().absolute(), test_item.test_file).absolute()) for additional_file in test_item.additional_files: additional_path = str(pathlib.Path(pathlib.Path().absolute(), additional_file).absolute()) results_as_string = results_as_string.replace( additional_path, str(pathlib.Path(GENERIC_PATH)) ) results_as_string = results_as_string.replace(current_path, str(pathlib.Path(GENERIC_PATH))) results = json.loads(results_as_string) diff = DeepDiff(results, expected_result, ignore_order=True, verbose_level=2) if diff: pprint(diff) diff_as_dict = diff.to_dict() if "iterable_item_added" in diff_as_dict: print("#### Findings added") for findings_added in diff_as_dict["iterable_item_added"].values(): for finding_added in findings_added: print(finding_added["description"]) if "iterable_item_removed" in diff_as_dict: print("#### Findings removed") for findings_added in diff_as_dict["iterable_item_removed"].values(): for finding_added in findings_added: print(finding_added["description"]) assert False def _generate_test(test_item: Test, skip_existing=False): if skip_existing: if os.path.isfile(test_item.expected_result): return set_solc(test_item) sl = Slither(test_item.test_file) sl.register_detector(test_item.detector) results = sl.run_detectors() results_as_string = json.dumps(results) current_path = str(pathlib.Path(pathlib.Path().absolute(), test_item.test_file).absolute()) results_as_string = results_as_string.replace(current_path, str(pathlib.Path(GENERIC_PATH))) for additional_file in test_item.additional_files: additional_path = str(pathlib.Path(pathlib.Path().absolute(), additional_file).absolute()) results_as_string = results_as_string.replace( additional_path, str(pathlib.Path(GENERIC_PATH)) ) results = json.loads(results_as_string) with open(test_item.expected_result, "w") as f: f.write(json.dumps(results, indent=4)) if __name__ == "__main__": if len(sys.argv) != 2: print("To generate the json artifacts run\n\tpython tests/test_detectors.py --generate") elif sys.argv[1] == "--generate": for next_test in ALL_TESTS: _generate_test(next_test, skip_existing=True) elif sys.argv[1] == "--overwrite": for next_test in ALL_TESTS: _generate_test(next_test)