From ed032165d44d8640582d97e8416afced80553774 Mon Sep 17 00:00:00 2001 From: e-ngo Date: Sun, 28 Jul 2019 23:43:40 -0700 Subject: [PATCH 1/9] Refactored interfaces to run custom modules --- mythril/analysis/security.py | 73 ++++++++++++++++++++--------- mythril/analysis/symbolic.py | 16 +++++-- mythril/interfaces/cli.py | 8 ++++ mythril/mythril/mythril_analyzer.py | 16 +++++-- 4 files changed, 83 insertions(+), 30 deletions(-) diff --git a/mythril/analysis/security.py b/mythril/analysis/security.py index 71f0a516..5c5e520a 100644 --- a/mythril/analysis/security.py +++ b/mythril/analysis/security.py @@ -6,22 +6,28 @@ from mythril.analysis import modules import pkgutil import importlib.util import logging +import os +import sys log = logging.getLogger(__name__) OPCODE_LIST = [c[0] for _, c in opcodes.items()] -def reset_callback_modules(): +def reset_callback_modules(module_names=(), custom_modules_directory=""): """Clean the issue records of every callback-based module.""" - modules = get_detection_modules("callback") + modules = get_detection_modules("callback", module_names, custom_modules_directory) for module in modules: module.detector.reset_module() -def get_detection_module_hooks(modules, hook_type="pre"): +def get_detection_module_hooks(modules, hook_type="pre", custom_modules_directory=""): hook_dict = defaultdict(list) - _modules = get_detection_modules(entrypoint="callback", include_modules=modules) + _modules = get_detection_modules( + entrypoint="callback", + include_modules=modules, + custom_modules_directory=custom_modules_directory, + ) for module in _modules: hooks = ( module.detector.pre_hooks @@ -45,14 +51,13 @@ def get_detection_module_hooks(modules, hook_type="pre"): return dict(hook_dict) -def get_detection_modules(entrypoint, include_modules=()): +def get_detection_modules(entrypoint, include_modules=(), custom_modules_directory=""): """ :param entrypoint: :param include_modules: :return: """ - module = importlib.import_module("mythril.analysis.modules.base") module.log.setLevel(log.level) @@ -60,27 +65,43 @@ def get_detection_modules(entrypoint, include_modules=()): _modules = [] - if not include_modules: - for loader, module_name, _ in pkgutil.walk_packages(modules.__path__): + for loader, module_name, _ in pkgutil.walk_packages(modules.__path__): + if include_modules and module_name not in include_modules: + continue + + if module_name != "base": + module = importlib.import_module("mythril.analysis.modules." + module_name) + module.log.setLevel(log.level) + if module.detector.entrypoint == entrypoint: + _modules.append(module) + if custom_modules_directory: + custom_modules = [os.path.abspath(custom_modules_directory)] + sys.path.append(custom_modules_directory) + + for loader, module_name, _ in pkgutil.walk_packages(custom_modules): + if include_modules and module_name not in include_modules: + continue + if module_name != "base": - module = importlib.import_module( - "mythril.analysis.modules." + module_name - ) + module = importlib.import_module(module_name, custom_modules[0]) module.log.setLevel(log.level) if module.detector.entrypoint == entrypoint: _modules.append(module) - else: - for module_name in include_modules: - module = importlib.import_module("mythril.analysis.modules." + module_name) - if module.__name__ != "base" and module.detector.entrypoint == entrypoint: - module.log.setLevel(log.level) - _modules.append(module) + """ + for loader, module_name, _ in pkgutil.walk_packages([custom_modules_path]): + + custom_modules_path = os.path.abspath("custom/") + sys.path.append(custom_modules_path); + module = importlib.import_module( + module_name, custom_modules_path + ) + """ log.info("Found %s detection modules", len(_modules)) return _modules -def fire_lasers(statespace, module_names=()): +def fire_lasers(statespace, module_names=(), custom_modules_directory=""): """ :param statespace: @@ -91,22 +112,28 @@ def fire_lasers(statespace, module_names=()): issues = [] for module in get_detection_modules( - entrypoint="post", include_modules=module_names + entrypoint="post", + include_modules=module_names, + custom_modules_directory=custom_modules_directory, ): log.info("Executing " + module.detector.name) issues += module.detector.execute(statespace) - issues += retrieve_callback_issues(module_names) + issues += retrieve_callback_issues(module_names, custom_modules_directory) return issues -def retrieve_callback_issues(module_names=()): +def retrieve_callback_issues(module_names=(), custom_modules_directory=""): issues = [] for module in get_detection_modules( - entrypoint="callback", include_modules=module_names + entrypoint="callback", + include_modules=module_names, + custom_modules_directory=custom_modules_directory, ): log.debug("Retrieving results for " + module.detector.name) issues += module.detector.issues - reset_callback_modules() + reset_callback_modules( + module_names=module_names, custom_modules_directory=custom_modules_directory + ) return issues diff --git a/mythril/analysis/symbolic.py b/mythril/analysis/symbolic.py index 4164758f..02b5c8ca 100644 --- a/mythril/analysis/symbolic.py +++ b/mythril/analysis/symbolic.py @@ -56,6 +56,7 @@ class SymExecWrapper: disable_dependency_pruning=False, run_analysis_modules=True, enable_coverage_strategy=False, + custom_modules_directory="", ): """ @@ -93,7 +94,8 @@ class SymExecWrapper: ) requires_statespace = ( - compulsory_statespace or len(get_detection_modules("post", modules)) > 0 + compulsory_statespace + or len(get_detection_modules("post", modules, custom_modules_directory)) > 0 ) if not contract.creation_code: self.accounts = {hex(ATTACKER_ADDRESS): attacker_account} @@ -135,11 +137,19 @@ class SymExecWrapper: if run_analysis_modules: self.laser.register_hooks( hook_type="pre", - hook_dict=get_detection_module_hooks(modules, hook_type="pre"), + hook_dict=get_detection_module_hooks( + modules, + hook_type="pre", + custom_modules_directory=custom_modules_directory, + ), ) self.laser.register_hooks( hook_type="post", - hook_dict=get_detection_module_hooks(modules, hook_type="post"), + hook_dict=get_detection_module_hooks( + modules, + hook_type="post", + custom_modules_directory=custom_modules_directory, + ), ) if isinstance(contract, SolidityContract): diff --git a/mythril/interfaces/cli.py b/mythril/interfaces/cli.py index 839d1fd5..486b5aaf 100644 --- a/mythril/interfaces/cli.py +++ b/mythril/interfaces/cli.py @@ -396,6 +396,11 @@ def create_analyzer_parser(analyzer_parser: ArgumentParser): action="store_true", help="enable coverage based search strategy", ) + options.add_argument( + "--custom-modules-directory", + help="designates a separate directory to search for security modules from", + metavar="PATH_TO_CUSTOM_MODULES_DIRECTORY", + ) def validate_args(args: Namespace): @@ -567,6 +572,9 @@ def execute_command( solver_timeout=args.solver_timeout, requires_dynld=not args.no_onchain_storage_access, enable_coverage_strategy=args.enable_coverage_strategy, + custom_modules_directory=args.custom_modules_directory + if args.custom_modules_directory + else "", ) if not disassembler.contracts: diff --git a/mythril/mythril/mythril_analyzer.py b/mythril/mythril/mythril_analyzer.py index deb86281..d5e87369 100644 --- a/mythril/mythril/mythril_analyzer.py +++ b/mythril/mythril/mythril_analyzer.py @@ -42,6 +42,7 @@ class MythrilAnalyzer: disable_dependency_pruning: bool = False, solver_timeout: Optional[int] = None, enable_coverage_strategy: bool = False, + custom_modules_directory: str = "", ): """ @@ -63,6 +64,7 @@ class MythrilAnalyzer: self.enable_iprof = enable_iprof self.disable_dependency_pruning = disable_dependency_pruning self.enable_coverage_strategy = enable_coverage_strategy + self.custom_modules_directory = custom_modules_directory analysis_args.set_loop_bound(loop_bound) analysis_args.set_solver_timeout(solver_timeout) @@ -89,6 +91,7 @@ class MythrilAnalyzer: disable_dependency_pruning=self.disable_dependency_pruning, run_analysis_modules=False, enable_coverage_strategy=self.enable_coverage_strategy, + custom_modules_directory=self.custom_modules_directory, ) return get_serializable_statespace(sym) @@ -125,6 +128,7 @@ class MythrilAnalyzer: disable_dependency_pruning=self.disable_dependency_pruning, run_analysis_modules=False, enable_coverage_strategy=self.enable_coverage_strategy, + custom_modules_directory=self.custom_modules_directory, ) return generate_graph(sym, physics=enable_physics, phrackify=phrackify) @@ -163,18 +167,22 @@ class MythrilAnalyzer: enable_iprof=self.enable_iprof, disable_dependency_pruning=self.disable_dependency_pruning, enable_coverage_strategy=self.enable_coverage_strategy, + custom_modules_directory=self.custom_modules_directory, ) - - issues = fire_lasers(sym, modules) + issues = fire_lasers(sym, modules, self.custom_modules_directory) except KeyboardInterrupt: log.critical("Keyboard Interrupt") - issues = retrieve_callback_issues(modules) + issues = retrieve_callback_issues( + modules, self.custom_modules_directory + ) except Exception: log.critical( "Exception occurred, aborting analysis. Please report this issue to the Mythril GitHub page.\n" + traceback.format_exc() ) - issues = retrieve_callback_issues(modules) + issues = retrieve_callback_issues( + modules, self.custom_modules_directory + ) exceptions.append(traceback.format_exc()) for issue in issues: issue.add_code_info(contract) From a8dc3aa7bc1c6e5809cdf020b5ce062e06ab60d2 Mon Sep 17 00:00:00 2001 From: e-ngo Date: Mon, 29 Jul 2019 20:24:04 -0700 Subject: [PATCH 2/9] Refactored code --- mythril/analysis/security.py | 46 +++++++++++++++++------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/mythril/analysis/security.py b/mythril/analysis/security.py index 5c5e520a..4d95497a 100644 --- a/mythril/analysis/security.py +++ b/mythril/analysis/security.py @@ -65,38 +65,36 @@ def get_detection_modules(entrypoint, include_modules=(), custom_modules_directo _modules = [] - for loader, module_name, _ in pkgutil.walk_packages(modules.__path__): + custom_modules_directory = os.path.abspath(custom_modules_directory) + + if custom_modules_directory and custom_modules_directory not in sys.path: + sys.path.append(custom_modules_directory) + + custom_packages = ( + list(pkgutil.walk_packages([custom_modules_directory])) + if custom_modules_directory + else [] + ) + packages = list(pkgutil.walk_packages(modules.__path__)) + custom_packages + + for loader, module_name, _ in packages: if include_modules and module_name not in include_modules: continue if module_name != "base": - module = importlib.import_module("mythril.analysis.modules." + module_name) + try: + module = importlib.import_module( + "mythril.analysis.modules." + module_name + ) + except ModuleNotFoundError: + try: + module = importlib.import_module(module_name) + except ModuleNotFoundError: + raise ModuleNotFoundError module.log.setLevel(log.level) if module.detector.entrypoint == entrypoint: _modules.append(module) - if custom_modules_directory: - custom_modules = [os.path.abspath(custom_modules_directory)] - sys.path.append(custom_modules_directory) - for loader, module_name, _ in pkgutil.walk_packages(custom_modules): - if include_modules and module_name not in include_modules: - continue - - if module_name != "base": - module = importlib.import_module(module_name, custom_modules[0]) - module.log.setLevel(log.level) - if module.detector.entrypoint == entrypoint: - _modules.append(module) - - """ - for loader, module_name, _ in pkgutil.walk_packages([custom_modules_path]): - - custom_modules_path = os.path.abspath("custom/") - sys.path.append(custom_modules_path); - module = importlib.import_module( - module_name, custom_modules_path - ) - """ log.info("Found %s detection modules", len(_modules)) return _modules From 80c21a37a2e6f1e0688119102a2fdf220f56c133 Mon Sep 17 00:00:00 2001 From: e-ngo Date: Mon, 29 Jul 2019 22:13:21 -0700 Subject: [PATCH 3/9] Fixed some logic --- mythril/analysis/security.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mythril/analysis/security.py b/mythril/analysis/security.py index 4d95497a..39c007e2 100644 --- a/mythril/analysis/security.py +++ b/mythril/analysis/security.py @@ -65,14 +65,14 @@ def get_detection_modules(entrypoint, include_modules=(), custom_modules_directo _modules = [] - custom_modules_directory = os.path.abspath(custom_modules_directory) + custom_modules_path = os.path.abspath(custom_modules_directory) - if custom_modules_directory and custom_modules_directory not in sys.path: - sys.path.append(custom_modules_directory) + if custom_modules_directory and custom_modules_path not in sys.path: + sys.path.append(custom_modules_path) custom_packages = ( - list(pkgutil.walk_packages([custom_modules_directory])) - if custom_modules_directory + list(pkgutil.walk_packages([custom_modules_path])) + if custom_modules_path else [] ) packages = list(pkgutil.walk_packages(modules.__path__)) + custom_packages @@ -88,7 +88,7 @@ def get_detection_modules(entrypoint, include_modules=(), custom_modules_directo ) except ModuleNotFoundError: try: - module = importlib.import_module(module_name) + module = importlib.import_module(module_name, custom_modules_path) except ModuleNotFoundError: raise ModuleNotFoundError module.log.setLevel(log.level) From 938dee8e6eb64bf740200d1ebf6f6248742edf03 Mon Sep 17 00:00:00 2001 From: e-ngo Date: Tue, 30 Jul 2019 18:20:40 -0700 Subject: [PATCH 4/9] Revert "Fixed some logic" This reverts commit 80c21a37a2e6f1e0688119102a2fdf220f56c133. --- mythril/analysis/security.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mythril/analysis/security.py b/mythril/analysis/security.py index 39c007e2..4d95497a 100644 --- a/mythril/analysis/security.py +++ b/mythril/analysis/security.py @@ -65,14 +65,14 @@ def get_detection_modules(entrypoint, include_modules=(), custom_modules_directo _modules = [] - custom_modules_path = os.path.abspath(custom_modules_directory) + custom_modules_directory = os.path.abspath(custom_modules_directory) - if custom_modules_directory and custom_modules_path not in sys.path: - sys.path.append(custom_modules_path) + if custom_modules_directory and custom_modules_directory not in sys.path: + sys.path.append(custom_modules_directory) custom_packages = ( - list(pkgutil.walk_packages([custom_modules_path])) - if custom_modules_path + list(pkgutil.walk_packages([custom_modules_directory])) + if custom_modules_directory else [] ) packages = list(pkgutil.walk_packages(modules.__path__)) + custom_packages @@ -88,7 +88,7 @@ def get_detection_modules(entrypoint, include_modules=(), custom_modules_directo ) except ModuleNotFoundError: try: - module = importlib.import_module(module_name, custom_modules_path) + module = importlib.import_module(module_name) except ModuleNotFoundError: raise ModuleNotFoundError module.log.setLevel(log.level) From c6e8cc8ffa354ce1a7a6b30afeaf9b66efbe771a Mon Sep 17 00:00:00 2001 From: e-ngo Date: Tue, 30 Jul 2019 18:24:36 -0700 Subject: [PATCH 5/9] Revert "Refactored code" This reverts commit a8dc3aa7bc1c6e5809cdf020b5ce062e06ab60d2. --- mythril/analysis/security.py | 46 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/mythril/analysis/security.py b/mythril/analysis/security.py index 4d95497a..5c5e520a 100644 --- a/mythril/analysis/security.py +++ b/mythril/analysis/security.py @@ -65,36 +65,38 @@ def get_detection_modules(entrypoint, include_modules=(), custom_modules_directo _modules = [] - custom_modules_directory = os.path.abspath(custom_modules_directory) - - if custom_modules_directory and custom_modules_directory not in sys.path: - sys.path.append(custom_modules_directory) - - custom_packages = ( - list(pkgutil.walk_packages([custom_modules_directory])) - if custom_modules_directory - else [] - ) - packages = list(pkgutil.walk_packages(modules.__path__)) + custom_packages - - for loader, module_name, _ in packages: + for loader, module_name, _ in pkgutil.walk_packages(modules.__path__): if include_modules and module_name not in include_modules: continue if module_name != "base": - try: - module = importlib.import_module( - "mythril.analysis.modules." + module_name - ) - except ModuleNotFoundError: - try: - module = importlib.import_module(module_name) - except ModuleNotFoundError: - raise ModuleNotFoundError + module = importlib.import_module("mythril.analysis.modules." + module_name) module.log.setLevel(log.level) if module.detector.entrypoint == entrypoint: _modules.append(module) + if custom_modules_directory: + custom_modules = [os.path.abspath(custom_modules_directory)] + sys.path.append(custom_modules_directory) + for loader, module_name, _ in pkgutil.walk_packages(custom_modules): + if include_modules and module_name not in include_modules: + continue + + if module_name != "base": + module = importlib.import_module(module_name, custom_modules[0]) + module.log.setLevel(log.level) + if module.detector.entrypoint == entrypoint: + _modules.append(module) + + """ + for loader, module_name, _ in pkgutil.walk_packages([custom_modules_path]): + + custom_modules_path = os.path.abspath("custom/") + sys.path.append(custom_modules_path); + module = importlib.import_module( + module_name, custom_modules_path + ) + """ log.info("Found %s detection modules", len(_modules)) return _modules From db5db7cc8f7b928281ff698879dfa0b11faa674e Mon Sep 17 00:00:00 2001 From: e-ngo Date: Tue, 30 Jul 2019 18:42:53 -0700 Subject: [PATCH 6/9] Removed unnecessary comments and cleaned up code --- mythril/analysis/security.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/mythril/analysis/security.py b/mythril/analysis/security.py index 5c5e520a..ed741ecb 100644 --- a/mythril/analysis/security.py +++ b/mythril/analysis/security.py @@ -75,28 +75,20 @@ def get_detection_modules(entrypoint, include_modules=(), custom_modules_directo if module.detector.entrypoint == entrypoint: _modules.append(module) if custom_modules_directory: - custom_modules = [os.path.abspath(custom_modules_directory)] - sys.path.append(custom_modules_directory) + custom_modules_path = os.path.abspath(custom_modules_directory) + if custom_modules_path not in sys.path: + sys.path.append(custom_modules_path) - for loader, module_name, _ in pkgutil.walk_packages(custom_modules): + for loader, module_name, _ in pkgutil.walk_packages([custom_modules_path]): if include_modules and module_name not in include_modules: continue if module_name != "base": - module = importlib.import_module(module_name, custom_modules[0]) + module = importlib.import_module(module_name, custom_modules_path) module.log.setLevel(log.level) if module.detector.entrypoint == entrypoint: _modules.append(module) - """ - for loader, module_name, _ in pkgutil.walk_packages([custom_modules_path]): - - custom_modules_path = os.path.abspath("custom/") - sys.path.append(custom_modules_path); - module = importlib.import_module( - module_name, custom_modules_path - ) - """ log.info("Found %s detection modules", len(_modules)) return _modules From db449a32d86266f5ee1055ac31ee7b2fb9ea8735 Mon Sep 17 00:00:00 2001 From: e-ngo <52668908+e-ngo@users.noreply.github.com> Date: Thu, 1 Aug 2019 05:48:10 -0700 Subject: [PATCH 7/9] Added ProgramCounterException to display a clearer message when pc is out of bounds. (#1180) --- mythril/laser/ethereum/evm_exceptions.py | 6 ++++++ mythril/laser/ethereum/state/global_state.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/mythril/laser/ethereum/evm_exceptions.py b/mythril/laser/ethereum/evm_exceptions.py index 63460e2b..fd99aed5 100644 --- a/mythril/laser/ethereum/evm_exceptions.py +++ b/mythril/laser/ethereum/evm_exceptions.py @@ -35,3 +35,9 @@ class OutOfGasException(VmException): """A VM exception denoting the current execution has run out of gas.""" pass + + +class ProgramCounterException(VmException): + """A VM exception denoting an invalid PC value (No stop instruction is reached).""" + + pass diff --git a/mythril/laser/ethereum/state/global_state.py b/mythril/laser/ethereum/state/global_state.py index 86364a69..0c43980b 100644 --- a/mythril/laser/ethereum/state/global_state.py +++ b/mythril/laser/ethereum/state/global_state.py @@ -9,6 +9,7 @@ from mythril.laser.ethereum.cfg import Node from mythril.laser.ethereum.state.environment import Environment from mythril.laser.ethereum.state.machine_state import MachineState from mythril.laser.ethereum.state.annotation import StateAnnotation +from mythril.laser.ethereum.evm_exceptions import ProgramCounterException if TYPE_CHECKING: from mythril.laser.ethereum.state.world_state import WorldState @@ -88,6 +89,10 @@ class GlobalState: """ instructions = self.environment.code.instruction_list + if self.mstate.pc >= len(instructions): + raise ProgramCounterException( + "PC: {} can not be reached.".format(self.mstate.pc) + ) return instructions[self.mstate.pc] @property From 838de40858421130b3313c71a0f8238571becdcf Mon Sep 17 00:00:00 2001 From: NeolithEra <3226592650@qq.com> Date: Thu, 1 Aug 2019 21:16:27 +0800 Subject: [PATCH 8/9] Fix dependency conflict for issue (#1181) * Fix dependency conflict for issue * Fix dependency conflict for issue --- requirements.txt | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1b49350c..11382df3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ eth-keys>=0.2.0b3,<0.3.0 eth-rlp>=0.1.0 eth-tester==0.1.0b32 eth-typing>=2.0.0 -eth-utils>=1.0.1 jinja2>=2.9 mock persistent>=4.2.0 diff --git a/setup.py b/setup.py index d76f546e..ce115193 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ REQUIRED = [ "py-solc", "plyvel", "eth_abi==1.3.0", - "eth-utils>=1.0.1", "eth-account>=0.1.0a2,<=0.3.0", "eth-hash>=0.1.0", "eth-keyfile>=0.5.1", From 8ada755aa7890cdddab66c1401b7ca3349dfa51b Mon Sep 17 00:00:00 2001 From: Nathan Date: Thu, 1 Aug 2019 10:51:01 -0700 Subject: [PATCH 9/9] Minor grammar improvements --- mythril/interfaces/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mythril/interfaces/cli.py b/mythril/interfaces/cli.py index 486b5aaf..8962fa16 100644 --- a/mythril/interfaces/cli.py +++ b/mythril/interfaces/cli.py @@ -398,8 +398,8 @@ def create_analyzer_parser(analyzer_parser: ArgumentParser): ) options.add_argument( "--custom-modules-directory", - help="designates a separate directory to search for security modules from", - metavar="PATH_TO_CUSTOM_MODULES_DIRECTORY", + help="designates a separate directory to search for custom analysis modules", + metavar="CUSTOM_MODULES_DIRECTORY", )