make sure use same signatures.json in every test

all_tests.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
 
+python --version
+echo "Please make sure you are using python 3.6.x"
+
 rm -rf ./tests/testdata/outputs_current/
 mkdir -p ./tests/testdata/outputs_current/
 python3 -m unittest discover -p "*_test.py"

coverage_report.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
 
+python --version
+echo "Please make sure you are using python 3.6.x"
+
 rm -rf ./tests/testdata/outputs_current/
 mkdir -p ./tests/testdata/outputs_current/
 rm -rf coverage_html_report

tests/__init__.py

@@ -1,5 +1,7 @@
 from pathlib import Path
+from unittest import TestCase
 import os
+import shutil
 
 TESTS_DIR = Path(__file__).parent
 PROJECT_DIR = TESTS_DIR.parent
@@ -7,8 +9,30 @@ TESTDATA = TESTS_DIR / "testdata"
 TESTDATA_INPUTS = TESTDATA / "inputs"
 TESTDATA_OUTPUTS_EXPECTED = TESTDATA / "outputs_expected"
 TESTDATA_OUTPUTS_CURRENT = TESTDATA / "outputs_current"
+MYTHRIL_DIR = TESTS_DIR / "mythril_dir"
 
-os.environ['MYTHRIL_DIR'] = str(TESTS_DIR / "mythril_dir")
 
-def compare_files_error_message(expected, current):
+class BaseTestCase(TestCase):
-    return "Expected output changes, compare the following files to see differences: \n- {}\n- {}\n".format(str(expected), str(current))
+
+    def setUp(self):
+        self.changed_files = []
+        self.ori_mythril_dir = getattr(os.environ, 'MYTHRIL_DIR', '')
+        os.environ['MYTHRIL_DIR'] = str(MYTHRIL_DIR)
+        shutil.copyfile(str(MYTHRIL_DIR / "signatures.json.example"), str(MYTHRIL_DIR / "signatures.json"))
+
+    def tearDown(self):
+        os.environ['MYTHRIL_DIR'] = self.ori_mythril_dir
+        os.remove(str(MYTHRIL_DIR / "signatures.json"))
+
+    def compare_files_error_message(self):
+        message = "Following output files are changed, compare them manually to see differences:"
+        for (input_file, expected, current) in self.changed_files:
+            message += "{}:\n".format(input_file.name)
+            message += "- {}\n".format(str(expected))
+            message += "- {}\n".format(str(current))
+
+    def found_changed_files(self, input_file, output_expected, output_current):
+        self.changed_files.append((input_file, output_expected, output_current))
+
+    def assert_and_show_changed_files(self):
+        self.assertEqual(self.changed_files, [], self.compare_files_error_message())

tests/cmd_line_test.py

@@ -1,4 +1,3 @@
-from unittest import TestCase
 from subprocess import check_output
 from tests import *
 
@@ -7,7 +6,7 @@ MYTH = str(PROJECT_DIR / "myth")
 def output_of(command):
     return check_output(command, shell=True).decode("UTF-8")
 
-class CommandLineToolTestCase(TestCase):
+class CommandLineToolTestCase(BaseTestCase):
 
     def test_disassemble_code_correctly(self):
         command = "python3 {} MYTH -d -c 0x5050".format(MYTH)
@@ -22,14 +21,14 @@ class CommandLineToolTestCase(TestCase):
         command = "python3 {} --hash 'setOwner(address)'".format(MYTH)
         self.assertEqual('0x13af4035\n', output_of(command))
 
-class TruffleTestCase(TestCase):
+class TruffleTestCase(BaseTestCase):
 
     def test_analysis_truffle_project(self):
         truffle_project_root = str(TESTS_DIR / "truffle_project")
         command = "cd {}; truffle compile; python3 {} --truffle".format(truffle_project_root, MYTH)
         self.assertIn("In the function 'withdrawfunds()' a non-zero amount of Ether is sent to msg.sender.", output_of(command))
 
-class InfuraTestCase(TestCase):
+class InfuraTestCase(BaseTestCase):
 
     def test_infura_mainnet(self):
         command = "python3 {} --rpc infura-mainnet -d -a 0x2a0c0dbecc7e4d658f48e01e3fa353f44050c208".format(MYTH)

tests/contractstorage_test.py

@@ -1,17 +1,18 @@
-import unittest
 from mythril.ether.contractstorage import get_persistent_storage
 import os
+from tests import BaseTestCase
 
-class GetAndSearchContractTestCase(unittest.TestCase):
+class GetAndSearchContractTestCase(BaseTestCase):
 
     def setUp(self):
+        super(GetAndSearchContractTestCase, self).setUp()
         script_path = os.path.dirname(os.path.realpath(__file__))
         storage_dir = os.path.join(script_path, 'teststorage')
         self.storage, self.db = get_persistent_storage(storage_dir)
-        pass
 
     def tearDown(self):
         self.db.close()
+        super(GetAndSearchContractTestCase, self).tearDown()
 
     def mockCallback(self, code_hash, code, addresses, balances):
         self.code_hash = code_hash

tests/graph_test.py

@@ -1,12 +1,10 @@
-from unittest import TestCase
-
 from mythril.analysis.callgraph import generate_graph
 from mythril.analysis.symbolic import SymExecWrapper
 from mythril.ether import util
 from mythril.ether.soliditycontract import SolidityContract
 from tests import *
 
-class GraphTest(TestCase):
+class GraphTest(BaseTestCase):
 
     def test_generate_graph(self):
         for input_file in TESTDATA_INPUTS.iterdir():
@@ -19,4 +17,7 @@ class GraphTest(TestCase):
             html = generate_graph(sym)
             output_current.write_text(html)
 
-            self.assertEqual(output_expected.read_text(), output_current.read_text(), compare_files_error_message(output_expected, output_current))
+            if not (output_expected.read_text() == output_expected.read_text()):
+                self.found_changed_files(input_file, output_expected, output_current)
+
+        self.assert_and_show_changed_files()

tests/mythril_dir/signatures.json

@@ -1 +0,0 @@
-{"0x07f9f7ba": "StandardBounties(address)", "0x8c590917": "contribute(uint256,uint256)", "0x626a413a": "activateBounty(uint256,uint256)", "0x1e688c14": "fulfillBounty(uint256,string)", "0x41ac5dd0": "updateFulfillment(uint256,uint256,string)", "0xd9583497": "acceptFulfillment(uint256,uint256)", "0x16b57509": "killBounty(uint256)", "0x2d1fdef6": "extendDeadline(uint256,uint256)", "0x5d19606e": "transferIssuer(uint256,address)", "0xd6c0ceab": "changeBountyDeadline(uint256,uint256)", "0xf3d3402a": "changeBountyData(uint256,string)", "0x452ccadb": "changeBountyFulfillmentAmount(uint256,uint256)", "0xcdad6576": "changeBountyArbiter(uint256,address)", "0x992a3e75": "changeBountyPaysTokens(uint256,bool,address)", "0x422d4cd6": "increasePayout(uint256,uint256,uint256)", "0xb94b0a3a": "getFulfillment(uint256,uint256)", "0xee8c4bbf": "getBounty(uint256)", "0x86647bac": "getBountyArbiter(uint256)", "0xa60745aa": "getBountyData(uint256)", "0x19dba3d2": "getBountyToken(uint256)", "0x3278ba2f": "getNumBounties()", "0xfbe334f8": "getNumFulfillments(uint256)", "0xdb3b6263": "transitionToState(uint256,BountyStages)", "0x4e3b52fe": "metaCoin()", "0x412664ae": "sendToken(address,uint256)", "0x56885cd8": "crowdfunding()", "0x6c343ffe": "withdrawfunds()", "0xe8b5e51f": "invest()", "0xaa3288f4": "getBalance())", "0xc11a4b47": "Origin()", "0xf2fde38b": "transferOwnership(address)", "0x00362a95": "donate(address)", "0x70a08231": "balanceOf(address)", "0x2e1a7d4d": "withdraw(uint256)", "0x6241bfd1": "Token(uint256)", "0xa3210e87": "sendeth(address,uint256)", "0xcd38aa87": "chooseWinner()", "0xd6d22fa4": "MetaCoin()", "0x90b98a11": "sendCoin(address,uint256)", "0x7bd703e8": "getBalanceInEth(address)", "0xf8b2cb4f": "getBalance(address)", "0xa360b26f": "Migrations()", "0xfdacd576": "setCompleted(uint256)", "0x0900f010": "upgrade(address)", "0xcae9ca51": "approveAndCall(address,uint256,bytes)", "0xa9059cbb": "transfer(address,uint256)", "0x23b872dd": "transferFrom(address,address,uint256)", "0x095ea7b3": "approve(address,uint256)", "0xdd62ed3e": "allowance(address,address)", "0x525f8a5c": "setSaleStartTime(uint256)", "0xd132391a": "setSaleEndTime(uint256)", "0x0a0cd8c8": "setupDone()", "0xd7bb99ba": "contribute()", "0xf0349d5f": "setupStages()", "0x2a4f6533": "createTokenContract())", "0x42a6b21a": "getContributionLimit(address)", "0x1a787915": "startConditions(bytes32)", "0xf3fde261": "onTransition(bytes32)", "0x27816235": "onSaleEnded()", "0x091cde0b": "DisbursementHandler(address)", "0xf3fef3a3": "withdraw(address,uint256)", "0x4bc9fdc2": "calcMaxWithdraw()", "0xc9e61599": "createTarget())", "0x200094e0": "deployContract())", "0x5a048d78": "claim(Target)", "0x16ae6b67": "checkInvariant())", "0x2aa5ed61": "DayLimit(uint256)", "0xe7dde9a3": "_setDailyLimit(uint256)", "0x4a4c82c6": "_resetSpentToday()", "0x180aadb7": "underLimit(uint256)", "0x9d4468ff": "today())", "0x19045a25": "recover(bytes32,bytes)", "0xe92dfb23": "LimitBalance(uint256)", "0xd73dd623": "increaseApproval(address,uint256)", "0x66188463": "decreaseApproval(address,uint256)", "0xabaf5880": "Crowdsale(uint256,uint256,uint256,address)", "0xec8ac4d8": "buyTokens(address)", "0x9d735286": "forwardFunds()", "0x605120cf": "validPurchase())", "0x6e42787f": "hasEnded())", "0xe5c46944": "MultiSigWallet(address[],uint256)", "0x7065cb48": "addOwner(address)", "0x173825d9": "removeOwner(address)", "0xe20056e6": "replaceOwner(address,address)", "0xba51a6df": "changeRequirement(uint256)", "0xc6427474": "submitTransaction(address,uint256,bytes)", "0xc01a8c84": "confirmTransaction(uint256)", "0x20ea8d86": "revokeConfirmation(uint256)", "0xee22610b": "executeTransaction(uint256)", "0x784547a7": "isConfirmed(uint256)", "0xec096f8d": "addTransaction(address,uint256,bytes)", "0x8b51d13f": "getConfirmationCount(uint256)", "0x54741525": "getTransactionCount(bool,bool)", "0xa0e67e2b": "getOwners()", "0xb5dc40c3": "getConfirmations(uint256)", "0xa8abe69a": "getTransactionIds(uint256,uint256,bool,bool)"}

tests/mythril_dir/signatures.json.example

@@ -0,0 +1 @@
+{"0xd132391a": "setSaleEndTime(uint256)", "0xdb3b6263": "transitionToState(uint256,BountyStages)", "0xd7bb99ba": "contribute()", "0x605120cf": "validPurchase())", "0x2aa5ed61": "DayLimit(uint256)", "0x412664ae": "sendToken(address,uint256)", "0xfdacd576": "setCompleted(uint256)", "0x5a048d78": "claim(Target)", "0xf3fef3a3": "withdraw(address,uint256)", "0xba51a6df": "changeRequirement(uint256)", "0xec8ac4d8": "buyTokens(address)", "0xa0e67e2b": "getOwners()", "0x42a6b21a": "getContributionLimit(address)", "0x00362a95": "donate(address)", "0x422d4cd6": "increasePayout(uint256,uint256,uint256)", "0x86647bac": "getBountyArbiter(uint256)", "0xf3fde261": "onTransition(bytes32)", "0x8b51d13f": "getConfirmationCount(uint256)", "0x180aadb7": "underLimit(uint256)", "0xe20056e6": "replaceOwner(address,address)", "0xb94b0a3a": "getFulfillment(uint256,uint256)", "0xa3210e87": "sendeth(address,uint256)", "0xf8b2cb4f": "getBalance(address)", "0x66188463": "decreaseApproval(address,uint256)", "0xb5dc40c3": "getConfirmations(uint256)", "0xf3d3402a": "changeBountyData(uint256,string)", "0xe7dde9a3": "_setDailyLimit(uint256)", "0xd9583497": "acceptFulfillment(uint256,uint256)", "0x0a0cd8c8": "setupDone()", "0x7bd703e8": "getBalanceInEth(address)", "0xd6d22fa4": "MetaCoin()", "0x2e1a7d4d": "withdraw(uint256)", "0xe5c46944": "MultiSigWallet(address[],uint256)", "0x2a4f6533": "createTokenContract())", "0x6241bfd1": "Token(uint256)", "0x5d19606e": "transferIssuer(uint256,address)", "0x6c343ffe": "withdrawfunds()", "0x19045a25": "recover(bytes32,bytes)", "0x200094e0": "deployContract())", "0x1e688c14": "fulfillBounty(uint256,string)", "0x16b57509": "killBounty(uint256)", "0xaa3288f4": "getBalance())", "0xcae9ca51": "approveAndCall(address,uint256,bytes)", "0x70a08231": "balanceOf(address)", "0x56885cd8": "crowdfunding()", "0xc01a8c84": "confirmTransaction(uint256)", "0x9d4468ff": "today())", "0x2d1fdef6": "extendDeadline(uint256,uint256)", "0x4a4c82c6": "_resetSpentToday()", "0x992a3e75": "changeBountyPaysTokens(uint256,bool,address)", "0x4bc9fdc2": "calcMaxWithdraw()", "0xdd62ed3e": "allowance(address,address)", "0xc6427474": "submitTransaction(address,uint256,bytes)", "0x16ae6b67": "checkInvariant())", "0x07f9f7ba": "StandardBounties(address)", "0x20ea8d86": "revokeConfirmation(uint256)", "0xcdad6576": "changeBountyArbiter(uint256,address)", "0xe92dfb23": "LimitBalance(uint256)", "0xa8abe69a": "getTransactionIds(uint256,uint256,bool,bool)", "0x23b872dd": "transferFrom(address,address,uint256)", "0x8c590917": "contribute(uint256,uint256)", "0xc11a4b47": "Origin()", "0xf2fde38b": "transferOwnership(address)", "0x9d735286": "forwardFunds()", "0x173825d9": "removeOwner(address)", "0xd6c0ceab": "changeBountyDeadline(uint256,uint256)", "0x1a787915": "startConditions(bytes32)", "0xa360b26f": "Migrations()", "0xec096f8d": "addTransaction(address,uint256,bytes)", "0xee8c4bbf": "getBounty(uint256)", "0x90b98a11": "sendCoin(address,uint256)", "0xabaf5880": "Crowdsale(uint256,uint256,uint256,address)", "0xc9e61599": "createTarget())", "0x7065cb48": "addOwner(address)", "0x452ccadb": "changeBountyFulfillmentAmount(uint256,uint256)", "0x626a413a": "activateBounty(uint256,uint256)", "0xf0349d5f": "setupStages()", "0xd73dd623": "increaseApproval(address,uint256)", "0xfbe334f8": "getNumFulfillments(uint256)", "0x19dba3d2": "getBountyToken(uint256)", "0x525f8a5c": "setSaleStartTime(uint256)", "0xe8b5e51f": "invest()", "0xa9059cbb": "transfer(address,uint256)", "0x6e42787f": "hasEnded())", "0x091cde0b": "DisbursementHandler(address)", "0x4e3b52fe": "metaCoin()", "0xee22610b": "executeTransaction(uint256)", "0x54741525": "getTransactionCount(bool,bool)", "0x0900f010": "upgrade(address)", "0xcd38aa87": "chooseWinner()", "0xa60745aa": "getBountyData(uint256)", "0x095ea7b3": "approve(address,uint256)", "0x41ac5dd0": "updateFulfillment(uint256,uint256,string)", "0x784547a7": "isConfirmed(uint256)", "0x27816235": "onSaleEnded()", "0x3278ba2f": "getNumBounties()"}

tests/report_test.py

@@ -1,5 +1,3 @@
-from unittest import TestCase
-
 from mythril.analysis.report import Report
 from mythril.analysis.security import fire_lasers
 from mythril.analysis.symbolic import SymExecWrapper
@@ -30,7 +28,7 @@ def _generate_report(input_file):
 
     return report
 
-class AnalysisReportTest(TestCase):
+class AnalysisReportTest(BaseTestCase):
 
     def test_json_reports(self):
         for input_file in TESTDATA_INPUTS.iterdir():
@@ -38,10 +36,12 @@ class AnalysisReportTest(TestCase):
             output_current = TESTDATA_OUTPUTS_CURRENT / (input_file.name + ".json")
 
             report = _generate_report(input_file)
-
             output_current.write_text(_fix_path(_fix_debug_data(report.as_json())).strip())
 
-            self.assertEqual(output_expected.read_text(), output_expected.read_text(), compare_files_error_message(output_expected, output_current))
+            if not (output_expected.read_text() == output_expected.read_text()):
+                self.found_changed_files(input_file, output_expected, output_current)
+
+        self.assert_and_show_changed_files()
 
     def test_markdown_reports(self):
         for input_file in TESTDATA_INPUTS.iterdir():
@@ -49,10 +49,12 @@ class AnalysisReportTest(TestCase):
             output_current = TESTDATA_OUTPUTS_CURRENT / (input_file.name + ".markdown")
 
             report = _generate_report(input_file)
-
             output_current.write_text(_fix_path(report.as_markdown()))
 
-            self.assertEqual(output_expected.read_text(), output_current.read_text(), compare_files_error_message(output_expected, output_current))
+            if not (output_expected.read_text() == output_expected.read_text()):
+                self.found_changed_files(input_file, output_expected, output_current)
+
+        self.assert_and_show_changed_files()
 
     def test_text_reports(self):
         for input_file in TESTDATA_INPUTS.iterdir():
@@ -60,7 +62,9 @@ class AnalysisReportTest(TestCase):
             output_current = TESTDATA_OUTPUTS_CURRENT / (input_file.name + ".text")
 
             report = _generate_report(input_file)
-
             output_current.write_text(_fix_path(report.as_text()))
 
-            self.assertEqual(output_expected.read_text(), output_current.read_text(), compare_files_error_message(output_expected, output_current))
+            if not (output_expected.read_text() == output_expected.read_text()):
+                self.found_changed_files(input_file, output_expected, output_current)
+
+        self.assert_and_show_changed_files()

tests/solidity_contract_test.py

@@ -1,11 +1,11 @@
-from unittest import TestCase
 from pathlib import Path
 
 from mythril.ether.soliditycontract import SolidityContract
+from tests import BaseTestCase
 
 TEST_FILES = Path(__file__).parent / "testdata/inputs"
 
-class SolidityContractTest(TestCase):
+class SolidityContractTest(BaseTestCase):
 
     def test_get_source_info_without_name_gets_latest_contract_info(self):
         input_file = TEST_FILES / "multi_contracts.sol"