Merge branch 'develop' of github.com:ConsenSys/mythril into develop

.circleci/config.yml

@@ -48,6 +48,7 @@ jobs:
           name: Unit-testing
           command: tox
           working_directory: /home/mythril
+          no_output_timeout: 10m
           environment:
             LC_ALL: en_US.ASCII
             LANG: en_US.ASCII

mythril/laser/ethereum/call.py

@@ -159,26 +159,14 @@ def get_call_data(
     state = global_state.mstate
     transaction_id = "{}_internalcall".format(global_state.current_transaction.id)
     try:
-        # TODO: This only allows for either fully concrete or fully symbolic calldata.
-        # Improve management of memory and callata to support a mix between both types.
         calldata_from_mem = state.memory[
             util.get_concrete_int(memory_start) : util.get_concrete_int(
                 memory_start + memory_size
             )
         ]
         i = 0
-        starting_calldata = []
-        while i < len(calldata_from_mem):
-            elem = calldata_from_mem[i]
-            if isinstance(elem, int):
-                starting_calldata.append(elem)
-                i += 1
-            else:  # BitVec
-                for j in range(0, elem.size(), 8):
-                    starting_calldata.append(Extract(j + 7, j, elem))
-                    i += 1
-
-        call_data = ConcreteCalldata(transaction_id, starting_calldata)
+
+        call_data = ConcreteCalldata(transaction_id, calldata_from_mem)
         call_data_type = CalldataType.CONCRETE
         logging.debug("Calldata: " + str(call_data))
     except TypeError:

mythril/laser/ethereum/instructions.py

@@ -510,7 +510,7 @@ class Instruction:
                 + ": + "
                 + str(size)
                 + "]",
-                256,
+                8,
             )
             return [global_state]
 
@@ -533,7 +533,7 @@ class Instruction:
                     + ": + "
                     + str(size)
                     + "]",
-                    256,
+                    8,
                 )
                 return [global_state]
 
@@ -561,7 +561,7 @@ class Instruction:
                     + ": + "
                     + str(size)
                     + "]",
-                    256,
+                    8,
                 )
         return [global_state]
 
@@ -680,7 +680,7 @@ class Instruction:
                 "code({})".format(
                     global_state.environment.active_account.contract_name
                 ),
-                256,
+                8,
             )
             return [global_state]
 
@@ -696,7 +696,7 @@ class Instruction:
                     "code({})".format(
                         global_state.environment.active_account.contract_name
                     ),
-                    256,
+                    8,
                 )
             return [global_state]
 
@@ -713,7 +713,7 @@ class Instruction:
                     "code({})".format(
                         global_state.environment.active_account.contract_name
                     ),
-                    256,
+                    8,
                 )
                 return [global_state]
 
@@ -734,7 +734,7 @@ class Instruction:
                     "code({})".format(
                         global_state.environment.active_account.contract_name
                     ),
-                    256,
+                    8,
                 )
 
         return [global_state]
@@ -838,12 +838,9 @@ class Instruction:
             data = global_state.new_bitvec("mem[" + str(simplify(op0)) + "]", 256)
             state.stack.append(data)
             return [global_state]
-        try:
-            state.mem_extend(offset, 32)
-            data = util.concrete_int_from_bytes(state.memory, offset)
-        except TypeError:  # Symbolic memory
-            # TODO: Handle this properly
-            data = state.memory[offset]
+
+        state.mem_extend(offset, 32)
+        data = state.memory.get_word_at(offset)
 
         logging.debug("Load from memory[" + str(offset) + "]: " + str(data))
 
@@ -870,16 +867,7 @@ class Instruction:
 
         logging.debug("MSTORE to mem[" + str(mstart) + "]: " + str(value))
 
-        try:
-            # Attempt to concretize value
-            _bytes = util.concrete_int_to_bytes(value)
-            assert len(_bytes) == 32
-            state.memory[mstart : mstart + 32] = _bytes
-        except:
-            try:
-                state.memory[mstart] = value
-            except TypeError:
-                logging.debug("Invalid memory access")
+        state.memory.write_word_at(mstart, value)
 
         return [global_state]
 
@@ -896,7 +884,14 @@ class Instruction:
 
         state.mem_extend(offset, 1)
 
-        state.memory[offset] = value % 256
+        try:
+            value_to_write = util.get_concrete_int(value) ^ 0xFF
+        except TypeError:  # BitVec
+            value_to_write = Extract(7, 0, value)
+        logging.debug("MSTORE8 to mem[" + str(offset) + "]: " + str(value_to_write))
+
+        state.memory[offset] = value_to_write
+
         return [global_state]
 
     @StateTransition()
@@ -1186,7 +1181,7 @@ class Instruction:
     def return_(self, global_state: GlobalState):
         state = global_state.mstate
         offset, length = state.stack.pop(), state.stack.pop()
-        return_data = [global_state.new_bitvec("return_data", 256)]
+        return_data = [global_state.new_bitvec("return_data", 8)]
         try:
             return_data = state.memory[
                 util.get_concrete_int(offset) : util.get_concrete_int(offset + length)
@@ -1230,7 +1225,7 @@ class Instruction:
     def revert_(self, global_state: GlobalState) -> None:
         state = global_state.mstate
         offset, length = state.stack.pop(), state.stack.pop()
-        return_data = [global_state.new_bitvec("return_data", 256)]
+        return_data = [global_state.new_bitvec("return_data", 8)]
         try:
             return_data = state.memory[
                 util.get_concrete_int(offset) : util.get_concrete_int(offset + length)
@@ -1312,7 +1307,7 @@ class Instruction:
                         + "("
                         + str(call_data)
                         + ")",
-                        256,
+                        8,
                     )
                 return [global_state]
 
@@ -1320,7 +1315,6 @@ class Instruction:
                 min(len(data), mem_out_sz)
             ):  # If more data is used then it's chopped off
                 global_state.mstate.memory[mem_out_start + i] = data[i]
-
             # TODO: maybe use BitVec here constrained to 1
             return [global_state]
 

mythril/laser/ethereum/state/machine_state.py

@@ -10,6 +10,7 @@ from mythril.laser.ethereum.evm_exceptions import (
     OutOfGasException,
 )
 from mythril.laser.ethereum.state.constraints import Constraints
+from mythril.laser.ethereum.state.memory import Memory
 
 
 class MachineStack(list):
@@ -88,7 +89,7 @@ class MachineState:
         """ Constructor for machineState """
         self.pc = pc
         self.stack = MachineStack(stack)
-        self.memory = memory or []
+        self.memory = memory or Memory()
         self.gas_limit = gas_limit
         self.min_gas_used = min_gas_used  # lower gas usage bound
         self.max_gas_used = max_gas_used  # upper gas usage bound
@@ -128,7 +129,7 @@ class MachineState:
             self.min_gas_used += extend_gas
             self.max_gas_used += extend_gas
             self.check_gas()
-            self.memory.extend(bytearray(m_extend))
+            self.memory.extend(m_extend)
 
     def memory_write(self, offset: int, data: List[int]) -> None:
         """ Writes data to memory starting at offset """

mythril/laser/ethereum/state/memory.py

@@ -0,0 +1,104 @@
+from typing import Union
+from z3 import BitVecRef, Extract, BitVecVal, If, BoolRef, Z3Exception, simplify, Concat
+
+from mythril.laser.ethereum import util
+
+
+class Memory:
+    def __init__(self):
+        self._memory = []
+
+    def __len__(self):
+        return len(self._memory)
+
+    def extend(self, size):
+        self._memory.extend(bytearray(size))
+
+    def get_word_at(self, index: int) -> Union[int, BitVecRef]:
+        """
+        Access a word from a specified memory index
+        :param index: integer representing the index to access
+        :return: 32 byte word at the specified index
+        """
+        try:
+            return util.concrete_int_from_bytes(
+                bytes([util.get_concrete_int(b) for b in self[index : index + 32]]), 0
+            )
+        except TypeError:
+            result = simplify(
+                Concat(
+                    [
+                        b if isinstance(b, BitVecRef) else BitVecVal(b, 8)
+                        for b in self[index : index + 32]
+                    ]
+                )
+            )
+            assert result.size() == 256
+            return result
+
+    def write_word_at(
+        self, index: int, value: Union[int, BitVecRef, bool, BoolRef]
+    ) -> None:
+        """
+        Writes a 32 byte word to memory at the specified index`
+        :param index: index to write to
+        :param value: the value to write to memory
+        """
+        try:
+            # Attempt to concretize value
+            if isinstance(value, bool):
+                _bytes = (
+                    int(1).to_bytes(32, byteorder="big")
+                    if value
+                    else int(0).to_bytes(32, byteorder="big")
+                )
+            else:
+                _bytes = util.concrete_int_to_bytes(value)
+            assert len(_bytes) == 32
+            self[index : index + 32] = _bytes
+        except (Z3Exception, AttributeError):  # BitVector or BoolRef
+            if isinstance(value, BoolRef):
+                value_to_write = If(value, BitVecVal(1, 256), BitVecVal(0, 256))
+            else:
+                value_to_write = value
+            assert value_to_write.size() == 256
+
+            for i in range(0, value_to_write.size(), 8):
+                self[index + 31 - (i // 8)] = Extract(i + 7, i, value_to_write)
+
+    def __getitem__(self, item: Union[int, slice]) -> Union[BitVecRef, int, list]:
+        if isinstance(item, slice):
+            start, step, stop = item.start, item.step, item.stop
+            if start is None:
+                start = 0
+            if stop is None:  # 2**256 is just a bit too big
+                raise IndexError("Invalid Memory Slice")
+            if step is None:
+                step = 1
+            return [self[i] for i in range(start, stop, step)]
+
+        try:
+            return self._memory[item]
+        except IndexError:
+            return 0
+
+    def __setitem__(self, key: Union[int, slice], value: Union[BitVecRef, int, list]):
+        if isinstance(key, slice):
+            start, step, stop = key.start, key.step, key.stop
+
+            if start is None:
+                start = 0
+            if stop is None:
+                raise IndexError("Invalid Memory Slice")
+            if step is None:
+                step = 1
+
+            for i in range(0, stop - start, step):
+                self[start + i] = value[i]
+
+        else:
+            if isinstance(value, int):
+                assert 0 <= value <= 0xFF
+            if isinstance(value, BitVecRef):
+                assert value.size() == 8
+            self._memory[key] = value

requirements.txt

@@ -1,7 +1,7 @@
 coloredlogs>=10.0
 configparser>=3.5.0
 coverage
-eth_abi>=1.0.0
+eth_abi==1.3.0
 eth-account>=0.1.0a2
 ethereum>=2.3.2
 ethereum-input-decoder>=0.2.2
@@ -10,7 +10,7 @@ eth-keyfile>=0.5.1
 eth-keys>=0.2.0b3
 eth-rlp>=0.1.0
 eth-tester>=0.1.0b21
-eth-typing<2.0.0,>=1.3.0
+eth-typing>=2.0.0
 eth-utils>=1.0.1
 jinja2>=2.9
 mock

setup.py

@@ -79,7 +79,7 @@ setup(
         "requests",
         "py-solc",
         "plyvel",
-        "eth_abi>=1.0.0",
+        "eth_abi==1.3.0",
         "eth-utils>=1.0.1",
         "eth-account>=0.1.0a2",
         "eth-hash>=0.1.0",
@@ -87,7 +87,7 @@ setup(
         "eth-keys>=0.2.0b3",
         "eth-rlp>=0.1.0",
         "eth-tester>=0.1.0b21",
-        "eth-typing>=1.3.0,<2.0.0",
+        "eth-typing>=2.0.0",
         "coverage",
         "jinja2>=2.9",
         "rlp>=1.0.1",

tests/laser/state/mstate_test.py

@@ -1,6 +1,9 @@
 import pytest
+from z3 import BitVec, simplify
+
 from mythril.laser.ethereum.state.machine_state import MachineState
 from mythril.laser.ethereum.evm_exceptions import StackUnderflowException
+from mythril.laser.ethereum.state.memory import Memory
 
 memory_extension_test_data = [(0, 0, 10), (0, 30, 10), (100, 22, 8)]
 
@@ -11,7 +14,8 @@ memory_extension_test_data = [(0, 0, 10), (0, 30, 10), (100, 22, 8)]
 def test_memory_extension(initial_size, start, extension_size):
     # Arrange
     machine_state = MachineState(gas_limit=8000000)
-    machine_state.memory = [0] * initial_size
+    machine_state.memory = Memory()
+    machine_state.memory.extend(initial_size)
 
     # Act
     machine_state.mem_extend(start, extension_size)
@@ -81,18 +85,39 @@ def test_stack_single_pop():
     assert isinstance(result, int)
 
 
-memory_write_test_data = [(5, 10, [1, 2, 3]), (0, 0, [3, 4]), (20, 1, [2, 4, 10])]
+def test_memory_zeroed():
+    # Arrange
+    mem = Memory()
+    mem.extend(2000 + 32)
+
+    # Act
+    mem[11] = 10
+    mem.write_word_at(2000, 0x12345)
+
+    # Assert
+    assert mem[10] == 0
+    assert mem[100] == 0
+    assert mem.get_word_at(1000) == 0
 
 
-@pytest.mark.parametrize("initial_size, memory_offset, data", memory_write_test_data)
-def test_memory_write(initial_size, memory_offset, data):
+def test_memory_write():
     # Arrange
-    machine_state = MachineState(8000000)
-    machine_state.memory = [0] * initial_size
+    mem = Memory()
+    mem.extend(200 + 32)
+
+    a = BitVec("a", 256)
+    b = BitVec("b", 8)
 
     # Act
-    machine_state.memory_write(memory_offset, data)
+    mem[11] = 10
+    mem[12] = b
+    mem.write_word_at(200, 0x12345)
+    mem.write_word_at(100, a)
 
     # Assert
-    assert len(machine_state.memory) == max(initial_size, memory_offset + len(data))
-    assert machine_state.memory[memory_offset : memory_offset + len(data)] == data
+    assert mem[0] == 0
+    assert mem[11] == 10
+    assert mem[200 + 31] == 0x45
+    assert mem.get_word_at(200) == 0x12345
+    assert simplify(a == mem.get_word_at(100))
+    assert simplify(b == mem[12])

tests/native_tests.sol

@@ -14,11 +14,15 @@ contract Caller {
     constructor (address addr) public {
         fixedAddress = addr;
     }
+    /*
+    // Commented out because this causes laser to enter an infinite loop... :/
+    // It sets the free memory pointer to a symbolic value, and things break
 
     //some typical function as a decoy
     function thisisfine() public {
         (bool success, bytes memory mem) = fixedAddress.call("");
     }
+    */
 
     function sha256Test1() public returns (uint256) {
         uint256 i;

tests/testdata/outputs_expected/kinds_of_calls.sol.o.json

@@ -1 +1,66 @@
-{"error": null, "issues": [{"address": 618, "contract": "Unknown", "debug": "<DEBUG-DATA>", "description": "The function `_function_0x141f32ff` uses callcode. Callcode does not persist sender or value over the call. Use delegatecall instead.", "function": "_function_0x141f32ff", "max_gas_used": 1141, "min_gas_used": 389, "swc-id": "111", "title": "Use of callcode", "type": "Warning"}, {"address": 626, "contract": "Unknown", "debug": "<DEBUG-DATA>", "description": "The return value of an external call is not checked. Note that execution continue even if the called contract throws.", "function": "_function_0x141f32ff", "max_gas_used": 35856, "min_gas_used": 1104, "swc-id": "104", "title": "Unchecked CALL return value", "type": "Informational"}, {"address": 857, "contract": "Unknown", "debug": "<DEBUG-DATA>", "description": "The return value of an external call is not checked. Note that execution continue even if the called contract throws.", "function": "_function_0x9b58bc26", "max_gas_used": 35919, "min_gas_used": 1167, "swc-id": "104", "title": "Unchecked CALL return value", "type": "Informational"}, {"address": 1038, "contract": "Unknown", "debug": "<DEBUG-DATA>", "description": "The contract executes a function call with high gas to a user-supplied address. Note that the callee can contain arbitrary code and may re-enter any function in this contract. Review the business logic carefully to prevent unanticipated effects on the contract state.", "function": "_function_0xeea4c864", "max_gas_used": 1229, "min_gas_used": 477, "swc-id": "107", "title": "External call to user-supplied address", "type": "Warning"}, {"address": 1046, "contract": "Unknown", "debug": "<DEBUG-DATA>", "description": "The return value of an external call is not checked. Note that execution continue even if the called contract throws.", "function": "_function_0xeea4c864", "max_gas_used": 35944, "min_gas_used": 1192, "swc-id": "104", "title": "Unchecked CALL return value", "type": "Informational"}], "success": true}
+{
+    "error": null,
+    "issues": [
+        {
+            "address": 618,
+            "contract": "Unknown",
+            "debug": "<DEBUG-DATA>",
+            "description": "The function `_function_0x141f32ff` uses callcode. Callcode does not persist sender or value over the call. Use delegatecall instead.",
+            "function": "_function_0x141f32ff",
+            "max_gas_used": 1141,
+            "min_gas_used": 389,
+            "swc-id": "111",
+            "title": "Use of callcode",
+            "type": "Warning"
+        },
+        {
+            "address": 626,
+            "contract": "Unknown",
+            "debug": "<DEBUG-DATA>",
+            "description": "The return value of an external call is not checked. Note that execution continue even if the called contract throws.",
+            "function": "_function_0x141f32ff",
+            "max_gas_used": 35856,
+            "min_gas_used": 1104,
+            "swc-id": "104",
+            "title": "Unchecked CALL return value",
+            "type": "Informational"
+        },
+        {
+            "address": 857,
+            "contract": "Unknown",
+            "debug": "<DEBUG-DATA>",
+            "description": "The return value of an external call is not checked. Note that execution continue even if the called contract throws.",
+            "function": "_function_0x9b58bc26",
+            "max_gas_used": 35913,
+            "min_gas_used": 1161,
+            "swc-id": "104",
+            "title": "Unchecked CALL return value",
+            "type": "Informational"
+        },
+        {
+            "address": 1038,
+            "contract": "Unknown",
+            "debug": "<DEBUG-DATA>",
+            "description": "The contract executes a function call with high gas to a user-supplied address. Note that the callee can contain arbitrary code and may re-enter any function in this contract. Review the business logic carefully to prevent unanticipated effects on the contract state.",
+            "function": "_function_0xeea4c864",
+            "max_gas_used": 1223,
+            "min_gas_used": 471,
+            "swc-id": "107",
+            "title": "External call to user-supplied address",
+            "type": "Warning"
+        },
+        {
+            "address": 1046,
+            "contract": "Unknown",
+            "debug": "<DEBUG-DATA>",
+            "description": "The return value of an external call is not checked. Note that execution continue even if the called contract throws.",
+            "function": "_function_0xeea4c864",
+            "max_gas_used": 35938,
+            "min_gas_used": 1186,
+            "swc-id": "104",
+            "title": "Unchecked CALL return value",
+            "type": "Informational"
+        }
+    ],
+    "success": true
+}

tests/testdata/outputs_expected/kinds_of_calls.sol.o.markdown

@@ -30,7 +30,7 @@ The return value of an external call is not checked. Note that execution continu
 - Contract: Unknown
 - Function name: `_function_0x9b58bc26`
 - PC address: 857
-- Estimated Gas Usage: 1167 - 35919
+- Estimated Gas Usage: 1161 - 35913
 
 ### Description
 
@@ -42,7 +42,7 @@ The return value of an external call is not checked. Note that execution continu
 - Contract: Unknown
 - Function name: `_function_0xeea4c864`
 - PC address: 1038
-- Estimated Gas Usage: 477 - 1229
+- Estimated Gas Usage: 471 - 1223
 
 ### Description
 
@@ -54,7 +54,7 @@ The contract executes a function call with high gas to a user-supplied address.
 - Contract: Unknown
 - Function name: `_function_0xeea4c864`
 - PC address: 1046
-- Estimated Gas Usage: 1192 - 35944
+- Estimated Gas Usage: 1186 - 35938
 
 ### Description
 

tests/testdata/outputs_expected/kinds_of_calls.sol.o.text

@@ -24,7 +24,7 @@ Type: Informational
 Contract: Unknown
 Function name: _function_0x9b58bc26
 PC address: 857
-Estimated Gas Usage: 1167 - 35919
+Estimated Gas Usage: 1161 - 35913
 The return value of an external call is not checked. Note that execution continue even if the called contract throws.
 --------------------
 
@@ -34,7 +34,7 @@ Type: Warning
 Contract: Unknown
 Function name: _function_0xeea4c864
 PC address: 1038
-Estimated Gas Usage: 477 - 1229
+Estimated Gas Usage: 471 - 1223
 The contract executes a function call with high gas to a user-supplied address. Note that the callee can contain arbitrary code and may re-enter any function in this contract. Review the business logic carefully to prevent unanticipated effects on the contract state.
 --------------------
 
@@ -44,7 +44,7 @@ Type: Informational
 Contract: Unknown
 Function name: _function_0xeea4c864
 PC address: 1046
-Estimated Gas Usage: 1192 - 35944
+Estimated Gas Usage: 1186 - 35938
 The return value of an external call is not checked. Note that execution continue even if the called contract throws.
 --------------------