From c3a5fb219beffe7e9043b8e64ecd66bcd397cc4e Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 12 Jun 2024 10:58:55 +0200 Subject: [PATCH 1/6] Add tests to slither-mutate and fix some Python 3.8 compatibility issues. --- .../mutator/mutators/abstract_mutator.py | 8 +- slither/tools/mutator/utils/file_handling.py | 2 +- .../mutator/utils/testing_generated_mutant.py | 7 +- tests/tools/mutator/__init__.py | 0 .../test_data/test_source_unit/README.md | 66 ++++++++++ .../test_data/test_source_unit/foundry.toml | 7 + .../test_source_unit/script/Counter.s.sol | 12 ++ .../test_source_unit/src/Counter.sol | 14 ++ .../test_source_unit/test/Counter.t.sol | 24 ++++ tests/tools/mutator/test_mutator.py | 122 ++++++++++++++++++ 10 files changed, 256 insertions(+), 6 deletions(-) create mode 100644 tests/tools/mutator/__init__.py create mode 100644 tests/tools/mutator/test_data/test_source_unit/README.md create mode 100644 tests/tools/mutator/test_data/test_source_unit/foundry.toml create mode 100644 tests/tools/mutator/test_data/test_source_unit/script/Counter.s.sol create mode 100644 tests/tools/mutator/test_data/test_source_unit/src/Counter.sol create mode 100644 tests/tools/mutator/test_data/test_source_unit/test/Counter.t.sol create mode 100644 tests/tools/mutator/test_mutator.py diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index 69c77a4ca..cb435f856 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -1,7 +1,7 @@ import abc import logging from pathlib import Path -from typing import Optional, Dict, Tuple, List +from typing import Optional, Dict, Tuple, List, Union from slither.core.compilation_unit import SlitherCompilationUnit from slither.formatters.utils.patches import apply_patch, create_diff from slither.tools.mutator.utils.testing_generated_mutant import test_patch @@ -27,7 +27,7 @@ class AbstractMutator( testing_command: str, testing_directory: str, contract_instance: Contract, - solc_remappings: str | None, + solc_remappings: Union[str, None], verbose: bool, very_verbose: bool, output_folder: Path, @@ -81,7 +81,7 @@ class AbstractMutator( (all_patches) = self._mutate() if "patches" not in all_patches: logger.debug("No patches found by %s", self.NAME) - return ([0, 0, 0], [0, 0, 0], self.dont_mutate_line) + return [0, 0, 0], [0, 0, 0], self.dont_mutate_line for file in all_patches["patches"]: # Note: This should only loop over a single file original_txt = self.slither.source_code[file].encode("utf8") @@ -146,4 +146,4 @@ class AbstractMutator( f"Found {self.uncaught_mutant_counts[2]} uncaught tweak mutants so far (out of {self.total_mutant_counts[2]} that compile)" ) - return (self.total_mutant_counts, self.uncaught_mutant_counts, self.dont_mutate_line) + return self.total_mutant_counts, self.uncaught_mutant_counts, self.dont_mutate_line diff --git a/slither/tools/mutator/utils/file_handling.py b/slither/tools/mutator/utils/file_handling.py index 7c02ce099..81e30efc6 100644 --- a/slither/tools/mutator/utils/file_handling.py +++ b/slither/tools/mutator/utils/file_handling.py @@ -111,7 +111,7 @@ def get_sol_file_list(codebase: Path, ignore_paths: Union[List[str], None]) -> L # if input is folder if codebase.is_dir(): for file_name in codebase.rglob("*.sol"): - if not any(part in ignore_paths for part in file_name.parts): + if file_name.is_file() and not any(part in ignore_paths for part in file_name.parts): sol_file_list.append(file_name.as_posix()) return sol_file_list diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 39e7d39de..d62fc3ff0 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -22,7 +22,12 @@ def compile_generated_mutant(file_path: str, mappings: str) -> bool: return False -def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None, verbose: bool) -> bool: +def run_test_cmd( + cmd: str, + timeout: Union[int, None] = None, + target_file: Union[str, None] = None, + verbose: bool = False, +) -> bool: """ function to run codebase tests returns: boolean whether the tests passed or not diff --git a/tests/tools/mutator/__init__.py b/tests/tools/mutator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/tools/mutator/test_data/test_source_unit/README.md b/tests/tools/mutator/test_data/test_source_unit/README.md new file mode 100644 index 000000000..9265b4558 --- /dev/null +++ b/tests/tools/mutator/test_data/test_source_unit/README.md @@ -0,0 +1,66 @@ +## Foundry + +**Foundry is a blazing fast, portable and modular toolkit for Ethereum application development written in Rust.** + +Foundry consists of: + +- **Forge**: Ethereum testing framework (like Truffle, Hardhat and DappTools). +- **Cast**: Swiss army knife for interacting with EVM smart contracts, sending transactions and getting chain data. +- **Anvil**: Local Ethereum node, akin to Ganache, Hardhat Network. +- **Chisel**: Fast, utilitarian, and verbose solidity REPL. + +## Documentation + +https://book.getfoundry.sh/ + +## Usage + +### Build + +```shell +$ forge build +``` + +### Test + +```shell +$ forge test +``` + +### Format + +```shell +$ forge fmt +``` + +### Gas Snapshots + +```shell +$ forge snapshot +``` + +### Anvil + +```shell +$ anvil +``` + +### Deploy + +```shell +$ forge script script/Counter.s.sol:CounterScript --rpc-url --private-key +``` + +### Cast + +```shell +$ cast +``` + +### Help + +```shell +$ forge --help +$ anvil --help +$ cast --help +``` diff --git a/tests/tools/mutator/test_data/test_source_unit/foundry.toml b/tests/tools/mutator/test_data/test_source_unit/foundry.toml new file mode 100644 index 000000000..908c595d0 --- /dev/null +++ b/tests/tools/mutator/test_data/test_source_unit/foundry.toml @@ -0,0 +1,7 @@ +[profile.default] +src = 'src' +out = 'out' +libs = ['lib'] +solc = "0.8.15" + +# See more config options https://github.com/foundry-rs/foundry/tree/master/config \ No newline at end of file diff --git a/tests/tools/mutator/test_data/test_source_unit/script/Counter.s.sol b/tests/tools/mutator/test_data/test_source_unit/script/Counter.s.sol new file mode 100644 index 000000000..df9ee8b02 --- /dev/null +++ b/tests/tools/mutator/test_data/test_source_unit/script/Counter.s.sol @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: UNLICENSED +pragma solidity ^0.8.13; + +import {Script, console} from "forge-std/Script.sol"; + +contract CounterScript is Script { + function setUp() public {} + + function run() public { + vm.broadcast(); + } +} diff --git a/tests/tools/mutator/test_data/test_source_unit/src/Counter.sol b/tests/tools/mutator/test_data/test_source_unit/src/Counter.sol new file mode 100644 index 000000000..7a83fefc4 --- /dev/null +++ b/tests/tools/mutator/test_data/test_source_unit/src/Counter.sol @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: UNLICENSED +pragma solidity ^0.8.15; + +contract Counter { + uint256 public number; + + function setNumber(uint256 newNumber) public { + number = newNumber; + } + + function increment() public { + number++; + } +} diff --git a/tests/tools/mutator/test_data/test_source_unit/test/Counter.t.sol b/tests/tools/mutator/test_data/test_source_unit/test/Counter.t.sol new file mode 100644 index 000000000..6178bf4d2 --- /dev/null +++ b/tests/tools/mutator/test_data/test_source_unit/test/Counter.t.sol @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: UNLICENSED +pragma solidity ^0.8.15; + +import {Test, console} from "forge-std/Test.sol"; +import {Counter} from "../src/Counter.sol"; + +contract CounterTest is Test { + Counter public counter; + + function setUp() public { + counter = new Counter(); + counter.setNumber(0); + } + + function test_Increment() public { + counter.increment(); + assertEq(counter.number(), 1); + } + + function testFuzz_SetNumber(uint256 x) public { + counter.setNumber(x); + assertEq(counter.number(), x); + } +} diff --git a/tests/tools/mutator/test_mutator.py b/tests/tools/mutator/test_mutator.py new file mode 100644 index 000000000..82ed0541e --- /dev/null +++ b/tests/tools/mutator/test_mutator.py @@ -0,0 +1,122 @@ +import os +import subprocess +import tempfile +from pathlib import Path +from unittest import mock +import argparse +from contextlib import contextmanager + +import pytest +from slither import Slither +from slither.tools.mutator.__main__ import _get_mutators, main +from slither.tools.mutator.utils.testing_generated_mutant import run_test_cmd +from slither.tools.mutator.utils.file_handling import get_sol_file_list, backup_source_file + + +TEST_DATA_DIR = Path(__file__).resolve().parent / "test_data" + + +@contextmanager +def change_directory(new_dir): + original_dir = os.getcwd() + os.chdir(new_dir) + try: + yield + finally: + os.chdir(original_dir) + + +def test_get_mutators(): + + mutators = _get_mutators(None) + assert mutators + + mutators = _get_mutators(["ASOR"]) + assert len(mutators) == 1 + assert mutators[0].NAME == "ASOR" + + mutators = _get_mutators(["ASOR", "NotExisiting"]) + assert len(mutators) == 1 + + +@mock.patch( + "argparse.ArgumentParser.parse_args", + return_value=argparse.Namespace( + test_cmd="forge test", + test_dir=None, + ignore_dirs="lib,mutation_campaign", + output_dir=None, + timeout=None, + solc_remaps="forge-std=./lib/forge-std", + verbose=None, + very_verbose=None, + mutators_to_run=None, + comprehensive=None, + codebase=(TEST_DATA_DIR / "test_source_unit" / "src" / "Counter.sol").as_posix(), + contract_names="Counter", + ), +) +@pytest.mark.skip(reason="Slow test") +def test_mutator(mock_args): # pylint: disable=unused-argument + + with change_directory(TEST_DATA_DIR / "test_source_unit"): + main() + + +def test_backup_source_file(): + + file_path = (TEST_DATA_DIR / "test_source_unit" / "src" / "Counter.sol").as_posix() + sl = Slither(file_path) + + with tempfile.TemporaryDirectory() as directory: + files_dict = backup_source_file(sl.source_code, Path(directory)) + + assert len(files_dict) == 1 + assert Path(files_dict[file_path]).exists() + + +def test_get_sol_file_list(): + + project_directory = TEST_DATA_DIR / "test_source_unit" + + files = get_sol_file_list(project_directory, None) + + assert len(files) == 46 + + files = get_sol_file_list(project_directory, ["lib"]) + assert len(files) == 3 + + files = get_sol_file_list(project_directory, ["lib", "script"]) + assert len(files) == 2 + + files = get_sol_file_list(project_directory / "src" / "Counter.sol", None) + assert len(files) == 1 + + (project_directory / "test.sol").mkdir() + files = get_sol_file_list(project_directory, None) + assert all("test.sol" not in file for file in files) + (project_directory / "test.sol").rmdir() + + +def test_run_test(caplog): + with change_directory(TEST_DATA_DIR / "test_source_unit"): + result = run_test_cmd("forge test", timeout=None, target_file=None, verbose=True) + assert result + assert not caplog.records + + # Failed command + result = run_test_cmd("forge non-test", timeout=None, target_file=None, verbose=True) + assert not result + assert caplog.records + + +def test_run_tests_timeout(caplog, monkeypatch): + def mock_run(*args, **kwargs): + raise subprocess.TimeoutExpired(cmd=args[0], timeout=kwargs.get("timeout")) + + monkeypatch.setattr(subprocess, "run", mock_run) + + with change_directory(TEST_DATA_DIR / "test_source_unit"): + result = run_test_cmd("forge test", timeout=1) + assert not result + assert "Tests took too long" in caplog.messages[0] From 7b983e3fc537762cda3233b91ce2cc6321d1c6ff Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 12 Jun 2024 11:14:09 +0200 Subject: [PATCH 2/6] Fix typo (thanks Gustavo) --- slither/tools/mutator/mutators/LIR.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slither/tools/mutator/mutators/LIR.py b/slither/tools/mutator/mutators/LIR.py index cc58cbae1..fc621829f 100644 --- a/slither/tools/mutator/mutators/LIR.py +++ b/slither/tools/mutator/mutators/LIR.py @@ -31,7 +31,7 @@ class LIR(AbstractMutator): # pylint: disable=too-few-public-methods literal_replacements.append(variable.type.max) # append data type max value if str(variable.type).startswith("uint"): literal_replacements.append("1") - elif str(variable.type).startswith("uint"): + elif str(variable.type).startswith("int"): literal_replacements.append("-1") # Get the string start = variable.source_mapping.start @@ -63,7 +63,7 @@ class LIR(AbstractMutator): # pylint: disable=too-few-public-methods literal_replacements.append(variable.type.max) if str(variable.type).startswith("uint"): literal_replacements.append("1") - elif str(variable.type).startswith("uint"): + elif str(variable.type).startswith("int"): literal_replacements.append("-1") start = variable.source_mapping.start stop = start + variable.source_mapping.length From f1df3ea7534f9c7ef36a8afee618f62e95ce0c2f Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 12 Jun 2024 11:20:36 +0200 Subject: [PATCH 3/6] Run tests only if forge is available. --- .../test_data/test_source_unit/README.md | 65 +------------------ tests/tools/mutator/test_mutator.py | 16 ++++- 2 files changed, 16 insertions(+), 65 deletions(-) diff --git a/tests/tools/mutator/test_data/test_source_unit/README.md b/tests/tools/mutator/test_data/test_source_unit/README.md index 9265b4558..554472962 100644 --- a/tests/tools/mutator/test_data/test_source_unit/README.md +++ b/tests/tools/mutator/test_data/test_source_unit/README.md @@ -1,66 +1,7 @@ -## Foundry +# Counter -**Foundry is a blazing fast, portable and modular toolkit for Ethereum application development written in Rust.** - -Foundry consists of: - -- **Forge**: Ethereum testing framework (like Truffle, Hardhat and DappTools). -- **Cast**: Swiss army knife for interacting with EVM smart contracts, sending transactions and getting chain data. -- **Anvil**: Local Ethereum node, akin to Ganache, Hardhat Network. -- **Chisel**: Fast, utilitarian, and verbose solidity REPL. - -## Documentation - -https://book.getfoundry.sh/ - -## Usage - -### Build - -```shell -$ forge build -``` - -### Test - -```shell -$ forge test -``` - -### Format - -```shell -$ forge fmt -``` - -### Gas Snapshots - -```shell -$ forge snapshot -``` - -### Anvil - -```shell -$ anvil -``` - -### Deploy - -```shell -$ forge script script/Counter.s.sol:CounterScript --rpc-url --private-key -``` - -### Cast - -```shell -$ cast -``` - -### Help +Init using : ```shell -$ forge --help -$ anvil --help -$ cast --help +forge install --no-commit --no-git . ``` diff --git a/tests/tools/mutator/test_mutator.py b/tests/tools/mutator/test_mutator.py index 82ed0541e..fbfe86b9a 100644 --- a/tests/tools/mutator/test_mutator.py +++ b/tests/tools/mutator/test_mutator.py @@ -1,10 +1,11 @@ +import argparse +from contextlib import contextmanager import os +from pathlib import Path +import shutil import subprocess import tempfile -from pathlib import Path from unittest import mock -import argparse -from contextlib import contextmanager import pytest from slither import Slither @@ -15,6 +16,9 @@ from slither.tools.mutator.utils.file_handling import get_sol_file_list, backup_ TEST_DATA_DIR = Path(__file__).resolve().parent / "test_data" +foundry_available = shutil.which("forge") is not None +project_ready = Path(TEST_DATA_DIR, "test_source_unit/lib/forge-std").exists() + @contextmanager def change_directory(new_dir): @@ -75,6 +79,9 @@ def test_backup_source_file(): assert Path(files_dict[file_path]).exists() +@pytest.mark.skipif( + not foundry_available or not project_ready, reason="requires Foundry and project setup" +) def test_get_sol_file_list(): project_directory = TEST_DATA_DIR / "test_source_unit" @@ -98,6 +105,9 @@ def test_get_sol_file_list(): (project_directory / "test.sol").rmdir() +@pytest.mark.skipif( + not foundry_available or not project_ready, reason="requires Foundry and project setup" +) def test_run_test(caplog): with change_directory(TEST_DATA_DIR / "test_source_unit"): result = run_test_cmd("forge test", timeout=None, target_file=None, verbose=True) From 94fc82cc74c78ef4ffc9910126b8a5aac22ef7bb Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 12 Jun 2024 11:28:18 +0200 Subject: [PATCH 4/6] Run tools tests in CI. --- .github/scripts/tool_test_runner.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/tool_test_runner.sh b/.github/scripts/tool_test_runner.sh index 30d8176a0..574358e10 100755 --- a/.github/scripts/tool_test_runner.sh +++ b/.github/scripts/tool_test_runner.sh @@ -2,11 +2,11 @@ # used to pass --cov=$path and --cov-append to pytest if [ "$1" != "" ]; then - pytest "$1" tests/tools/read-storage/test_read_storage.py + pytest "$1" tests/tools status_code=$? python -m coverage report else - pytest tests/tools/read-storage/test_read_storage.py + pytest tests/tools status_code=$? fi From 43301c750d27363a56f370dd0e6dc5a5b84b5c35 Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 12 Jun 2024 14:26:40 +0200 Subject: [PATCH 5/6] Remove last wrongly typed annotation --- slither/tools/mutator/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 8a7ce3e1a..dea11676a 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -6,7 +6,7 @@ import shutil import sys import time from pathlib import Path -from typing import Type, List, Any, Optional +from typing import Type, List, Any, Optional, Union from crytic_compile import cryticparser from slither import Slither from slither.tools.mutator.utils.testing_generated_mutant import run_test_cmd @@ -116,7 +116,7 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def _get_mutators(mutators_list: List[str] | None) -> List[Type[AbstractMutator]]: +def _get_mutators(mutators_list: Union[List[str], None]) -> List[Type[AbstractMutator]]: detectors_ = [getattr(all_mutators, name) for name in dir(all_mutators)] if mutators_list is not None: detectors = [ From 5c0eccfbc54f529c57863a3ea8dc894972d6d4bf Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 12 Jun 2024 15:17:00 +0200 Subject: [PATCH 6/6] Fix test in CI. --- tests/tools/mutator/test_mutator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/tools/mutator/test_mutator.py b/tests/tools/mutator/test_mutator.py index fbfe86b9a..68b595319 100644 --- a/tests/tools/mutator/test_mutator.py +++ b/tests/tools/mutator/test_mutator.py @@ -61,16 +61,17 @@ def test_get_mutators(): ), ) @pytest.mark.skip(reason="Slow test") -def test_mutator(mock_args): # pylint: disable=unused-argument +def test_mutator(mock_args, solc_binary_path): # pylint: disable=unused-argument with change_directory(TEST_DATA_DIR / "test_source_unit"): main() -def test_backup_source_file(): +def test_backup_source_file(solc_binary_path): + solc_path = solc_binary_path("0.8.15") file_path = (TEST_DATA_DIR / "test_source_unit" / "src" / "Counter.sol").as_posix() - sl = Slither(file_path) + sl = Slither(file_path, solc=solc_path) with tempfile.TemporaryDirectory() as directory: files_dict = backup_source_file(sl.source_code, Path(directory))