Add tests to slither-mutate and fix some Python 3.8 compatibility issues.

slither/tools/mutator/mutators/abstract_mutator.py

@@ -1,7 +1,7 @@
 import abc
 import logging
 from pathlib import Path
-from typing import Optional, Dict, Tuple, List
+from typing import Optional, Dict, Tuple, List, Union
 from slither.core.compilation_unit import SlitherCompilationUnit
 from slither.formatters.utils.patches import apply_patch, create_diff
 from slither.tools.mutator.utils.testing_generated_mutant import test_patch
@@ -27,7 +27,7 @@ class AbstractMutator(
         testing_command: str,
         testing_directory: str,
         contract_instance: Contract,
-        solc_remappings: str | None,
+        solc_remappings: Union[str, None],
         verbose: bool,
         very_verbose: bool,
         output_folder: Path,
@@ -81,7 +81,7 @@ class AbstractMutator(
         (all_patches) = self._mutate()
         if "patches" not in all_patches:
             logger.debug("No patches found by %s", self.NAME)
-            return ([0, 0, 0], [0, 0, 0], self.dont_mutate_line)
+            return [0, 0, 0], [0, 0, 0], self.dont_mutate_line
 
         for file in all_patches["patches"]:  # Note: This should only loop over a single file
             original_txt = self.slither.source_code[file].encode("utf8")
@@ -146,4 +146,4 @@ class AbstractMutator(
                             f"Found {self.uncaught_mutant_counts[2]} uncaught tweak mutants so far (out of {self.total_mutant_counts[2]} that compile)"
                         )
 
-        return (self.total_mutant_counts, self.uncaught_mutant_counts, self.dont_mutate_line)
+        return self.total_mutant_counts, self.uncaught_mutant_counts, self.dont_mutate_line

slither/tools/mutator/utils/file_handling.py

@@ -111,7 +111,7 @@ def get_sol_file_list(codebase: Path, ignore_paths: Union[List[str], None]) -> L
     # if input is folder
     if codebase.is_dir():
         for file_name in codebase.rglob("*.sol"):
-            if not any(part in ignore_paths for part in file_name.parts):
+            if file_name.is_file() and not any(part in ignore_paths for part in file_name.parts):
                 sol_file_list.append(file_name.as_posix())
 
     return sol_file_list

slither/tools/mutator/utils/testing_generated_mutant.py

@@ -22,7 +22,12 @@ def compile_generated_mutant(file_path: str, mappings: str) -> bool:
         return False
 
 
-def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None, verbose: bool) -> bool:
+def run_test_cmd(
+    cmd: str,
+    timeout: Union[int, None] = None,
+    target_file: Union[str, None] = None,
+    verbose: bool = False,
+) -> bool:
     """
     function to run codebase tests
     returns: boolean whether the tests passed or not

tests/tools/mutator/test_data/test_source_unit/README.md

@@ -0,0 +1,66 @@
+## Foundry
+
+**Foundry is a blazing fast, portable and modular toolkit for Ethereum application development written in Rust.**
+
+Foundry consists of:
+
+-   **Forge**: Ethereum testing framework (like Truffle, Hardhat and DappTools).
+-   **Cast**: Swiss army knife for interacting with EVM smart contracts, sending transactions and getting chain data.
+-   **Anvil**: Local Ethereum node, akin to Ganache, Hardhat Network.
+-   **Chisel**: Fast, utilitarian, and verbose solidity REPL.
+
+## Documentation
+
+https://book.getfoundry.sh/
+
+## Usage
+
+### Build
+
+```shell
+$ forge build
+```
+
+### Test
+
+```shell
+$ forge test
+```
+
+### Format
+
+```shell
+$ forge fmt
+```
+
+### Gas Snapshots
+
+```shell
+$ forge snapshot
+```
+
+### Anvil
+
+```shell
+$ anvil
+```
+
+### Deploy
+
+```shell
+$ forge script script/Counter.s.sol:CounterScript --rpc-url <your_rpc_url> --private-key <your_private_key>
+```
+
+### Cast
+
+```shell
+$ cast <subcommand>
+```
+
+### Help
+
+```shell
+$ forge --help
+$ anvil --help
+$ cast --help
+```

tests/tools/mutator/test_data/test_source_unit/foundry.toml

@@ -0,0 +1,7 @@
+[profile.default]
+src = 'src'
+out = 'out'
+libs = ['lib']
+solc = "0.8.15"
+
+# See more config options https://github.com/foundry-rs/foundry/tree/master/config

tests/tools/mutator/test_data/test_source_unit/script/Counter.s.sol

@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: UNLICENSED
+pragma solidity ^0.8.13;
+
+import {Script, console} from "forge-std/Script.sol";
+
+contract CounterScript is Script {
+    function setUp() public {}
+
+    function run() public {
+        vm.broadcast();
+    }
+}

tests/tools/mutator/test_data/test_source_unit/src/Counter.sol

@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: UNLICENSED
+pragma solidity ^0.8.15;
+
+contract Counter {
+    uint256 public number;
+
+    function setNumber(uint256 newNumber) public {
+        number = newNumber;
+    }
+
+    function increment() public {
+        number++;
+    }
+}

tests/tools/mutator/test_data/test_source_unit/test/Counter.t.sol

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: UNLICENSED
+pragma solidity ^0.8.15;
+
+import {Test, console} from "forge-std/Test.sol";
+import {Counter} from "../src/Counter.sol";
+
+contract CounterTest is Test {
+    Counter public counter;
+
+    function setUp() public {
+        counter = new Counter();
+        counter.setNumber(0);
+    }
+
+    function test_Increment() public {
+        counter.increment();
+        assertEq(counter.number(), 1);
+    }
+
+    function testFuzz_SetNumber(uint256 x) public {
+        counter.setNumber(x);
+        assertEq(counter.number(), x);
+    }
+}

tests/tools/mutator/test_mutator.py

@@ -0,0 +1,122 @@
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+from unittest import mock
+import argparse
+from contextlib import contextmanager
+
+import pytest
+from slither import Slither
+from slither.tools.mutator.__main__ import _get_mutators, main
+from slither.tools.mutator.utils.testing_generated_mutant import run_test_cmd
+from slither.tools.mutator.utils.file_handling import get_sol_file_list, backup_source_file
+
+
+TEST_DATA_DIR = Path(__file__).resolve().parent / "test_data"
+
+
+@contextmanager
+def change_directory(new_dir):
+    original_dir = os.getcwd()
+    os.chdir(new_dir)
+    try:
+        yield
+    finally:
+        os.chdir(original_dir)
+
+
+def test_get_mutators():
+
+    mutators = _get_mutators(None)
+    assert mutators
+
+    mutators = _get_mutators(["ASOR"])
+    assert len(mutators) == 1
+    assert mutators[0].NAME == "ASOR"
+
+    mutators = _get_mutators(["ASOR", "NotExisiting"])
+    assert len(mutators) == 1
+
+
+@mock.patch(
+    "argparse.ArgumentParser.parse_args",
+    return_value=argparse.Namespace(
+        test_cmd="forge test",
+        test_dir=None,
+        ignore_dirs="lib,mutation_campaign",
+        output_dir=None,
+        timeout=None,
+        solc_remaps="forge-std=./lib/forge-std",
+        verbose=None,
+        very_verbose=None,
+        mutators_to_run=None,
+        comprehensive=None,
+        codebase=(TEST_DATA_DIR / "test_source_unit" / "src" / "Counter.sol").as_posix(),
+        contract_names="Counter",
+    ),
+)
+@pytest.mark.skip(reason="Slow test")
+def test_mutator(mock_args):  # pylint: disable=unused-argument
+
+    with change_directory(TEST_DATA_DIR / "test_source_unit"):
+        main()
+
+
+def test_backup_source_file():
+
+    file_path = (TEST_DATA_DIR / "test_source_unit" / "src" / "Counter.sol").as_posix()
+    sl = Slither(file_path)
+
+    with tempfile.TemporaryDirectory() as directory:
+        files_dict = backup_source_file(sl.source_code, Path(directory))
+
+        assert len(files_dict) == 1
+        assert Path(files_dict[file_path]).exists()
+
+
+def test_get_sol_file_list():
+
+    project_directory = TEST_DATA_DIR / "test_source_unit"
+
+    files = get_sol_file_list(project_directory, None)
+
+    assert len(files) == 46
+
+    files = get_sol_file_list(project_directory, ["lib"])
+    assert len(files) == 3
+
+    files = get_sol_file_list(project_directory, ["lib", "script"])
+    assert len(files) == 2
+
+    files = get_sol_file_list(project_directory / "src" / "Counter.sol", None)
+    assert len(files) == 1
+
+    (project_directory / "test.sol").mkdir()
+    files = get_sol_file_list(project_directory, None)
+    assert all("test.sol" not in file for file in files)
+    (project_directory / "test.sol").rmdir()
+
+
+def test_run_test(caplog):
+    with change_directory(TEST_DATA_DIR / "test_source_unit"):
+        result = run_test_cmd("forge test", timeout=None, target_file=None, verbose=True)
+        assert result
+        assert not caplog.records
+
+        # Failed command
+        result = run_test_cmd("forge non-test", timeout=None, target_file=None, verbose=True)
+        assert not result
+        assert caplog.records
+
+
+def test_run_tests_timeout(caplog, monkeypatch):
+    def mock_run(*args, **kwargs):
+        raise subprocess.TimeoutExpired(cmd=args[0], timeout=kwargs.get("timeout"))
+
+    monkeypatch.setattr(subprocess, "run", mock_run)
+
+    with change_directory(TEST_DATA_DIR / "test_source_unit"):
+        result = run_test_cmd("forge test", timeout=1)
+        assert not result
+        assert "Tests took too long" in caplog.messages[0]