Merge pull request #2482 from crytic/features/test-mutator

Improve slither-mutate testing

.github/scripts/tool_test_runner.sh

@@ -2,11 +2,11 @@
 
 # used to pass --cov=$path and --cov-append to pytest
 if [ "$1" != "" ]; then
-    pytest "$1" tests/tools/read-storage/test_read_storage.py
+    pytest "$1" tests/tools
     status_code=$?
     python -m coverage report
 else
-    pytest tests/tools/read-storage/test_read_storage.py
+    pytest tests/tools
     status_code=$?
 fi
 

slither/tools/mutator/__main__.py

@@ -6,7 +6,7 @@ import shutil
 import sys
 import time
 from pathlib import Path
-from typing import Type, List, Any, Optional
+from typing import Type, List, Any, Optional, Union
 from crytic_compile import cryticparser
 from slither import Slither
 from slither.tools.mutator.utils.testing_generated_mutant import run_test_cmd
@@ -116,7 +116,7 @@ def parse_args() -> argparse.Namespace:
     return parser.parse_args()
 
 
-def _get_mutators(mutators_list: List[str] | None) -> List[Type[AbstractMutator]]:
+def _get_mutators(mutators_list: Union[List[str], None]) -> List[Type[AbstractMutator]]:
     detectors_ = [getattr(all_mutators, name) for name in dir(all_mutators)]
     if mutators_list is not None:
         detectors = [

slither/tools/mutator/mutators/LIR.py

@@ -31,7 +31,7 @@ class LIR(AbstractMutator):  # pylint: disable=too-few-public-methods
                         literal_replacements.append(variable.type.max)  # append data type max value
                         if str(variable.type).startswith("uint"):
                             literal_replacements.append("1")
-                        elif str(variable.type).startswith("uint"):
+                        elif str(variable.type).startswith("int"):
                             literal_replacements.append("-1")
                     # Get the string
                     start = variable.source_mapping.start
@@ -63,7 +63,7 @@ class LIR(AbstractMutator):  # pylint: disable=too-few-public-methods
                         literal_replacements.append(variable.type.max)
                         if str(variable.type).startswith("uint"):
                             literal_replacements.append("1")
-                        elif str(variable.type).startswith("uint"):
+                        elif str(variable.type).startswith("int"):
                             literal_replacements.append("-1")
                     start = variable.source_mapping.start
                     stop = start + variable.source_mapping.length

slither/tools/mutator/mutators/abstract_mutator.py

@@ -1,7 +1,7 @@
 import abc
 import logging
 from pathlib import Path
-from typing import Optional, Dict, Tuple, List
+from typing import Optional, Dict, Tuple, List, Union
 from slither.core.compilation_unit import SlitherCompilationUnit
 from slither.formatters.utils.patches import apply_patch, create_diff
 from slither.tools.mutator.utils.testing_generated_mutant import test_patch
@@ -27,7 +27,7 @@ class AbstractMutator(
         testing_command: str,
         testing_directory: str,
         contract_instance: Contract,
-        solc_remappings: str | None,
+        solc_remappings: Union[str, None],
         verbose: bool,
         very_verbose: bool,
         output_folder: Path,
@@ -81,7 +81,7 @@ class AbstractMutator(
         (all_patches) = self._mutate()
         if "patches" not in all_patches:
             logger.debug("No patches found by %s", self.NAME)
-            return ([0, 0, 0], [0, 0, 0], self.dont_mutate_line)
+            return [0, 0, 0], [0, 0, 0], self.dont_mutate_line
 
         for file in all_patches["patches"]:  # Note: This should only loop over a single file
             original_txt = self.slither.source_code[file].encode("utf8")
@@ -146,4 +146,4 @@ class AbstractMutator(
                             f"Found {self.uncaught_mutant_counts[2]} uncaught tweak mutants so far (out of {self.total_mutant_counts[2]} that compile)"
                         )
 
-        return (self.total_mutant_counts, self.uncaught_mutant_counts, self.dont_mutate_line)
+        return self.total_mutant_counts, self.uncaught_mutant_counts, self.dont_mutate_line

slither/tools/mutator/utils/file_handling.py

@@ -111,7 +111,7 @@ def get_sol_file_list(codebase: Path, ignore_paths: Union[List[str], None]) -> L
     # if input is folder
     if codebase.is_dir():
         for file_name in codebase.rglob("*.sol"):
-            if not any(part in ignore_paths for part in file_name.parts):
+            if file_name.is_file() and not any(part in ignore_paths for part in file_name.parts):
                 sol_file_list.append(file_name.as_posix())
 
     return sol_file_list

slither/tools/mutator/utils/testing_generated_mutant.py

@@ -22,7 +22,12 @@ def compile_generated_mutant(file_path: str, mappings: str) -> bool:
         return False
 
 
-def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None, verbose: bool) -> bool:
+def run_test_cmd(
+    cmd: str,
+    timeout: Union[int, None] = None,
+    target_file: Union[str, None] = None,
+    verbose: bool = False,
+) -> bool:
     """
     function to run codebase tests
     returns: boolean whether the tests passed or not

tests/tools/mutator/test_data/test_source_unit/README.md

@@ -0,0 +1,7 @@
+# Counter
+
+Init using :
+
+```shell
+forge install --no-commit --no-git .
+```

tests/tools/mutator/test_data/test_source_unit/foundry.toml

@@ -0,0 +1,7 @@
+[profile.default]
+src = 'src'
+out = 'out'
+libs = ['lib']
+solc = "0.8.15"
+
+# See more config options https://github.com/foundry-rs/foundry/tree/master/config

tests/tools/mutator/test_data/test_source_unit/script/Counter.s.sol

@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: UNLICENSED
+pragma solidity ^0.8.13;
+
+import {Script, console} from "forge-std/Script.sol";
+
+contract CounterScript is Script {
+    function setUp() public {}
+
+    function run() public {
+        vm.broadcast();
+    }
+}

tests/tools/mutator/test_data/test_source_unit/src/Counter.sol

@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: UNLICENSED
+pragma solidity ^0.8.15;
+
+contract Counter {
+    uint256 public number;
+
+    function setNumber(uint256 newNumber) public {
+        number = newNumber;
+    }
+
+    function increment() public {
+        number++;
+    }
+}

tests/tools/mutator/test_data/test_source_unit/test/Counter.t.sol

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: UNLICENSED
+pragma solidity ^0.8.15;
+
+import {Test, console} from "forge-std/Test.sol";
+import {Counter} from "../src/Counter.sol";
+
+contract CounterTest is Test {
+    Counter public counter;
+
+    function setUp() public {
+        counter = new Counter();
+        counter.setNumber(0);
+    }
+
+    function test_Increment() public {
+        counter.increment();
+        assertEq(counter.number(), 1);
+    }
+
+    function testFuzz_SetNumber(uint256 x) public {
+        counter.setNumber(x);
+        assertEq(counter.number(), x);
+    }
+}

tests/tools/mutator/test_mutator.py

@@ -0,0 +1,133 @@
+import argparse
+from contextlib import contextmanager
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+from unittest import mock
+
+import pytest
+from slither import Slither
+from slither.tools.mutator.__main__ import _get_mutators, main
+from slither.tools.mutator.utils.testing_generated_mutant import run_test_cmd
+from slither.tools.mutator.utils.file_handling import get_sol_file_list, backup_source_file
+
+
+TEST_DATA_DIR = Path(__file__).resolve().parent / "test_data"
+
+foundry_available = shutil.which("forge") is not None
+project_ready = Path(TEST_DATA_DIR, "test_source_unit/lib/forge-std").exists()
+
+
+@contextmanager
+def change_directory(new_dir):
+    original_dir = os.getcwd()
+    os.chdir(new_dir)
+    try:
+        yield
+    finally:
+        os.chdir(original_dir)
+
+
+def test_get_mutators():
+
+    mutators = _get_mutators(None)
+    assert mutators
+
+    mutators = _get_mutators(["ASOR"])
+    assert len(mutators) == 1
+    assert mutators[0].NAME == "ASOR"
+
+    mutators = _get_mutators(["ASOR", "NotExisiting"])
+    assert len(mutators) == 1
+
+
+@mock.patch(
+    "argparse.ArgumentParser.parse_args",
+    return_value=argparse.Namespace(
+        test_cmd="forge test",
+        test_dir=None,
+        ignore_dirs="lib,mutation_campaign",
+        output_dir=None,
+        timeout=None,
+        solc_remaps="forge-std=./lib/forge-std",
+        verbose=None,
+        very_verbose=None,
+        mutators_to_run=None,
+        comprehensive=None,
+        codebase=(TEST_DATA_DIR / "test_source_unit" / "src" / "Counter.sol").as_posix(),
+        contract_names="Counter",
+    ),
+)
+@pytest.mark.skip(reason="Slow test")
+def test_mutator(mock_args, solc_binary_path):  # pylint: disable=unused-argument
+
+    with change_directory(TEST_DATA_DIR / "test_source_unit"):
+        main()
+
+
+def test_backup_source_file(solc_binary_path):
+    solc_path = solc_binary_path("0.8.15")
+
+    file_path = (TEST_DATA_DIR / "test_source_unit" / "src" / "Counter.sol").as_posix()
+    sl = Slither(file_path, solc=solc_path)
+
+    with tempfile.TemporaryDirectory() as directory:
+        files_dict = backup_source_file(sl.source_code, Path(directory))
+
+        assert len(files_dict) == 1
+        assert Path(files_dict[file_path]).exists()
+
+
+@pytest.mark.skipif(
+    not foundry_available or not project_ready, reason="requires Foundry and project setup"
+)
+def test_get_sol_file_list():
+
+    project_directory = TEST_DATA_DIR / "test_source_unit"
+
+    files = get_sol_file_list(project_directory, None)
+
+    assert len(files) == 46
+
+    files = get_sol_file_list(project_directory, ["lib"])
+    assert len(files) == 3
+
+    files = get_sol_file_list(project_directory, ["lib", "script"])
+    assert len(files) == 2
+
+    files = get_sol_file_list(project_directory / "src" / "Counter.sol", None)
+    assert len(files) == 1
+
+    (project_directory / "test.sol").mkdir()
+    files = get_sol_file_list(project_directory, None)
+    assert all("test.sol" not in file for file in files)
+    (project_directory / "test.sol").rmdir()
+
+
+@pytest.mark.skipif(
+    not foundry_available or not project_ready, reason="requires Foundry and project setup"
+)
+def test_run_test(caplog):
+    with change_directory(TEST_DATA_DIR / "test_source_unit"):
+        result = run_test_cmd("forge test", timeout=None, target_file=None, verbose=True)
+        assert result
+        assert not caplog.records
+
+        # Failed command
+        result = run_test_cmd("forge non-test", timeout=None, target_file=None, verbose=True)
+        assert not result
+        assert caplog.records
+
+
+def test_run_tests_timeout(caplog, monkeypatch):
+    def mock_run(*args, **kwargs):
+        raise subprocess.TimeoutExpired(cmd=args[0], timeout=kwargs.get("timeout"))
+
+    monkeypatch.setattr(subprocess, "run", mock_run)
+
+    with change_directory(TEST_DATA_DIR / "test_source_unit"):
+        result = run_test_cmd("forge test", timeout=1)
+        assert not result
+        assert "Tests took too long" in caplog.messages[0]