Merge branch 'proper-xfail-skip' into testing-strategy

tests/test_features.py

@@ -1,5 +1,7 @@
 import inspect
+import shutil
 from pathlib import Path
+import pytest
 
 from crytic_compile import CryticCompile
 from crytic_compile.platform.solc_standard_json import SolcStandardJson
@@ -23,6 +25,10 @@ def _run_all_detectors(slither: Slither) -> None:
     slither.run_detectors()
 
 
+hardhat_available = shutil.which("hardhat") is not None
+
+
+@pytest.mark.skipif(not hardhat_available, reason="requires Hardhat and project setup")
 def test_node() -> None:
     # hardhat must have been installed in tests/test_node_modules
     # For the CI its done through the github action config

tests/test_ssa_generation.py

@@ -375,7 +375,7 @@ def test_basic_loop_phi():
     verify_properties_hold(contract)
 
 
-@pytest.mark.skip(reason="Fails in current slither version. Fix in #1102.")
+@pytest.mark.xfail(strict=True, reason="Fails in current slither version. Fix in #1102.")
 def test_phi_propagation_loop():
     contract = """
      pragma solidity ^0.8.11;
@@ -396,7 +396,7 @@ def test_phi_propagation_loop():
     verify_properties_hold(contract)
 
 
-@pytest.mark.skip(reason="Fails in current slither version. Fix in #1102.")
+@pytest.mark.xfail(strict=True, reason="Fails in current slither version. Fix in #1102.")
 def test_free_function_properties():
     contract = """
         pragma solidity ^0.8.11;
@@ -460,7 +460,7 @@ def test_ssa_inter_transactional():
         assert assign2.lvalue in phi.rvalues
 
 
-@pytest.mark.skip(reason="Fails in current slither version. Fix in #1102.")
+@pytest.mark.xfail(strict=True, reason="Fails in current slither version. Fix in #1102.")
 def test_ssa_phi_callbacks():
     source = """
     pragma solidity ^0.8.11;
@@ -519,7 +519,7 @@ def test_ssa_phi_callbacks():
         assert len(after_call_phi.rvalues) == 2
 
 
-@pytest.mark.skip(reason="Fails in current slither version. Fix in #1102.")
+@pytest.mark.xfail(strict=True, reason="Fails in current slither version. Fix in #1102.")
 def test_storage_refers_to():
     """Test the storage aspects of the SSA IR
 
@@ -586,7 +586,6 @@ def test_storage_refers_to():
         assert phinodes[1].lvalue in entryphi[0].rvalues or entryphi[1].rvalues
 
 
-@pytest.mark.skip(reason="Fails in current slither version. Fix in #1102.")
 @pytest.mark.skipif(
     not valid_version("0.4.0"), reason="Solidity version 0.4.0 not available on this platform"
 )
@@ -623,7 +622,7 @@ def test_initial_version_exists_for_locals():
         assert a_0.non_ssa_version == a_1.non_ssa_version
 
 
-@pytest.mark.skip(reason="Fails in current slither version. Fix in #1102.")
+@pytest.mark.xfail(strict=True, reason="Fails in current slither version. Fix in #1102.")
 @pytest.mark.skipif(
     not valid_version("0.4.0"), reason="Solidity version 0.4.0 not available on this platform"
 )
@@ -663,7 +662,7 @@ def test_initial_version_exists_for_state_variables():
         assert len(get_ssa_of_type(f, Phi)) == 0
 
 
-@pytest.mark.skip(reason="Fails in current slither version. Fix in #1102.")
+@pytest.mark.xfail(strict=True, reason="Fails in current slither version. Fix in #1102.")
 def test_initial_version_exists_for_state_variables_function_assign():
     """
     In solidity you can write statements such as
@@ -760,7 +759,7 @@ def test_shadow_local():
         assert all(map(lambda x: x.lvalue.index == 1, get_ssa_of_type(f, Assignment)))
 
 
-@pytest.mark.skip(reason="Fails in current slither version. Fix in #1102.")
+@pytest.mark.xfail(strict=True, reason="Fails in current slither version. Fix in #1102.")
 def test_multiple_named_args_returns():
     """Verifies that named arguments and return values have correct versions
 
@@ -789,7 +788,7 @@ def test_multiple_named_args_returns():
         )
 
 
-@pytest.mark.xfail(reason="Tests for wanted state of SSA IR, not current.")
+@pytest.mark.xfail(reason="Tests for wanted state of SSA IR, not current.", strict=True)
 def test_memory_array():
     src = """
     contract MemArray {
@@ -855,7 +854,7 @@ def test_memory_array():
         assert len(phi_entry_b.rvalues) == len(b_args) + 1
 
 
-@pytest.mark.xfail(reason="Tests for wanted state of SSA IR, not current.")
+@pytest.mark.xfail(reason="Tests for wanted state of SSA IR, not current.", strict=True)
 def test_storage_array():
     src = """
     contract StorageArray {
@@ -910,7 +909,7 @@ def test_storage_array():
         assert len(phi_entry_b.rvalues) == 3  # See comment in b above
 
 
-@pytest.mark.skip(reason="Fails in current slither version. Fix in #1102.")
+@pytest.mark.xfail(strict=True, reason="Fails in current slither version. Fix in #1102.")
 def test_issue_468():
     """
     Ensure issue 468 is corrected as per
@@ -964,7 +963,7 @@ def test_issue_468():
         assert phi_endif.lvalue in phi_entry.rvalues
 
 
-@pytest.mark.skip(reason="Fails in current slither version. Fix in #1102.")
+@pytest.mark.xfail(strict=True, reason="Fails in current slither version. Fix in #1102.")
 def test_issue_434():
     source = """
      contract Contract {
@@ -1018,7 +1017,7 @@ def test_issue_434():
         assert aftercall_phi.lvalue in (add_f.variable_left, add_f.variable_right)
 
 
-@pytest.mark.skip(reason="Fails in current slither version. Fix in #1102.")
+@pytest.mark.xfail(strict=True, reason="Fails in current slither version. Fix in #1102.")
 def test_issue_473():
     source = """
     contract Contract {