Merge pull request #682 from crytic/dev-fix-parsing-test

Fix parsing tests

CONTRIBUTING.md

@@ -23,6 +23,7 @@ Some pull request guidelines:
 ## Development Environment
 Instructions for installing a development version of Slither can be found in our [wiki](https://github.com/crytic/slither/wiki/Developer-installation).
 
+To run the unit tests, you need `deepdiff` installed (`pip install deepdiff`).
 ## Linters
 
 Several linters and security checkers are run on the PRs.

slither/solc_parsing/slitherSolc.py

@@ -129,7 +129,6 @@ class SlitherSolc:
             return
 
         for contract_data in data_loaded[self.get_children()]:
-
             assert contract_data[self.get_key()] in [
                 "ContractDefinition",
                 "PragmaDirective",

tests/test_ast_parsing.py

@@ -1,7 +1,7 @@
-import errno
 import json
 import os
 import subprocess
+import sys
 from collections import namedtuple
 from distutils.version import StrictVersion
 from typing import List, Dict
@@ -80,6 +80,8 @@ XFAIL = [
     "for_0.7.0_legacy",
     "for_0.7.1_legacy",
     "for_0.7.2_legacy",
+    "for_0.7.3_legacy",
+    "for_0.7.4_legacy",
     "function_0.6.0_legacy",
     "function_0.6.1_legacy",
     "function_0.6.2_legacy",
@@ -98,6 +100,10 @@ XFAIL = [
     "function_0.7.1_compact",
     "function_0.7.2_legacy",
     "function_0.7.2_compact",
+    "function_0.7.3_legacy",
+    "function_0.7.3_compact",
+    "function_0.7.4_legacy",
+    "function_0.7.4_compact",
     "import_0.4.0_legacy",
     "import_0.4.1_legacy",
     "import_0.4.2_legacy",
@@ -208,6 +214,10 @@ XFAIL = [
     "import_0.7.1_compact",
     "import_0.7.2_legacy",
     "import_0.7.2_compact",
+    "import_0.7.3_legacy",
+    "import_0.7.3_compact",
+    "import_0.7.4_legacy",
+    "import_0.7.4_compact",
     "indexrangeaccess_0.6.1_legacy",
     "indexrangeaccess_0.6.2_legacy",
     "indexrangeaccess_0.6.3_legacy",
@@ -223,12 +233,18 @@ XFAIL = [
     "indexrangeaccess_0.7.0_legacy",
     "indexrangeaccess_0.7.1_legacy",
     "indexrangeaccess_0.7.2_legacy",
+    "indexrangeaccess_0.7.3_legacy",
+    "indexrangeaccess_0.7.4_legacy",
     "literal_0.7.0_legacy",
     "literal_0.7.0_compact",
     "literal_0.7.1_legacy",
     "literal_0.7.1_compact",
     "literal_0.7.2_legacy",
     "literal_0.7.2_compact",
+    "literal_0.7.3_legacy",
+    "literal_0.7.3_compact",
+    "literal_0.7.4_legacy",
+    "literal_0.7.4_compact",
     "memberaccess_0.6.8_legacy",
     "memberaccess_0.6.9_legacy",
     "memberaccess_0.6.10_legacy",
@@ -253,6 +269,8 @@ XFAIL = [
     "struct_0.7.0_legacy",
     "struct_0.7.1_legacy",
     "struct_0.7.2_legacy",
+    "struct_0.7.3_legacy",
+    "struct_0.7.4_legacy",
     "trycatch_0.6.0_legacy",
     "trycatch_0.6.1_legacy",
     "trycatch_0.6.2_legacy",
@@ -269,6 +287,8 @@ XFAIL = [
     "trycatch_0.7.0_legacy",
     "trycatch_0.7.1_legacy",
     "trycatch_0.7.2_legacy",
+    "trycatch_0.7.3_legacy",
+    "trycatch_0.7.4_legacy",
     "variable_0.6.5_legacy",
     "variable_0.6.5_compact",
     "variable_0.6.6_legacy",
@@ -365,6 +385,8 @@ XFAIL = [
     "variabledeclaration_0.7.0_legacy",
     "variabledeclaration_0.7.1_legacy",
     "variabledeclaration_0.7.2_legacy",
+    "variabledeclaration_0.7.3_legacy",
+    "variabledeclaration_0.7.4_legacy",
 ]
 
 
@@ -480,6 +502,14 @@ def generate_output(sl: Slither) -> Dict[str, Dict[str, str]]:
 ALL_TESTS = get_all_test()
 
 
+def set_solc(test_item: Item):
+    # hacky hack hack to pick the solc version we want
+    env = dict(os.environ)
+    env["SOLC_VERSION"] = test_item.solc_ver
+    os.environ.clear()
+    os.environ.update(env)
+
+
 @pytest.mark.parametrize("test_item", ALL_TESTS, ids=id_test)
 def test_parsing(test_item: Item):
     flavor = "legacy" if test_item.is_legacy else "compact"
@@ -491,11 +521,7 @@ def test_parsing(test_item: Item):
     if id_test(test_item) in XFAIL:
         pytest.xfail("this test needs to be fixed")
 
-    # hacky hack hack to pick the solc version we want
+    set_solc(test_item)
-    env = dict(os.environ)
-    env["SOLC_VERSION"] = test_item.solc_ver
-    os.environ.clear()
-    os.environ.update(env)
 
     sl = Slither(
         test_file,
@@ -509,15 +535,56 @@ def test_parsing(test_item: Item):
     try:
         with open(expected_file, "r") as f:
             expected = json.load(f)
-    except OSError as e:
+    except OSError:
-        if e.errno != errno.ENOENT:
+        pytest.xfail("the file for this test was not generated")
-            raise
+        raise
-
-        # if the expected output doesn't exist, make it
-        with open(expected_file, "w") as f:
-            json.dump(actual, f, indent="  ")
-            expected = actual
 
     diff = DeepDiff(expected, actual, ignore_order=True, verbose_level=2)
 
     assert not diff, diff.pretty()
+
+
+def _generate_test(test_item: Item, skip_existing=False):
+    flavor = "legacy" if test_item.is_legacy else "compact"
+    test_file = os.path.join(TEST_ROOT, f"{test_item.test_id}-{test_item.base_ver}.sol")
+    expected_file = os.path.join(
+        TEST_ROOT, "expected", f"{test_item.test_id}-{test_item.solc_ver}-{flavor}.json"
+    )
+
+    if skip_existing:
+        if os.path.isfile(expected_file):
+            return
+
+    if id_test(test_item) in XFAIL:
+        return
+
+    set_solc(test_item)
+
+    sl = Slither(
+        test_file,
+        solc_force_legacy_json=test_item.is_legacy,
+        disallow_partial=True,
+        skip_analyze=True,
+    )
+
+    actual = generate_output(sl)
+
+    with open(expected_file, "w") as f:
+        json.dump(actual, f, ident="  ")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2 or sys.argv[1] not in ["--generate", "--overwrite"]:
+        print(
+            "To generate the missing json artifacts run\n\tpython tests/test_ast_parsing.py --generate"
+        )
+        print(
+            "To re-generate all the json artifacts run\n\tpython tests/test_ast_parsing.py --overwrite"
+        )
+        print("\tThis will overwrite the previous json files")
+    elif sys.argv[1] == "--generate":
+        for next_test in ALL_TESTS:
+            _generate_test(next_test, skip_existing=True)
+    elif sys.argv[1] == "--overwrite":
+        for next_test in ALL_TESTS:
+            _generate_test(next_test)