Merge pull request #263 from crytic/dev-generic-sourcemapping

Generic source mapped elements in JSON results

scripts/tests_generate_expected_json_4.sh

@@ -54,5 +54,5 @@ generate_expected_json(){
 #generate_expected_json tests/shadowing_builtin_symbols.sol "shadowing-builtin"
 #generate_expected_json tests/shadowing_local_variable.sol "shadowing-local"
 #generate_expected_json tests/solc_version_incorrect.sol "solc-version"
-#generate_expected_json tests/right_to_left_override.sol "rtlo"
+generate_expected_json tests/right_to_left_override.sol "rtlo"
 #generate_expected_json tests/unchecked_lowlevel.sol "unchecked-lowlevel"

slither/detectors/abstract_detector.py

@@ -2,9 +2,10 @@ import abc
 import re
 
 from slither.utils.colors import green, yellow, red
-
+from slither.core.source_mapping.source_mapping import SourceMapping
 from collections import OrderedDict
 
+
 class IncorrectDetectorInitialization(Exception):
     pass
 
@@ -33,6 +34,7 @@ classification_txt = {
     DetectorClassification.HIGH: 'High',
 }
 
+
 class AbstractDetector(metaclass=abc.ABCMeta):
     ARGUMENT = ''  # run the detector with slither.py --ARGUMENT
     HELP = ''  # help information
@@ -46,7 +48,6 @@ class AbstractDetector(metaclass=abc.ABCMeta):
     WIKI_EXPLOIT_SCENARIO = ''
     WIKI_RECOMMENDATION = ''
 
-
     def __init__(self, slither, logger):
         self.slither = slither
         self.contracts = slither.contracts
@@ -92,7 +93,6 @@ class AbstractDetector(metaclass=abc.ABCMeta):
                                        DetectorClassification.OPTIMIZATION]:
             raise IncorrectDetectorInitialization('CONFIDENCE is not initialized {}'.format(self.__class__.__name__))
 
-
     def _log(self, info):
         self.logger.info(self.color(info))
 
@@ -135,7 +135,6 @@ class AbstractDetector(metaclass=abc.ABCMeta):
                     self.logger.error(yellow('Malformed input. Example of valid input: 0,1,2,3'))
         return results
 
-
     @property
     def color(self):
         return classification_colors[self.IMPACT]
@@ -162,139 +161,152 @@ class AbstractDetector(metaclass=abc.ABCMeta):
             element['additional_fields'] = additional_fields
         return element
 
-    @staticmethod
-    def _create_parent_element(element):
+    def _create_parent_element(self, element):
         from slither.core.children.child_contract import ChildContract
         from slither.core.children.child_function import ChildFunction
         from slither.core.children.child_inheritance import ChildInheritance
         if isinstance(element, ChildInheritance):
             if element.contract_declarer:
                 contract = {'elements': []}
-                AbstractDetector.add_contract_to_json(element.contract_declarer, contract)
+                self.add_contract_to_json(element.contract_declarer, contract)
                 return contract['elements'][0]
         elif isinstance(element, ChildContract):
             if element.contract:
                 contract = {'elements': []}
-                AbstractDetector.add_contract_to_json(element.contract, contract)
+                self.add_contract_to_json(element.contract, contract)
                 return contract['elements'][0]
         elif isinstance(element, ChildFunction):
             if element.function:
                 function = {'elements': []}
-                AbstractDetector.add_function_to_json(element.function, function)
+                self.add_function_to_json(element.function, function)
                 return function['elements'][0]
         return None
 
-    @staticmethod
-    def add_variable_to_json(variable, d, additional_fields={}):
+    def add_variable_to_json(self, variable, d, additional_fields={}):
         type_specific_fields = {
-            'parent': AbstractDetector._create_parent_element(variable)
+            'parent': self._create_parent_element(variable)
         }
-        element = AbstractDetector._create_base_element('variable',
-                                                        variable.name,
-                                                        variable.source_mapping,
-                                                        type_specific_fields,
-                                                        additional_fields)
+        element = self._create_base_element('variable',
+                                            variable.name,
+                                            variable.source_mapping,
+                                            type_specific_fields,
+                                            additional_fields)
         d['elements'].append(element)
 
-    @staticmethod
-    def add_variables_to_json(variables, d):
+    def add_variables_to_json(self, variables, d):
         for variable in sorted(variables, key=lambda x:x.name):
-            AbstractDetector.add_variable_to_json(variable, d)
-
-    @staticmethod
-    def add_contract_to_json(contract, d, additional_fields={}):
-        element = AbstractDetector._create_base_element('contract',
-                                                        contract.name,
-                                                        contract.source_mapping,
-                                                        {},
-                                                        additional_fields)
+            self.add_variable_to_json(variable, d)
+
+    def add_contract_to_json(self, contract, d, additional_fields={}):
+        element = self._create_base_element('contract',
+                                            contract.name,
+                                            contract.source_mapping,
+                                            {},
+                                            additional_fields)
         d['elements'].append(element)
 
-    @staticmethod
-    def add_function_to_json(function, d, additional_fields={}):
+    def add_function_to_json(self, function, d, additional_fields={}):
         type_specific_fields = {
-            'parent': AbstractDetector._create_parent_element(function),
+            'parent': self._create_parent_element(function),
             'signature': function.full_name
         }
-        element = AbstractDetector._create_base_element('function',
-                                                        function.name,
-                                                        function.source_mapping,
-                                                        type_specific_fields,
-                                                        additional_fields)
+        element = self._create_base_element('function',
+                                            function.name,
+                                            function.source_mapping,
+                                            type_specific_fields,
+                                            additional_fields)
         d['elements'].append(element)
 
-
-    @staticmethod
-    def add_functions_to_json(functions, d, additional_fields={}):
+    def add_functions_to_json(self, functions, d, additional_fields={}):
         for function in sorted(functions, key=lambda x: x.name):
-            AbstractDetector.add_function_to_json(function, d, additional_fields)
+            self.add_function_to_json(function, d, additional_fields)
 
-    @staticmethod
-    def add_enum_to_json(enum, d, additional_fields={}):
+    def add_enum_to_json(self, enum, d, additional_fields={}):
         type_specific_fields = {
-            'parent': AbstractDetector._create_parent_element(enum)
+            'parent': self._create_parent_element(enum)
         }
-        element = AbstractDetector._create_base_element('enum',
-                                                        enum.name,
-                                                        enum.source_mapping,
-                                                        type_specific_fields,
-                                                        additional_fields)
+        element = self._create_base_element('enum',
+                                            enum.name,
+                                            enum.source_mapping,
+                                            type_specific_fields,
+                                            additional_fields)
         d['elements'].append(element)
 
-    @staticmethod
-    def add_struct_to_json(struct, d, additional_fields={}):
+    def add_struct_to_json(self, struct, d, additional_fields={}):
         type_specific_fields = {
-            'parent': AbstractDetector._create_parent_element(struct)
+            'parent': self._create_parent_element(struct)
         }
-        element = AbstractDetector._create_base_element('struct',
-                                                        struct.name,
-                                                        struct.source_mapping,
-                                                        type_specific_fields,
-                                                        additional_fields)
+        element = self._create_base_element('struct',
+                                            struct.name,
+                                            struct.source_mapping,
+                                            type_specific_fields,
+                                            additional_fields)
         d['elements'].append(element)
 
-    @staticmethod
-    def add_event_to_json(event, d, additional_fields={}):
+    def add_event_to_json(self, event, d, additional_fields={}):
         type_specific_fields = {
-            'parent': AbstractDetector._create_parent_element(event),
+            'parent': self._create_parent_element(event),
             'signature': event.full_name
         }
-        element = AbstractDetector._create_base_element('event',
-                                                        event.name,
-                                                        event.source_mapping,
-                                                        type_specific_fields,
-                                                        additional_fields)
+        element = self._create_base_element('event',
+                                            event.name,
+                                            event.source_mapping,
+                                            type_specific_fields,
+                                            additional_fields)
 
         d['elements'].append(element)
 
-    @staticmethod
-    def add_node_to_json(node, d, additional_fields={}):
+    def add_node_to_json(self, node, d, additional_fields={}):
         type_specific_fields = {
-            'parent': AbstractDetector._create_parent_element(node),
+            'parent': self._create_parent_element(node),
         }
         node_name = str(node.expression) if node.expression else ""
-        element = AbstractDetector._create_base_element('node',
-                                                        node_name,
-                                                        node.source_mapping,
-                                                        type_specific_fields,
-                                                        additional_fields)
+        element = self._create_base_element('node',
+                                            node_name,
+                                            node.source_mapping,
+                                            type_specific_fields,
+                                            additional_fields)
         d['elements'].append(element)
 
-
-    @staticmethod
-    def add_nodes_to_json(nodes, d):
+    def add_nodes_to_json(self, nodes, d):
         for node in sorted(nodes, key=lambda x: x.node_id):
-            AbstractDetector.add_node_to_json(node, d)
+            self.add_node_to_json(node, d)
 
-    @staticmethod
-    def add_pragma_to_json(pragma, d, additional_fields={}):
+    def add_pragma_to_json(self, pragma, d, additional_fields={}):
         type_specific_fields = {
             'directive': pragma.directive
         }
-        element = AbstractDetector._create_base_element('pragma',
-                                                        pragma.version,
-                                                        pragma.source_mapping,
-                                                        type_specific_fields,
-                                                        additional_fields)
+        element = self._create_base_element('pragma',
+                                            pragma.version,
+                                            pragma.source_mapping,
+                                            type_specific_fields,
+                                            additional_fields)
+        d['elements'].append(element)
 
+    def add_other_to_json(self, name, source_mapping, d, additional_fields={}):
+        # If this a tuple with (filename, start, end), convert it to a source mapping.
+        if isinstance(source_mapping, tuple):
+            # Parse the source id
+            (filename, start, end) = source_mapping
+            source_id = next((source_unit_id for (source_unit_id, source_unit_filename) in self.slither.source_units.items() if source_unit_filename == filename), -1)
+
+            # Convert to a source mapping string
+            source_mapping = f"{start}:{end}:{source_id}"
+
+        # If this is a source mapping string, parse it.
+        if isinstance(source_mapping, str):
+            source_mapping_str = source_mapping
+            source_mapping = SourceMapping()
+            source_mapping.set_offset(source_mapping_str, self.slither)
+
+        # If this is a source mapping object, get the underlying source mapping dictionary
+        if isinstance(source_mapping, SourceMapping):
+            source_mapping = source_mapping.source_mapping
+
+        # Create the underlying element and add it to our resulting json
+        element = self._create_base_element('other',
+                                            name,
+                                            source_mapping,
+                                            {},
+                                            additional_fields)
         d['elements'].append(element)

slither/detectors/source/rtlo.py

@@ -44,20 +44,39 @@ contract Token
 '''
     WIKI_RECOMMENDATION = 'Special control characters must not be allowed.'
 
+    RTLO_CHARACTER_ENCODED = "\u202e".encode('utf-8')
+
     def _detect(self):
         results = []
+        pattern = re.compile(".*\u202e.*".encode('utf-8'))
 
-        pattern = re.compile(".*\u202e.*")
         for filename, source in self.slither.source_code.items():
-            info = "{} contains a unicode right-to-left-override character:\n".format(filename)
-            found = False
-            for match in pattern.finditer(source):
-                match_line = match.group(0)
-                info += "\t- {}\n".format(match_line)
-                found = True
-
-            if found:
-                json = self.generate_json_result(info)
-                results.append(json)
+            # Attempt to find all RTLO characters in this source file.
+            original_source_encoded = source.encode('utf-8')
+            start_index = 0
+
+            # Keep searching all file contents for the character.
+            while True:
+                source_encoded = original_source_encoded[start_index:]
+                result_index = source_encoded.find(self.RTLO_CHARACTER_ENCODED)
+
+                # If we couldn't find the character in the remainder of source, stop.
+                if result_index == -1:
+                    break
+                else:
+                    # We found another instance of the character, define our output
+                    idx = start_index + result_index
+                    info = f"{filename} contains a unicode right-to-left-override character at byte offset {idx}:\n"
+
+                    # We have a patch, so pattern.find will return at least one result
+
+                    info += f"\t- {pattern.findall(source_encoded)[0]}\n"
+                    json = self.generate_json_result(info)
+                    self.add_other_to_json("rtlo-character",
+                                           (filename, idx, len(self.RTLO_CHARACTER_ENCODED)), json)
+                    results.append(json)
+
+                    # Advance the start index for the next iteration
+                    start_index = result_index + 1
 
         return results

tests/expected_json/right_to_left_override.rtlo.json

@@ -7,8 +7,26 @@
         "check": "rtlo",
         "impact": "High",
         "confidence": "High",
-        "description": "/home/travis/build/crytic/slither/tests/right_to_left_override.sol contains a unicode right-to-left-override character:\n\t-         test1(/*A\u202e/*B*/2 , 1/*\u202d\n",
-        "elements": []
+        "description": "/home/travis/build/crytic/slither/tests/right_to_left_override.sol contains a unicode right-to-left-override character at byte offset 96:\n\t- b'        test1(/*A\\xe2\\x80\\xae/*B*/2 , 1/*\\xe2\\x80\\xad'\n",
+        "elements": [
+          {
+            "type": "other",
+            "name": "rtlo-character",
+            "source_mapping": {
+              "start": 96,
+              "length": 3,
+              "filename_used": "/home/travis/build/crytic/slither/tests/right_to_left_override.sol",
+              "filename_relative": "tests/right_to_left_override.sol",
+              "filename_absolute": "/home/travis/build/crytic/slither/tests/right_to_left_override.sol",
+              "filename_short": "tests/right_to_left_override.sol",
+              "lines": [
+                7
+              ],
+              "starting_column": 18,
+              "ending_column": 21
+            }
+          }
+        ]
       }
     ]
   }

tests/expected_json/right_to_left_override.rtlo.txt

@@ -1,5 +1,5 @@
 INFO:Detectors:
-/home/travis/build/crytic/slither/tests/right_to_left_override.sol contains a unicode right-to-left-override character:
-	-         test1(/*A‮/*B*/2 , 1/*‭
+/home/travis/build/crytic/slither/tests/right_to_left_override.sol contains a unicode right-to-left-override character at byte offset 96:
+	- b'        test1(/*A\xe2\x80\xae/*B*/2 , 1/*\xe2\x80\xad'
 Reference: https://github.com/crytic/slither/wiki/Detector-Documentation#right-to-left-override-character
 INFO:Slither:tests/right_to_left_override.sol analyzed (1 contracts), 1 result(s) found