Merge pull request #60 from returntocorp/clara/add-new-hashes

feat: add 4 new hashes to finding type for alternatives to syntactic_id and match_based_id that do not change when a file or rule gets renamed

Author: chmccreery

README.md

@@ -9,3 +9,10 @@ rely on this.
 
 This repository is meant to be used as a submodule.
 
+You may need to `install opam` as a pre-requisite for contributing to this repository.
+
+To initialize your environment, run `make setup`, which should initialize the modules specified in `dev.opam`
+
+To update an interface:
+1. Make changes to the appropriate .atd file
+2. Run `make`. This will propagate that change to the respective .py, .ts, .ml, etc.

dev.opam

@@ -1,4 +1,4 @@
-# This is meant to be installed with 'opam install --deps-only .'
+# This is meant to be installed with 'opam install --deps-only . '
 #
 opam-version: "2.0"
 maintainer: "r2c"
@@ -10,4 +10,5 @@ synopsis: "OCaml development dependencies"
 depends: [
   "atdpy" {>= "2.10.0"}
   "atdts" {>= "2.10.0"}
+  "atdgen" {>= "2.10.0"}
 ]

semgrep_output_v1.atd

@@ -787,6 +787,15 @@ type api_scans_findings <ocaml attr="deriving show"> = {
    cai_ids: string list;
 }
 
+type finding_hashes <ocaml attr="deriving show"> = {
+  start_line_hash: string;
+  end_line_hash: string;
+  (* hash of the syntactic_context/code contents from start_line through end_line *)
+  code_hash: string;
+  (* hash of the rule pattern with metavariables substituted in *)
+  pattern_hash: string;
+}
+
 (* TODO: rewrite rule_matches.to_app_finding_format() *)
 type finding <ocaml attr="deriving show"> = {
   check_id: rule_id;
@@ -804,6 +813,8 @@ type finding <ocaml attr="deriving show"> = {
   syntactic_id: string;
   (* since semgrep 0.98 *)
   ?match_based_id: string option;
+  (* since semgrep 1.14.0 *)
+  ?hashes: finding_hashes option;
   metadata: raw_json;
   is_blocking: bool;
   ?fixed_lines: string list option;

semgrep_output_v1.jsonschema

@@ -87,8 +87,7 @@
           "minItems": 2,
           "items": false,
           "prefixItems": [
-            { "const": "CoreLoc" },
-            { "$ref": "#/definitions/location" }
+            { "const": "CoreLoc" }, { "$ref": "#/definitions/location" }
           ]
         },
         {
@@ -433,8 +432,7 @@
               "minItems": 2,
               "items": false,
               "prefixItems": [
-                { "$ref": "#/definitions/location" },
-                { "type": "string" }
+                { "$ref": "#/definitions/location" }, { "type": "string" }
               ]
             }
           ]
@@ -709,6 +707,18 @@
         "cai_ids": { "type": "array", "items": { "type": "string" } }
       }
     },
+    "finding_hashes": {
+      "type": "object",
+      "required": [
+        "start_line_hash", "end_line_hash", "code_hash", "pattern_hash"
+      ],
+      "properties": {
+        "start_line_hash": { "type": "string" },
+        "end_line_hash": { "type": "string" },
+        "code_hash": { "type": "string" },
+        "pattern_hash": { "type": "string" }
+      }
+    },
     "finding": {
       "type": "object",
       "required": [
@@ -729,6 +739,7 @@
         "commit_date": { "type": "string" },
         "syntactic_id": { "type": "string" },
         "match_based_id": { "type": "string" },
+        "hashes": { "$ref": "#/definitions/finding_hashes" },
         "metadata": { "$ref": "#/definitions/raw_json" },
         "is_blocking": { "type": "boolean" },
         "fixed_lines": { "type": "array", "items": { "type": "string" } },

semgrep_output_v1.py

@@ -1934,6 +1934,43 @@ def to_json_string(self, **kw: Any) -> str:
         return json.dumps(self.to_json(), **kw)
 
 
+@dataclass
+class FindingHashes:
+    """Original type: finding_hashes = { ... }"""
+
+    start_line_hash: str
+    end_line_hash: str
+    code_hash: str
+    pattern_hash: str
+
+    @classmethod
+    def from_json(cls, x: Any) -> 'FindingHashes':
+        if isinstance(x, dict):
+            return cls(
+                start_line_hash=_atd_read_string(x['start_line_hash']) if 'start_line_hash' in x else _atd_missing_json_field('FindingHashes', 'start_line_hash'),
+                end_line_hash=_atd_read_string(x['end_line_hash']) if 'end_line_hash' in x else _atd_missing_json_field('FindingHashes', 'end_line_hash'),
+                code_hash=_atd_read_string(x['code_hash']) if 'code_hash' in x else _atd_missing_json_field('FindingHashes', 'code_hash'),
+                pattern_hash=_atd_read_string(x['pattern_hash']) if 'pattern_hash' in x else _atd_missing_json_field('FindingHashes', 'pattern_hash'),
+            )
+        else:
+            _atd_bad_json('FindingHashes', x)
+
+    def to_json(self) -> Any:
+        res: Dict[str, Any] = {}
+        res['start_line_hash'] = _atd_write_string(self.start_line_hash)
+        res['end_line_hash'] = _atd_write_string(self.end_line_hash)
+        res['code_hash'] = _atd_write_string(self.code_hash)
+        res['pattern_hash'] = _atd_write_string(self.pattern_hash)
+        return res
+
+    @classmethod
+    def from_json_string(cls, x: str) -> 'FindingHashes':
+        return cls.from_json(json.loads(x))
+
+    def to_json_string(self, **kw: Any) -> str:
+        return json.dumps(self.to_json(), **kw)
+
+
 @dataclass(frozen=True)
 class CliMatchDataflowTrace:
     """Original type: cli_match_dataflow_trace = { ... }"""
@@ -1989,6 +2026,7 @@ class Finding:
     metadata: RawJson
     is_blocking: bool
     match_based_id: Optional[str] = None
+    hashes: Optional[FindingHashes] = None
     fixed_lines: Optional[List[str]] = None
     sca_info: Optional[ScaInfo] = None
     dataflow_trace: Optional[CliMatchDataflowTrace] = None
@@ -2011,6 +2049,7 @@ def from_json(cls, x: Any) -> 'Finding':
                 metadata=RawJson.from_json(x['metadata']) if 'metadata' in x else _atd_missing_json_field('Finding', 'metadata'),
                 is_blocking=_atd_read_bool(x['is_blocking']) if 'is_blocking' in x else _atd_missing_json_field('Finding', 'is_blocking'),
                 match_based_id=_atd_read_string(x['match_based_id']) if 'match_based_id' in x else None,
+                hashes=FindingHashes.from_json(x['hashes']) if 'hashes' in x else None,
                 fixed_lines=_atd_read_list(_atd_read_string)(x['fixed_lines']) if 'fixed_lines' in x else None,
                 sca_info=ScaInfo.from_json(x['sca_info']) if 'sca_info' in x else None,
                 dataflow_trace=CliMatchDataflowTrace.from_json(x['dataflow_trace']) if 'dataflow_trace' in x else None,
@@ -2035,6 +2074,8 @@ def to_json(self) -> Any:
         res['is_blocking'] = _atd_write_bool(self.is_blocking)
         if self.match_based_id is not None:
             res['match_based_id'] = _atd_write_string(self.match_based_id)
+        if self.hashes is not None:
+            res['hashes'] = (lambda x: x.to_json())(self.hashes)
         if self.fixed_lines is not None:
             res['fixed_lines'] = _atd_write_list(_atd_write_string)(self.fixed_lines)
         if self.sca_info is not None: