feat: implement CAS for Data Model deduplication

Author: ayushgupta704

api_app/analyzers_manager/models.py

@@ -121,14 +121,20 @@ def _create_data_model_dictionary(self) -> Dict:
         return result
 
     def create_data_model(self) -> Optional[BaseDataModel]:
-        # TODO we don't need to actually crate a new object every time.
-        #  if the report is the same of the previous one, we can just link it
         if not self._validation_before_data_model():
             return None
         dictionary = self._create_data_model_dictionary()
+        temp_instance = self.data_model_class()
+        fingerprint = temp_instance.generate_fingerprint(data=dictionary)
+        existing_data_model = self.data_model_class.objects.filter(fingerprint=fingerprint).first()
+
+        if existing_data_model:
+            logger.info(f"Deduplicated: Linking existing Data Model {existing_data_model.pk}")
+            self.data_model = existing_data_model
+        else:
+            self.data_model: BaseDataModel = self.data_model_class.objects.create(fingerprint=fingerprint)
+            self.data_model.merge(dictionary)
 
-        self.data_model: BaseDataModel = self.data_model_class.objects.create()
-        self.data_model.merge(dictionary)
         self.save()
         return self.data_model
 

api_app/data_model_manager/models.py

@@ -1,6 +1,7 @@
+import hashlib
 import json
 import logging
-from typing import Dict, Type, Union
+from typing import Any, Dict, Type, Union
 
 from django.contrib.contenttypes.fields import GenericRelation
 from django.contrib.contenttypes.models import ContentType
@@ -26,6 +27,14 @@
 logger = logging.getLogger(__name__)
 
 
+def normalize_dict(obj: Any) -> Any:
+    if isinstance(obj, dict):
+        return {k: normalize_dict(v) for k, v in sorted(obj.items())}
+    if isinstance(obj, list):
+        return [normalize_dict(i) for i in obj]
+    return obj
+
+
 class IETFReport(models.Model):
     rrname = LowercaseCharField(max_length=100)
     rrtype = LowercaseCharField(max_length=100)
@@ -91,6 +100,7 @@ class BaseDataModel(models.Model):
         default=dict
     )  # field for additional information related to a specific analyzer
     date = models.DateTimeField(default=now)
+    fingerprint = models.CharField(max_length=64, db_index=True, blank=True, default="")
     analyzers_report = GenericRelation(
         to="analyzers_manager.AnalyzerReport",
         object_id_field="data_model_object_id",
@@ -125,6 +135,30 @@ def owner(self) -> User:
         elif self.jobs.exists():
             return self.jobs.first().user
 
+    def get_content_map(self, data: Dict = None) -> Dict:
+        if data is None:
+            data = {}
+            for field in self._meta.fields:
+                name = field.name
+                if name in ["id", "date", "fingerprint"]:
+                    continue
+                value = getattr(self, name)
+                if hasattr(value, "isoformat"):
+                    value = value.isoformat()
+                data[name] = value
+        data.pop("id", None)
+        data.pop("date", None)
+        data.pop("fingerprint", None)
+        data.pop("analyzers_report", None)
+        data.pop("jobs", None)
+        data.pop("user_events", None)
+        return normalize_dict(data)
+
+    def generate_fingerprint(self, data: Dict = None) -> str:
+        content_map = self.get_content_map(data)
+        encoded_data = json.dumps(content_map, sort_keys=True).encode("utf-8")
+        return hashlib.sha256(encoded_data).hexdigest()
+
     def merge(self, other: Union["BaseDataModel", Dict], append: bool = True) -> "BaseDataModel":
         if not self.pk:
             raise ValueError("Unable to merge a model that was not saved.")