Merge pull request #23 from nbargnesi/report-api

start building out MongoDB impl of report API

Author: rkoumis

lib/cuckoo/core/reporting/__init__.py

@@ -0,0 +1,44 @@
+import enum
+
+from lib.cuckoo.common import config
+from .api import Reports
+from .backends import mongodb, elasticsearch, null
+
+
+class Backend(enum.Enum):
+    MONGODB = enum.auto()
+    ELASTICSEARCH = enum.auto()
+    NONE = enum.auto()
+
+
+# Reports are disabled by default. Enable it by configuring MongoDB or Elasticsearch
+# and calling init_reports.
+_enabled: bool = False
+_backend: Backend = Backend.NONE
+
+
+def enabled():
+    return _enabled
+
+
+def disabled():
+    return not enabled()
+
+
+def configured(cfg: config.Config) -> bool:
+    if cfg.mongodb.enabled:
+        return True
+    if cfg.elasticsearchdb.enabled:
+        return True
+    return False
+
+
+def init_reports(cfg: config.Config) -> Reports:
+    global _enabled, _backend
+    if cfg.mongodb.enabled:
+        _enabled, _backend = True, Backend.MONGODB
+        return mongodb.MongoDBReports(cfg)
+    elif cfg.elasticsearchdb.enabled:
+        _enabled, _backend = True, Backend.ELASTICSEARCH
+        return elasticsearch.ElasticsearchReports(cfg)
+    _enabled, _backend = False, null.Null(cfg)

lib/cuckoo/core/reporting/api.py

@@ -0,0 +1,32 @@
+class Reports:
+    """Interface to report backends."""
+
+    def get(self, task_id: int) -> dict:
+        raise NotImplementedError()
+
+    def delete(self, task_id: int) -> bool:
+        raise NotImplementedError()
+
+    def search(self, term, value, limit=False, projection=None) -> list:
+        raise NotImplementedError()
+
+    def search_by_user(self, term, value, user_id=False, privs=False) -> list:
+        raise NotImplementedError()
+
+    def search_by_sha256(self, sha256: str, limit=False) -> list:
+        raise NotImplementedError()
+
+    def cape_configs(self, task_id: int) -> dict:
+        raise NotImplementedError()
+
+    def detections_by_sha256(self, sha256: str) -> dict:
+        raise NotImplementedError()
+
+    def iocs(self, task_id: int) -> dict:
+        raise NotImplementedError()
+
+    def summary(self, task_id: int) -> dict:
+        raise NotImplementedError()
+
+    def recent_suricata_alerts(self, minutes=60) -> list:
+        raise NotImplementedError()

lib/cuckoo/core/reporting/backends/__init__.py

@@ -0,0 +1,37 @@
+from lib.cuckoo.common import config
+from lib.cuckoo.core.reporting import api
+
+
+class ElasticsearchReports(api.Reports):
+    def __init__(self, cfg: config.Config):
+        pass
+
+    def get(self, task_id: int) -> dict:
+        pass
+
+    def delete(self, task_id: int) -> bool:
+        pass
+
+    def search(self, term, value, limit=False, projection=None) -> list:
+        pass
+
+    def search_by_user(self, term, value, user_id=False, privs=False) -> list:
+        pass
+
+    def search_by_sha256(self, sha256: str, limit=False) -> list:
+        pass
+
+    def cape_configs(self, task_id: int) -> dict:
+        pass
+
+    def detections_by_sha256(self, sha256: str) -> dict:
+        pass
+
+    def iocs(self, task_id: int) -> dict:
+        pass
+
+    def summary(self, task_id: int) -> dict:
+        pass
+
+    def recent_suricata_alerts(self, minutes=60) -> list:
+        pass

lib/cuckoo/core/reporting/backends/elasticsearch.py

@@ -0,0 +1,111 @@
+import logging
+
+import pymongo
+import pymongo.collection
+import pymongo.database
+import pymongo.results
+
+from lib.cuckoo.common import config
+from lib.cuckoo.common.exceptions import CuckooOperationalError
+from lib.cuckoo.core.reporting import api
+
+log = logging.getLogger(__name__)
+
+
+class MongoDBReports(api.Reports):
+    def __init__(self, cfg: config.Config):
+        if not hasattr(cfg, "mongodb"):
+            raise CuckooOperationalError("mongodb must be configured")
+        mongodb_cfg: dict = cfg.mongodb
+
+        self._client: pymongo.MongoClient = _pymongo_client(mongodb_cfg)
+        dbname = mongodb_cfg.get("db", "cuckoo")
+        self._database: pymongo.database.Database = self._client[dbname]
+        self._reports: pymongo.collection.Collection = self._database[_analysis_coll]
+
+        _init_pymongo_logging(mongodb_cfg)
+
+    def get(self, task_id: int) -> dict:
+        query = {_info_id: task_id}
+        report = self._reports.find_one(filter=query)
+        return {} if not report else report
+
+    def delete(self, task_id: int) -> bool:
+        query = {_info_id: task_id}
+        rslt: pymongo.results.DeleteResult = self._reports.delete_one(filter=query)
+        return True if rslt.deleted_count > 0 else False
+
+    def search(self, term, value, limit=False, projection=None) -> list:
+        pass
+
+    def search_by_user(self, term, value, user_id=False, privs=False) -> list:
+        pass
+
+    def search_by_sha256(self, sha256: str, limit=False) -> list:
+        pass
+
+    def cape_configs(self, task_id: int) -> dict:
+        pass
+
+    def detections_by_sha256(self, sha256: str) -> dict:
+        pass
+
+    def iocs(self, task_id: int) -> dict:
+        # there's no well-defined representation of iocs data yet; defer to full get
+        return self.get(task_id)
+
+    def summary(self, task_id: int) -> dict:
+        query = {_info_id: task_id}
+        projection = {
+            _id: 0,
+            _info: 1,
+            "target.file.virustotal.summary": 1,
+            "url.virustotal.summary": 1,
+            "malscore": 1,
+            "detections": 1,
+            "network.pcap_sha256": 1,
+            "mlist_cnt": 1,
+            "f_mlist_cnt": 1,
+            "target.file.clamav": 1,
+            "suri_tls_cnt": 1,
+            "suri_alert_cnt": 1,
+            "suri_http_cnt": 1,
+            "suri_file_cnt": 1,
+            "trid": 1,
+        }
+        report = self._reports.find_one(filter=query, projection=projection)
+        return {} if not report else report
+
+    def recent_suricata_alerts(self, minutes=60) -> list:
+        pass
+
+
+# Temporarily duped with mongodb_constants
+_analysis_coll = "analysis"
+_calls_coll = "calls"
+_cuckoo_coll = "cuckoo_schema"
+_files_coll = "files"
+_file_key = "sha256"
+_id = "_id"
+_info = "info"
+_info_id = "info.id"
+_target = "target"
+_task_ids_key = "_task_ids"
+_version = "version"
+
+
+def _pymongo_client(cfg: dict) -> pymongo.MongoClient:
+    return pymongo.MongoClient(
+        host=cfg.get("host", "127.0.0.1"),
+        port=cfg.get("port", 27017),
+        username=cfg.get("username"),
+        password=cfg.get("password"),
+        authSource=cfg.get("authsource", "cuckoo"),
+        tlsCAFile=cfg.get("tlscafile", None),
+    )
+
+
+def _init_pymongo_logging(cfg: dict) -> None:
+    mongodb_log_level = cfg.get("log_level", "ERROR")
+    level = logging.getLevelName(mongodb_log_level.upper())
+    logging.getLogger("pymongo").setLevel(level)

lib/cuckoo/core/reporting/backends/mongodb.py

@@ -0,0 +1,11 @@
+from lib.cuckoo.common import config
+from lib.cuckoo.common.exceptions import CuckooOperationalError
+from lib.cuckoo.core.reporting import api
+
+
+class Null(api.Reports):
+    def __init__(self, cfg: config.Config):
+        pass
+
+    def __getattr__(self, name):
+        raise CuckooOperationalError("reporting backend does not support '%s'" % name)