Merge pull request #2 from honeynet/dev

0.0.1

Author: mlodic

.github/dependabot.yml

@@ -0,0 +1,35 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "tuesday"
+    target-branch: "dev"
+    open-pull-requests-limit: 1
+    ignore:
+      # ignore all patch updates since we are using ~=
+      # this does not work for security updates
+      - dependency-name: "*"
+        update-types: [ "version-update:semver-patch" ]
+
+  - package-ecosystem: "docker"
+    directory: "/docker"
+    schedule:
+      interval: "weekly"
+      day: "tuesday"
+    target-branch: "dev"
+    open-pull-requests-limit: 1
+    ignore:
+      # ignore all patch updates since we are using ~=
+      # this does not work for security updates
+      - dependency-name: "*"
+        update-types: [ "version-update:semver-patch" ]
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "tuesday"
+    target-branch: "dev"
+    open-pull-requests-limit: 1

README.md

@@ -1,6 +1,6 @@
 # GreedyBear
 
-The project goal is to extract data of the attacks detected by a TPOT or a cluster of them and to generate some feeds that can be used to prevent and detect attacks.
+The project goal is to extract data of the attacks detected by a [TPOT](https://github.com/telekom-security/tpotce) or a cluster of them and to generate some feeds that can be used to prevent and detect attacks.
 
 ## Available Feeds
 The feeds are reachable through the following URL: 
@@ -10,7 +10,8 @@ https://<greedybear_site>/api/feeds/<feed_type>/<attack_type>/<age>.<format>
 
 The available `feed_type` are:
 
-* `log4j`: attacks detected from the Log4pot.
+* `log4j`: attacks detected from the [Log4pot](https://github.com/thomaspatzke/Log4Pot).
+* `cowrie`: attacks detected from the [Cowrie Honeypot](https://github.com/cowrie/cowrie)
 
 The available `attack_type` are:
 

api/views.py

@@ -43,7 +43,7 @@ def feeds(request, feed_type, attack_type, age, format_):
         f" Age: {age}, format: {format_}"
     )
 
-    feed_choices = "log4j"
+    feed_choices = ["log4j", "cowrie"]
     if feed_type not in feed_choices:
         return _formatted_bad_request(format_)
 
@@ -61,7 +61,7 @@ def feeds(request, feed_type, attack_type, age, format_):
 
     query_dict = {}
 
-    if feed_type == "log4j":
+    if feed_type:
         query_dict[feed_type] = True
     else:
         logger.error("this is impossible. check the code")
@@ -74,7 +74,7 @@ def feeds(request, feed_type, attack_type, age, format_):
         # everything in the last 3 days
         three_days_ago = datetime.utcnow() - timedelta(days=3)
         query_dict["last_seen__gte"] = three_days_ago
-        iocs = IOC.objects.filter(**query_dict).order_by("-last_seen")[:1000]
+        iocs = IOC.objects.filter(**query_dict).order_by("-last_seen")[:5000]
     elif age == "persistent":
         # scanners detected in the last 14 days
         fourteen_days_ago = datetime.utcnow() - timedelta(days=14)
@@ -83,7 +83,7 @@ def feeds(request, feed_type, attack_type, age, format_):
         number_of_days_seen = 10
         query_dict["number_of_days_seen__gte"] = number_of_days_seen
         # order by the number of times seen
-        iocs = IOC.objects.filter(**query_dict).order_by("-times_seen")[:100]
+        iocs = IOC.objects.filter(**query_dict).order_by("-times_seen")[:1000]
     else:
         logger.error("this is impossible. check the code")
         return HttpResponseServerError()

greedybear/celery.py

@@ -4,6 +4,7 @@
 
 from celery import Celery
 from celery.schedules import crontab
+from celery.signals import setup_logging
 from django.conf import settings
 from kombu import Exchange, Queue
 
@@ -43,6 +44,15 @@
 )
 
 
+@setup_logging.connect
+def setup_loggers(*args, **kwargs):
+    from logging.config import dictConfig
+
+    from django.conf import settings
+
+    dictConfig(settings.LOGGING)
+
+
 app.conf.beat_schedule = {
     # every 10 minutes
     "extract_attacks": {

greedybear/cronjobs/attacks.py

@@ -1,23 +1,19 @@
-import base64
-import re
+from abc import ABCMeta
 from datetime import datetime
 from ipaddress import IPv4Address
-from urllib.parse import urlparse
 
 from greedybear.consts import PAYLOAD_REQUEST, SCANNER
-from greedybear.cronjobs.base import ExtractDataFromElastic, Honeypot
+from greedybear.cronjobs.base import ExtractDataFromElastic
 from greedybear.cronjobs.sensors import ExtractSensors
 from greedybear.models import IOC, Sensors
-from greedybear.regex import REGEX_CVE_BASE64COMMAND, REGEX_CVE_LOG4J, REGEX_URL
 
 
-class ExtractAttacks(ExtractDataFromElastic):
+class ExtractAttacks(ExtractDataFromElastic, metaclass=ABCMeta):
     class IOCWhitelist(Exception):
         pass
 
     def __init__(self):
         super().__init__()
-        self.honeypot = Honeypot("Log4pot")
         self.first_time_run = False
 
     @property
@@ -28,147 +24,7 @@ def minutes_back_to_lookup(self):
             minutes = 11
         return minutes
 
-    def _log4pot_lookup(self):
-        search = self._base_search(self.honeypot)
-        # we want to get only probes that tried to exploit the specific log4j CVE
-        search = search.filter("term", reason="exploit")
-        search = search.source(["deobfuscated_payload", "correlation_id"])
-        hits = search[:10000].execute()
-
-        url = None
-        hostname = None
-        hidden_url = None
-        hidden_hostname = None
-        added_scanners = 0
-        added_payloads = 0
-        added_hidden_payloads = 0
-
-        for hit in hits:
-            scanner_ip = self._get_scanner_ip(hit.correlation_id)
-
-            match = re.search(REGEX_CVE_LOG4J, hit.deobfuscated_payload)
-            if match:
-                # we are losing the protocol but that's ok for now
-                url = match.group()
-                url_adjusted = "tcp:" + url
-                # removing double slash
-                url = url[2:]
-                self.log.info(f"found URL {url} in payload for CVE-2021-44228")
-                # protocol required or extraction won't work
-                hostname = urlparse(url_adjusted).hostname
-                self.log.info(f"extracted hostname {hostname} from {url}")
-
-            # it is possible to extract another payload from base64 encoded string.
-            # this is a behavior related to the attack that leverages LDAP
-            match_command = re.search(REGEX_CVE_BASE64COMMAND, hit.deobfuscated_payload)
-            if match_command:
-                # we are losing the protocol but that's ok for now
-                base64_encoded = match_command.group(1)
-                self.log.info(
-                    f"found base64 encoded command {base64_encoded}"
-                    f" in payload from base64 code for CVE-2021-44228"
-                )
-                try:
-                    decoded_str = base64.b64decode(base64_encoded).decode()
-                    self.log.info(
-                        f"decoded base64 command to {decoded_str}"
-                        f" from payload from base64 code for CVE-2021-44228"
-                    )
-                except Exception as e:
-                    self.log.warning(e, stack_info=True)
-                else:
-                    match_url = re.search(REGEX_URL, decoded_str)
-                    if match_url:
-                        hidden_url = match_url.group()
-                        if "://" not in hidden_url:
-                            hidden_url = "tcp://" + hidden_url
-                        self.log.info(
-                            f"found hidden URL {hidden_url}"
-                            f" in payload for CVE-2021-44228"
-                        )
-
-                        hidden_hostname = urlparse(hidden_url).hostname
-                        self.log.info(
-                            f"extracted hostname {hidden_hostname} from {hidden_url}"
-                        )
-
-            # add scanner
-            if scanner_ip:
-                self._add_ioc(scanner_ip, SCANNER)
-                added_scanners += 1
-
-            # add first URL
-            if hostname:
-                related_urls = [url] if url else None
-                self._add_ioc(hostname, PAYLOAD_REQUEST, related_urls=related_urls)
-                added_payloads += 1
-
-            # add hidden URL
-            if hidden_hostname:
-                related_urls = [hidden_url] if hidden_url else None
-                self._add_ioc(
-                    hidden_hostname, PAYLOAD_REQUEST, related_urls=related_urls
-                )
-                added_hidden_payloads += 1
-
-            # once all have added, we can add the foreign keys
-            self._add_fks(scanner_ip, hostname, hidden_hostname)
-
-        self.log.info(
-            f"added {added_scanners} scanners, {added_payloads} payloads"
-            f" and {added_hidden_payloads} hidden payloads"
-        )
-
-    def _add_fks(self, scanner_ip, hostname, hidden_hostname):
-        self.log.info(
-            f"adding foreign keys for the following iocs: {scanner_ip}, {hostname}, {hidden_hostname}"
-        )
-        scanner_ip_instance = IOC.objects.filter(name=scanner_ip).first()
-        hostname_instance = IOC.objects.filter(name=hostname).first()
-        hidden_hostname_instance = IOC.objects.filter(name=hidden_hostname).first()
-
-        if scanner_ip_instance:
-            if (
-                hostname_instance
-                and hostname_instance not in scanner_ip_instance.related_ioc.all()
-            ):
-                scanner_ip_instance.related_ioc.add(hostname_instance)
-            if (
-                hidden_hostname_instance
-                and hidden_hostname_instance
-                not in scanner_ip_instance.related_ioc.all()
-            ):
-                scanner_ip_instance.related_ioc.add(hidden_hostname_instance)
-            scanner_ip_instance.save()
-
-        if hostname_instance:
-            if (
-                scanner_ip_instance
-                and scanner_ip_instance not in hostname_instance.related_ioc.all()
-            ):
-                hostname_instance.related_ioc.add(scanner_ip_instance)
-            if (
-                hidden_hostname_instance
-                and hidden_hostname_instance not in hostname_instance.related_ioc.all()
-            ):
-                hostname_instance.related_ioc.add(hidden_hostname_instance)
-            hostname_instance.save()
-
-        if hidden_hostname_instance:
-            if (
-                hostname_instance
-                and hostname_instance not in hidden_hostname_instance.related_ioc.all()
-            ):
-                hidden_hostname_instance.related_ioc.add(hostname_instance)
-            if (
-                scanner_ip_instance
-                and scanner_ip_instance
-                not in hidden_hostname_instance.related_ioc.all()
-            ):
-                hidden_hostname_instance.related_ioc.add(scanner_ip_instance)
-            hidden_hostname_instance.save()
-
-    def _add_ioc(self, ioc, attack_type, related_urls=None):
+    def _add_ioc(self, ioc, attack_type, related_urls=None, log4j=False, cowrie=False):
         self.log.info(
             f"saving ioc {ioc} for attack_type {attack_type} and related_urls {related_urls}"
         )
@@ -202,7 +58,11 @@ def _add_ioc(self, ioc, attack_type, related_urls=None):
             if attack_type == PAYLOAD_REQUEST:
                 ioc_instance.payload_request = True
 
-            ioc_instance.log4j = True
+            if log4j:
+                ioc_instance.log4j = True
+
+            if cowrie:
+                ioc_instance.cowrie = True
 
             if ioc_instance:
                 ioc_instance.save()
@@ -227,42 +87,16 @@ def _get_ioc_type(self, ioc):
             ioc_type = "ip"
         return ioc_type
 
-    def _get_scanner_ip(self, correlation_id):
-        self.log.info(f"extracting scanner IP from correlation_id {correlation_id}")
-        scanner_ip = None
-        search = self._base_search(self.honeypot)
-        search = search.filter(
-            "term", **{"correlation_id.keyword": str(correlation_id)}
-        )
-        search = search.filter("term", reason="request")
-        search = search.source(["src_ip"])
-        # only one should be available
-        hits = search[:10].execute()
-        for hit in hits:
-            scanner_ip = hit.src_ip
-            break
-
-        if scanner_ip:
-            self.log.info(
-                f"extracted scanner IP {scanner_ip} from correlation_id {correlation_id}"
-            )
-        else:
-            self.log.warning(
-                f"scanner IP was not extracted from correlation_id {correlation_id}"
-            )
-
-        return scanner_ip
-
-    def _check_first_time_run(self):
+    def _check_first_time_run(self, honeypot_flag):
         all_ioc = IOC.objects.all()
         if not all_ioc:
-            # first time we execute this project.
-            # So we increment the time range to get the data from the last 3 days
-            self.first_time_run = True
             # plus, we extract the sensors addresses so we can whitelist them
             ExtractSensors().execute()
-
-    def run(self):
-        self._healthcheck()
-        self._check_first_time_run()
-        self._log4pot_lookup()
+            self.first_time_run = True
+        else:
+            # if this is not the overall first time, it could that honeypot first time
+            honeypot_ioc = IOC.objects.filter(**{f"{honeypot_flag}": True})
+            if not honeypot_ioc:
+                # first time we execute this project.
+                # So we increment the time range to get the data from the last 3 days
+                self.first_time_run = True

greedybear/cronjobs/base.py

@@ -1,17 +1,10 @@
 import logging
 from abc import ABCMeta, abstractmethod
-from dataclasses import dataclass
 
 from django.conf import settings
 from elasticsearch_dsl import Search
 
 
-@dataclass
-class Honeypot:
-    name: str
-    description: str = ""
-
-
 class ExtractDataFromElastic(metaclass=ABCMeta):
     class ElasticServerDownException(Exception):
         pass