Add auto extraction of FireHol lists. Closes #548

api/serializers.py

@@ -127,6 +127,7 @@ class FeedsResponseSerializer(serializers.Serializer):
     attack_count = serializers.IntegerField(min_value=1)
     interaction_count = serializers.IntegerField(min_value=1)
     ip_reputation = serializers.CharField(allow_blank=True, max_length=32)
+    firehol_categories = serializers.ListField(child=serializers.CharField(max_length=64), allow_empty=True)
     asn = serializers.IntegerField(allow_null=True, min_value=1)
     destination_port_count = serializers.IntegerField(min_value=0)
     login_attempts = serializers.IntegerField(min_value=0)

api/views/utils.py

@@ -240,6 +240,7 @@ def feeds_response(iocs, feed_params, valid_feed_types, dict_only=False, verbose
                 "scanner",
                 "payload_request",
                 "ip_reputation",
+                "firehol_categories",
                 "asn",
                 "destination_ports",
                 "login_attempts",

greedybear/admin.py

@@ -5,7 +5,7 @@
 from django.contrib import admin, messages
 from django.db.models import Q
 from django.utils.translation import ngettext
-from greedybear.models import IOC, CommandSequence, CowrieSession, GeneralHoneypot, MassScanner, Sensor, Statistics, WhatsMyIPDomain
+from greedybear.models import IOC, CommandSequence, CowrieSession, FireHolList, GeneralHoneypot, MassScanner, Sensor, Statistics, WhatsMyIPDomain
 
 logger = logging.getLogger(__name__)
 
@@ -38,9 +38,24 @@ class MassScannersModelAdmin(admin.ModelAdmin):
     search_help_text = ["search for the IP address source"]
 
 
+@admin.register(FireHolList)
+class FireHolListModelAdmin(admin.ModelAdmin):
+    list_display = ["ip_address", "added", "source"]
+    list_filter = ["source"]
+    search_fields = ["ip_address"]
+    search_help_text = ["search for the IP address"]
+
+
 class SessionInline(admin.TabularInline):
     model = CowrieSession
-    fields = ["source", "start_time", "duration", "credentials", "interaction_count", "commands"]
+    fields = [
+        "source",
+        "start_time",
+        "duration",
+        "credentials",
+        "interaction_count",
+        "commands",
+    ]
     readonly_fields = fields
     show_change_link = True
     extra = 0
@@ -49,7 +64,16 @@ class SessionInline(admin.TabularInline):
 
 @admin.register(CowrieSession)
 class CowrieSessionModelAdmin(admin.ModelAdmin):
-    list_display = ["session_id", "start_time", "duration", "login_attempt", "credentials", "command_execution", "interaction_count", "source"]
+    list_display = [
+        "session_id",
+        "start_time",
+        "duration",
+        "login_attempt",
+        "credentials",
+        "command_execution",
+        "interaction_count",
+        "source",
+    ]
     search_fields = ["source__name"]
     search_help_text = ["search for the IP address source"]
     raw_id_fields = ["source", "commands"]
@@ -82,11 +106,20 @@ class IOCModelAdmin(admin.ModelAdmin):
         "cowrie",
         "general_honeypots",
         "ip_reputation",
+        "firehol_categories",
         "asn",
         "destination_ports",
         "login_attempts",
     ]
-    list_filter = ["type", "log4j", "cowrie", "scanner", "payload_request", "ip_reputation", "asn"]
+    list_filter = [
+        "type",
+        "log4j",
+        "cowrie",
+        "scanner",
+        "payload_request",
+        "ip_reputation",
+        "asn",
+    ]
     search_fields = ["name", "related_ioc__name"]
     search_help_text = ["search for the IP address source"]
     raw_id_fields = ["related_ioc"]

greedybear/celery.py

@@ -112,4 +112,9 @@ def setup_loggers(*args, **kwargs):
         "schedule": crontab(hour=4, minute=3, day_of_week=6),
         "options": {"queue": "default"},
     },
+    "extract_firehol_lists": {
+        "task": "greedybear.tasks.extract_firehol_lists",
+        "schedule": crontab(hour=4, minute=15, day_of_week=0),
+        "options": {"queue": "default"},
+    },
 }

greedybear/cronjobs/extraction/utils.py

@@ -1,13 +1,13 @@
 from collections import defaultdict
 from datetime import datetime
-from ipaddress import IPv4Address, ip_address
+from ipaddress import IPv4Address, ip_address, ip_network
 from logging import Logger
 from urllib.parse import urlparse
 
 import requests
 from django.conf import settings
 from greedybear.consts import DOMAIN, IP
-from greedybear.models import IOC, MassScanner, WhatsMyIPDomain
+from greedybear.models import IOC, FireHolList, MassScanner, WhatsMyIPDomain
 
 
 def is_whatsmyip_domain(domain: str) -> bool:
@@ -55,6 +55,8 @@ def iocs_from_hits(hits: list[dict]) -> list[IOC]:
     Convert Elasticsearch hits into IOC objects.
     Groups hits by source IP, filters out non-global addresses, and
     constructs IOC objects with aggregated data.
+    Enriches IOCs with FireHol categories at creation time to ensure
+    only fresh data is used.
 
     Args:
         hits: List of Elasticsearch hit dictionaries.
@@ -72,6 +74,26 @@ def iocs_from_hits(hits: list[dict]) -> list[IOC]:
         if extracted_ip.is_loopback or extracted_ip.is_private or extracted_ip.is_multicast or extracted_ip.is_link_local or extracted_ip.is_reserved:
             continue
 
+        # Get FireHol categories for this IP at creation time
+        # Handle both exact IP matches and network range membership (for netsets)
+        firehol_categories = []
+
+        # First check for exact IP match (for .ipset files)
+        exact_matches = FireHolList.objects.filter(ip_address=ip).values_list("source", flat=True)
+        firehol_categories.extend(exact_matches)
+
+        # Then check if IP is within any network ranges (for .netset files)
+        # Only query entries that contain '/' (CIDR notation)
+        network_entries = FireHolList.objects.filter(ip_address__contains="/")
+        for entry in network_entries:
+            try:
+                network_range = ip_network(entry.ip_address, strict=False)
+                if extracted_ip in network_range and entry.source not in firehol_categories:
+                    firehol_categories.append(entry.source)
+            except (ValueError, IndexError):
+                # Not a valid network range, skip
+                continue
+
         ioc = IOC(
             name=ip,
             type=get_ioc_type(ip),
@@ -80,6 +102,7 @@ def iocs_from_hits(hits: list[dict]) -> list[IOC]:
             asn=hits[0].get("geoip", {}).get("asn"),
             destination_ports=sorted(set(dest_ports)),
             login_attempts=len(hits) if hits[0].get("type", "") == "Heralding" else 0,
+            firehol_categories=firehol_categories,
         )
         timestamps = [hit["@timestamp"] for hit in hits if "@timestamp" in hit]
         if timestamps:

greedybear/cronjobs/firehol.py

@@ -0,0 +1,56 @@
+import requests
+from greedybear.cronjobs.base import Cronjob
+from greedybear.models import IOC, FireHolList
+
+
+class FireHolCron(Cronjob):
+    def run(self) -> None:
+        base_path = "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master"
+        sources = {
+            "blocklist_de": f"{base_path}/blocklist_de.ipset",
+            "greensnow": f"{base_path}/greensnow.ipset",
+            "bruteforceblocker": f"{base_path}/bruteforceblocker.ipset",
+            "dshield": f"{base_path}/dshield.netset",
+        }
+
+        for source, url in sources.items():
+            self.log.info(f"Processing {source} from {url}")
+            try:
+                try:
+                    response = requests.get(url, timeout=60)
+                    response.raise_for_status()
+                except requests.RequestException as e:
+                    self.log.error(f"Network error fetching {source}: {e}")
+                    continue
+
+                lines = response.text.splitlines()
+                for line in lines:
+                    line = line.strip()
+                    if not line or line.startswith("#"):
+                        continue
+
+                    # FireHol .ipset and .netset files contain IPs or CIDRs, one per line
+                    # Comments (lines starting with #) are filtered out above
+
+                    try:
+                        FireHolList.objects.get(ip_address=line, source=source)
+                    except FireHolList.DoesNotExist:
+                        FireHolList(ip_address=line, source=source).save()
+
+            except Exception as e:
+                self.log.exception(f"Unexpected error processing {source}: {e}")
+
+        # Clean up old FireHolList entries
+        self._cleanup_old_entries()
+
+    def _cleanup_old_entries(self):
+        """
+        Delete FireHolList entries older than 30 days to keep database clean.
+        """
+        from datetime import datetime, timedelta
+
+        cutoff_date = datetime.now() - timedelta(days=30)
+        deleted_count, _ = FireHolList.objects.filter(added__lt=cutoff_date).delete()
+
+        if deleted_count > 0:
+            self.log.info(f"Cleaned up {deleted_count} old FireHolList entries")

greedybear/migrations/0023_ioc_firehol_categories_alter_statistics_view_and_more.py

@@ -0,0 +1,46 @@
+# Generated by Django 5.2.8 on 2025-12-22 11:24
+
+import datetime
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("greedybear", "0022_whatsmyip"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="ioc",
+            name="firehol_categories",
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=64), blank=True, default=list, size=None),
+        ),
+        migrations.AlterField(
+            model_name="statistics",
+            name="view",
+            field=models.CharField(
+                choices=[
+                    ("feeds", "Feeds View"),
+                    ("enrichment", "Enrichment View"),
+                    ("command sequence", "Command Sequence View"),
+                    ("cowrie session", "Cowrie Session View"),
+                ],
+                default="feeds",
+                max_length=32,
+            ),
+        ),
+        migrations.CreateModel(
+            name="FireHolList",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("ip_address", models.CharField(max_length=256)),
+                ("added", models.DateTimeField(default=datetime.datetime.now)),
+                ("source", models.CharField(blank=True, max_length=64, null=True)),
+            ],
+            options={
+                "indexes": [models.Index(fields=["ip_address"], name="greedybear__ip_addr_e01f2f_idx")],
+            },
+        ),
+    ]

greedybear/migrations/0024_merge_20251223_2100.py

@@ -0,0 +1,13 @@
+# Generated by Django 5.2.8 on 2025-12-23 21:00
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("greedybear", "0023_ioc_firehol_categories_alter_statistics_view_and_more"),
+        ("greedybear", "0023_rename_massscanners_massscanner_and_more"),
+    ]
+
+    operations = []

greedybear/models.py

@@ -30,6 +30,17 @@ def __str__(self):
         return self.name
 
 
+class FireHolList(models.Model):
+    ip_address = models.CharField(max_length=256, blank=False)
+    added = models.DateTimeField(blank=False, default=datetime.now)
+    source = models.CharField(max_length=64, blank=True, null=True)
+
+    class Meta:
+        indexes = [
+            models.Index(fields=["ip_address"]),
+        ]
+
+
 class IOC(models.Model):
     name = models.CharField(max_length=256, blank=False)
     type = models.CharField(max_length=32, blank=False, choices=iocType.choices)
@@ -48,6 +59,7 @@ class IOC(models.Model):
     related_ioc = models.ManyToManyField("self", blank=True, symmetrical=True)
     related_urls = pg_fields.ArrayField(models.CharField(max_length=900, blank=True), blank=True, default=list)
     ip_reputation = models.CharField(max_length=32, blank=True)
+    firehol_categories = pg_fields.ArrayField(models.CharField(max_length=64, blank=True), blank=True, default=list)
     asn = models.IntegerField(blank=True, null=True)
     destination_ports = pg_fields.ArrayField(models.IntegerField(), blank=False, null=False, default=list)
     login_attempts = models.IntegerField(blank=False, null=False, default=0)

greedybear/tasks.py

@@ -69,3 +69,10 @@ def get_whatsmyip():
     from greedybear.cronjobs.whatsmyip import WhatsMyIPCron
 
     WhatsMyIPCron().execute()
+
+
+@shared_task()
+def extract_firehol_lists():
+    from greedybear.cronjobs.firehol import FireHolCron
+
+    FireHolCron().execute()

tests/greedybear/cronjobs/test_firehol.py

@@ -0,0 +1,59 @@
+from unittest.mock import MagicMock, patch
+
+from greedybear.cronjobs.firehol import FireHolCron
+from greedybear.models import IOC, FireHolList
+from tests import CustomTestCase
+
+
+class FireHolCronTestCase(CustomTestCase):
+    @patch("greedybear.cronjobs.firehol.requests.get")
+    def test_run(self, mock_get):
+        # Setup mock responses
+        mock_response_blocklist_de = MagicMock()
+        mock_response_blocklist_de.status_code = 200
+        mock_response_blocklist_de.text = "# blocklist_de\n1.1.1.1\n2.2.2.2"
+
+        mock_response_greensnow = MagicMock()
+        mock_response_greensnow.status_code = 200
+        mock_response_greensnow.text = "# greensnow\n3.3.3.3"
+
+        mock_response_bruteforceblocker = MagicMock()
+        mock_response_bruteforceblocker.status_code = 200
+        mock_response_bruteforceblocker.text = "# bruteforceblocker\n1.1.1.1"
+
+        mock_response_dshield = MagicMock()
+        mock_response_dshield.status_code = 200
+        mock_response_dshield.text = "# dshield\n4.4.4.0/24"
+
+        # Side effect for multiple calls
+        def side_effect(url, timeout):
+            if "blocklist_de" in url:
+                return mock_response_blocklist_de
+            elif "greensnow" in url:
+                return mock_response_greensnow
+            elif "bruteforceblocker" in url:
+                return mock_response_bruteforceblocker
+            elif "dshield" in url:
+                return mock_response_dshield
+            return MagicMock(status_code=404)
+
+        mock_get.side_effect = side_effect
+
+        # Run the cronjob
+        cronjob = FireHolCron()
+        cronjob.execute()
+
+        # Check FireHolList entries were created
+        self.assertTrue(FireHolList.objects.filter(ip_address="1.1.1.1", source="blocklist_de").exists())
+        self.assertTrue(FireHolList.objects.filter(ip_address="2.2.2.2", source="blocklist_de").exists())
+        self.assertTrue(FireHolList.objects.filter(ip_address="3.3.3.3", source="greensnow").exists())
+        self.assertTrue(FireHolList.objects.filter(ip_address="1.1.1.1", source="bruteforceblocker").exists())
+        self.assertTrue(FireHolList.objects.filter(ip_address="4.4.4.0/24", source="dshield").exists())
+
+        # Verify FireHolList data is available for IOC enrichment at creation time
+        # (Note: Enrichment now happens in iocs_from_hits during IOC creation, not here)
+        firehol_entries = FireHolList.objects.filter(ip_address="1.1.1.1")
+        self.assertEqual(firehol_entries.count(), 2)
+        sources = list(firehol_entries.values_list("source", flat=True))
+        self.assertIn("blocklist_de", sources)
+        self.assertIn("bruteforceblocker", sources)

tests/test_serializers.py

@@ -115,6 +115,7 @@ def test_valid_fields(self):
                 "attack_count": "5",
                 "interaction_count": "50",
                 "ip_reputation": "known attacker",
+                "firehol_categories": [],
                 "asn": "8400",
                 "destination_port_count": "14",
                 "login_attempts": "0",