Merge pull request #2657 from sebix/viriback

 feeds: remove viriback feed, is offline

Author: sebix

CHANGELOG.md

@@ -68,7 +68,7 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o
 
 #### Parsers
 - `intelmq.bots.parsers.cymru.parser_cap_program`: Add mapping for TOR and ipv6-icmp protocol (PR#2621 by Mikk Margus Möll).
-- Remove `intelmq.bots.collectors.blueliv` as it is obsolete with the removed collector (PR#2632 by Sebastian Wagner).
+- Remove `intelmq.bots.parsers.blueliv` as it is obsolete with the removed collector (PR#2632 by Sebastian Wagner).
 - `intelmq.bots.parser.json.parser`:
   - Support data containing lists of JSON Events (PR#2545 by Tim de Boer).
   - Add default `classification.type` with value `undetermined` if input data has now classification itself (PR#2545 by Sebastian Wagner).
@@ -95,6 +95,7 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o
 ### Documentation
 - Fix and refresh links to mailing lists (PR#2609 by Kamil Mańkowski)
 - `Aggregate Bot`: Add illustration graphics (PR#2612 by Sebastian Wagner).
+- Feeds: Remove discontinued feed Viriback (PR#2567 by Sebastian Wagner).
 
 ### Packaging
 - Replace `/opt/intelmq` example paths in bots with variable `VAR_STATE_PATH` for correct paths in LSB-path setups like with packages (PR#2587 by Sebastian Wagner).

NEWS.md

@@ -18,6 +18,13 @@ Python `>=3.9` is now required, which is available on all platforms supported by
 
 ### Tools
 
+### Bots
+#### Blueliv
+The bots `intelmq.bots.collectors.blueliv` and `intelmq.bots.collectors.blueliv` are removed as they used an unmaintained library and do not work any more.
+
+#### Viriback
+The Feed *Viriback C2 Tracker* is removed as the feed and website are no longer reachable and seem to be discontinued.
+
 ### Data Format
 To save new fields from IntelMQ Data Format in existing PostgreSQL instances, the following schema
 update is necessary:
@@ -63,7 +70,7 @@ ALTER TABLE events
 ### Requirements
 Python 3.8 or newer is required.
 
-## Bots
+### Bots
 #### CIF 3 API Output deprecation
 The CIF 3 API Output bot is not compatible with Python version greater or equal to 3.12 and will be removed in the future due to lack of maintenance.
 See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html for more information.

intelmq/etc/feeds.yaml

@@ -4,31 +4,6 @@
 
 ---
 providers:
-  ViriBack:
-    C2 Tracker:
-      description: Latest detected C2 servers.
-      bots:
-        collector:
-          module: intelmq.bots.collectors.http.collector_http
-          parameters:
-            http_url: https://tracker.viriback.com/dump.php
-            rate_limit: 86400
-            name: __FEED__
-            provider: __PROVIDER__
-        parser:
-          module: intelmq.bots.parsers.generic.csv_parser
-          parameters:
-            skip_header: true
-            defaults_fields:
-              classification.type: malware-distribution
-            columns:
-              - malware.name
-              - source.url
-              - source.ip
-              - time.source
-      revision: 2022-11-15
-      documentation: https://viriback.com/
-      public: true
   Surbl:
     Malicious Domains:
       description: Detected malicious domains. Note that you have to opened up Sponsored Datafeed Service (SDS) access to the SURBL data via rsync for your IP address.

intelmq/lib/upgrades.py

@@ -42,7 +42,7 @@
            'v322_url_replacement',
            'v322_removed_feeds_and_bots',
            'v340_deprecations',
-           'v350_blueliv_removal',
+           'v350_feed_removals',
            'v350_new_fields',
            ]
 
@@ -723,7 +723,7 @@ def v301_deprecations(configuration, harmonization, dry_run, **kwargs):
             continue
         if bot["module"] == "intelmq.bots.parsers.malwaredomains.parser":
             found_malwaredomainsparser.append(bot_id)
-        if bot["module"] == "intelmq.bots.collectors.http.collector":
+        if bot["module"] == "intelmq.bots.collectors.http.collector_http":
             if "http_url" not in bot["parameters"]:
                 continue
             if bot["parameters"]["http_url"] == 'http://mirror1.malwaredomains.com/files/domains.txt':
@@ -788,7 +788,7 @@ def v310_feed_changes(configuration, harmonization, dry_run, **kwargs):
             continue
         if bot["module"] == "intelmq.bots.parsers.malc0de.parser":
             found_malc0de.append(bot_id)
-        if bot["module"] == "intelmq.bots.collectors.http.collector":
+        if bot["module"] == "intelmq.bots.collectors.http.collector_http":
             http_url = bot["parameters"].get("http_url", "")
             if http_url.startswith("https://malc0de.com/bl"):
                 found_malc0de.append(bot_id)
@@ -876,7 +876,7 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa
     messages = []
 
     for bot_id, bot in configuration.items():
-        if bot.get("module") == "intelmq.bots.collectors.http.collector":
+        if bot.get("module") == "intelmq.bots.collectors.http.collector_http":
             if bot.get("parameters", {}).get("http_url", "").startswith("https://project.turris.cz/greylist-data/greylist-latest.csv"):
                 bot["parameters"]["http_url"] = "https://view.sentinel.turris.cz/greylist-data/greylist-latest.csv"
                 messages.append("Turris Greylist feed URL updated.")
@@ -937,7 +937,7 @@ def v322_removed_feeds_and_bots(configuration, harmonization, dry_run, **kwargs)
         if bot["module"] in discontinued_bots_modules:
             discontinued_bots.append(bot_id)
 
-        elif bot["module"] == "intelmq.bots.collectors.http.collector":
+        elif bot["module"] == "intelmq.bots.collectors.http.collector_http":
             url: str = bot["parameters"].get("http_url", "")
 
             if url in discontinued_feeds_urls:
@@ -976,27 +976,37 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs):
     return message or changed, configuration, harmonization
 
 
-def v350_blueliv_removal(configuration, harmonization, dry_run, **kwargs):
+def v350_feed_removals(configuration, harmonization, dry_run, **kwargs):
     """
     Remove blueliv collector and parser
     """
-    message = None
+    messages = []
     discontinued_bots = []
     discontinued_bots_modules = (
         "intelmq.bots.collectors.blueliv.collector_crimeserver",
         "intelmq.bots.parsers.blueliv.parser_crimeserver",
     )
+    discontinued_feeds = []
 
     for bot_id, bot in configuration.items():
         if bot_id == 'global':
             continue
         if bot["module"] in discontinued_bots_modules:
             discontinued_bots.append(bot_id)
+        elif bot["module"] == "intelmq.bots.collectors.http.collector_http":
+            if bot["parameters"].get("http_url", "") == 'https://tracker.viriback.com/dump.php':
+                discontinued_feeds.append(bot_id)
+
+    if discontinued_feeds:
+        messages.append(f"Found discontinued feeds collected by bots: {', '.join(discontinued_feeds)}")
 
     if discontinued_bots:
-        message = f"Found discontinued bots: {', '.join(discontinued_bots)}. Remove the affected bots from the configuration."
+        messages.append(f"Found discontinued bots: {', '.join(discontinued_bots)}.")
+
+    if messages:
+        messages.append("Remove the affected bots from the configuration.")
 
-    return message, configuration, harmonization
+    return '\n'.join(messages) if messages else None, configuration, harmonization
 
 
 def v350_new_fields(configuration, harmonization, dry_run, **kwargs):
@@ -1058,7 +1068,7 @@ def v350_new_fields(configuration, harmonization, dry_run, **kwargs):
     ((3, 3, 0), ()),
     ((3, 3, 1), ()),
     ((3, 4, 0), (v340_deprecations, )),
-    ((3, 5, 0), (v350_blueliv_removal, v350_new_fields)),
+    ((3, 5, 0), (v350_feed_removals, v350_new_fields)),
 ])
 
 ALWAYS = (harmonization,)

intelmq/tests/lib/test_upgrades.py

@@ -516,7 +516,7 @@
         }
     },
     "malwaredomains-collector": {
-        "module": "intelmq.bots.collectors.http.collector",
+        "module": "intelmq.bots.collectors.http.collector_http",
         "parameters": {
             "http_url": "http://mirror1.malwaredomains.com/files/domains.txt"
         }
@@ -530,13 +530,13 @@
         }
     },
     "autoshun-collector": {
-        "module": "intelmq.bots.collectors.http.collector",
+        "module": "intelmq.bots.collectors.http.collector_http",
         "parameters": {
             "http_url": "https://www.autoshun.org/download"
         }
     },
     "malc0de-collector": {
-        "module": "intelmq.bots.collectors.http.collector",
+        "module": "intelmq.bots.collectors.http.collector_http",
         "parameters": {
             "http_url": "https://malc0de.com/bl/ZONES"
         }
@@ -586,7 +586,7 @@
         "module": "intelmq.bots.parsers.netlab_360.parser"
     },
     "sucuri-collector": {
-        "module": "intelmq.bots.collectors.http.collector",
+        "module": "intelmq.bots.collectors.http.collector_http",
         "parameters": {
             "http_url": "http://labs.sucuri.net/?malware"
         }
@@ -616,18 +616,23 @@
         "module": "intelmq.bots.collectors.twitter.collector",
     },
 }
-V350_BLUELIV_REMOVAL = {
+V350_FEED_REMOVAL = {
     "global": {},
     "blueliv-collector": {
         "module": "intelmq.bots.collectors.blueliv.collector_crimeserver"
     },
     "blueliv-parser": {
         "module": "intelmq.bots.parsers.blueliv.parser_crimeserver"
+    },
+    "viriback-collector": {
+        "module": "intelmq.bots.collectors.http.collector_http",
+        "parameters": {
+            "http_url": "https://tracker.viriback.com/dump.php"
+        }
     }
 }
 
 
-
 def generate_function(function):
     def test_function(self):
         """ Test if no errors happen for upgrade function %s. """ % function.__name__
@@ -865,12 +870,13 @@ def test_v340_twitter_collector(self):
         self.assertIn('twitter-collector', result[0])
         self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1])
 
-    def test_v350_blueliv_removal(self):
-        """ Test v350_blueliv_removal deprecation warning """
-        result = upgrades.v350_blueliv_removal(V350_BLUELIV_REMOVAL, {}, False)
+    def test_v350_feed(self):
+        """ Test v350_feed_removals deprecation warning """
+        result = upgrades.v350_feed_removals(V350_FEED_REMOVAL, {}, False)
         self.assertIn('blueliv-collector', result[0])
         self.assertIn('blueliv-parser', result[0])
-        self.assertEqual(V350_BLUELIV_REMOVAL, result[1])
+        self.assertIn('viriback-collector', result[0])
+        self.assertEqual(V350_FEED_REMOVAL, result[1])
 
     def test_v350_new_fields(self):
         """ Test adding new harmonisation fields """
@@ -886,7 +892,6 @@ def test_v350_new_fields(self):
         self.assertIn("severity", result[2]["event"])
 
 
-
 for name in upgrades.__all__:
     setattr(TestUpgradeLib, 'test_function_%s' % name,
             generate_function(getattr(upgrades, name)))