Merge pull request #409 from target/iqy_scanner

Adding IQY Scanner

Author: phutelmyer

CHANGELOG.md

@@ -1,6 +1,12 @@
 # Changelog
 Changes to the project will be tracked in this file via the date of change.
 
+## 2023-11-04
+- Added `ScanIqy` to target and extract network addressed from IQY (Internet Query) files
+- Added tests for `ScanIqy`
+- Fix for a `poetry` build issue
+- Fix for `ScanPcap` tests
+
 ## 2023-10-25
 - Changes to `ScanExiftool` scanner and tests
 - Update `google.golang.org/grpc` dependency

build/python/backend/Dockerfile

@@ -153,15 +153,21 @@ RUN mkdir jtr && cd jtr && git init && git remote add origin https://github.com/
     chmod -R 777 /jtr && \
     chown -R $USER_UID:$USER_UID /jtr
 
-# Poetry setup
-RUN curl -sSL https://install.python-poetry.org | python3 - && \
-    export PATH="/root/.local/bin:$PATH" && \
-    poetry config virtualenvs.create false
+# Install Poetry globally and copy project files
+RUN python3 -m pip install -U pip setuptools && \
+    python3 -m pip install poetry && \
+    rm -rf /root/.cache/pip
 
-# Project setup
-COPY ./pyproject.toml ./poetry.lock /strelka/
+# Set the working directory and copy the project files
 WORKDIR /strelka/
-RUN /root/.local/bin/poetry install --no-dev
+COPY pyproject.toml poetry.lock ./
+
+# Use Poetry to install the project dependencies globally
+# This step is after the COPY step because it is more likely to change,
+# and therefore should not be included in earlier layers that can be cached.
+RUN poetry config virtualenvs.create false && \
+    poetry install --no-dev && \
+    rm -rf /root/.cache/pypoetry
 
 # Copy Strelka files
 COPY ./src/python/ /strelka/

configs/python/backend/backend.yaml

@@ -249,6 +249,11 @@ scanners:
       priority: 5
       options:
         limit: 50
+  'ScanIqy':
+    - positive:
+        flavors:
+          - 'iqy_file'
+      priority: 5
   'ScanJarManifest':
     - positive:
         flavors:

configs/python/backend/taste/taste.yara

@@ -335,6 +335,23 @@ rule excel4_file
         (uint32be(0) == 0x504b0304 and $rels and $sheet and $xlsstr)
 }
 
+rule iqy_file {
+   meta:
+      description = "Detects potential IQY (Excel Web Query) files with various protocols"
+      author = "Paul Hutelmyer"
+      date = "2023-11-02"
+   strings:
+      $iqy_header = /^WEB\n/ nocase
+      $http = /http:\/\// nocase
+      $https = /https:\/\// nocase
+      $ftp = /ftp:\/\// nocase
+      $ftps = /ftps:\/\// nocase
+      $file = /file:\/\// nocase
+      $smb = /smb:\/\// nocase
+condition:
+    $iqy_header at 0 and ($http or $https or $ftp or $ftps or $file or $smb)
+}
+
 rule onenote_file
 {
     meta:

docs/README.md

@@ -2,7 +2,11 @@
 name = "strelka-worker"
 version = "0.23.10.19"
 description = "Strelka's backend Python worker"
-authors = ["Paul.Hutelmyer@Target.com", "Ryan.Ohoro@Target.com", "Sara.Kalupa@Target.com"]
+authors = [
+    "Paul Hutelmyer <Paul.Hutelmyer@Target.com>",
+    "Ryan Ohoro <Ryan.Ohoro@Target.com>",
+    "Sara Kalupa <Sara.Kalupa@Target.com>"
+]
 
 [tool.poetry.dependencies]
 python = ">=3.10,<=3.12"

pyproject.toml

@@ -0,0 +1,64 @@
+# Description #
+# This scanner is looking for iqy files used with excel.
+#
+# author: Tasha Taylor
+# date: 10/30/2023
+
+import re
+
+from strelka import strelka
+
+
+class ScanIqy(strelka.Scanner):
+    """
+    Extract URLs from IQY files.
+
+    IQY files, or Excel Web Query Internet Inquire files, are typically created from a VBA Web Query output.
+    The following is a typical format:
+        WEB
+        1
+        [URL]
+        [optional parameters]
+    Additional properties can be found at: https://learn.microsoft.com/en-us/office/vba/api/excel.querytable
+    """
+
+    def scan(self, data, file, options, expire_at):
+        try:
+            # Regular expression for detecting a URL-like pattern
+            address_pattern = re.compile(
+                r"\b(?:http|https|ftp|ftps|file|smb)://\S+|"
+                r"\\{2}\w+\\(?:[\w$]+\\)*[\w$]+",
+                re.IGNORECASE,
+            )
+
+            # Attempt UTF-8 decoding first, fall back to latin-1 if necessary
+            try:
+                data = data.decode("utf-8")
+            except UnicodeDecodeError:
+                data = data.decode("latin-1")
+
+            # Split lines to review each record separately
+            data_lines = data.splitlines()
+
+            addresses = set()
+            # For each line, check if the line matches the address pattern.
+            # In a typical IQY file, the "WEB" keyword is at the beginning of the file,
+            # and what follows is usually just one URL with optional additional parameters.
+            # However, because we are iterating lines anyway, lets check for additional addresses anyway.
+            for entry in data_lines[1:]:
+                match = address_pattern.search(entry)
+                if match:
+                    address = match.group().strip()
+                    if address:
+                        addresses.add(address)
+
+            # Evaluate if any addresses were found and assign the boolean result.
+            self.event["address_found"] = bool(addresses)
+
+            # Send all addresses to the IOC parser.
+            self.add_iocs(list(addresses), self.type.url)
+
+        except UnicodeDecodeError as e:
+            self.flags.append(f"Unicode decoding error: {e}")
+        except Exception as e:
+            self.flags.append(f"Unexpected exception: {e}")

src/python/strelka/scanners/scan_iqy.py

@@ -47,6 +47,10 @@ def run_test_scan(
         expire_at=datetime.date.today(),
     )
 
+    # If a scanner outputs IOCs, append them to the event for test coverage
+    if scanner.iocs:
+        scanner.event.update({"iocs": scanner.iocs})
+
     return scanner.event
 
 

src/python/strelka/tests/__init__.py

@@ -0,0 +1,3 @@
+WEB
+1
+https://github.com/target/strelka/blob/master/docs/index.html  // Test case: Valid HTTPS URL

src/python/strelka/tests/fixtures/test.iqy

@@ -0,0 +1,40 @@
+from pathlib import Path
+from unittest import TestCase, mock
+
+from strelka.scanners.scan_iqy import ScanIqy as ScanUnderTest
+from strelka.tests import run_test_scan
+
+
+def test_scan_iqy(mocker):
+    """
+    Pass: Sample event matches output of scanner.
+    Failure: Unable to load file or sample event fails to match.
+    """
+
+    test_scan_event = {
+        "elapsed": mock.ANY,
+        "flags": [],
+        "address_found": True,
+        "iocs": [
+            {
+                "description": "",
+                "ioc": "github.com",
+                "ioc_type": "domain",
+                "scanner": "ScanIqy",
+            },
+            {
+                "description": "",
+                "ioc": "https://github.com/target/strelka/blob/master/docs/index.html",
+                "ioc_type": "url",
+                "scanner": "ScanIqy",
+            },
+        ],
+    }
+    scanner_event = run_test_scan(
+        mocker=mocker,
+        scan_class=ScanUnderTest,
+        fixture_path=Path(__file__).parent / "fixtures/test.iqy",
+    )
+
+    TestCase.maxDiff = None
+    TestCase().assertDictEqual(test_scan_event, scanner_event)