Dev 1.1.2 (#22)

* #21

* update filigran-sseclient version

* change version to 1.1.2

Author: romain-filigran

README.md

@@ -18,10 +18,10 @@ The app is installed
 
 ### Installing from file
 
-1. Download latest version of the Splunk App: [TA-opencti-add-on-1.1.1.spl](https://github.com/OpenCTI-Platform/splunk-add-on/releases/download/1.1.1/TA-opencti-add-on-1.1.1.spl)
+1. Download latest version of the Splunk App: [TA-opencti-add-on-1.1.2.spl](https://github.com/OpenCTI-Platform/splunk-add-on/releases/download/1.1.2/TA-opencti-add-on-1.1.2.spl)
 2. Log in to the Splunk Web UI and navigate to "Apps" and click on "Manage Apps"
 3. Click "Install app from file"
-4. Choose file and select the "TA-opencti-add-on-1.1.1.spl" file
+4. Choose file and select the "TA-opencti-add-on-1.1.2.spl" file
 5. Click on Upload
 The app is installed
 

TA-opencti-add-on/app.manifest

@@ -5,7 +5,7 @@
     "id": {
       "group": null,
       "name": "TA-opencti-add-on",
-      "version": "1.1.1"
+      "version": "1.1.2"
     },
     "author": [
       {

TA-opencti-add-on/appserver/static/js/build/globalConfig.json

@@ -2,7 +2,7 @@
     "meta": {
         "name": "TA-opencti-add-on",
         "displayName": "OpenCTI Add-on",
-        "version": "1.1.1",
+        "version": "1.1.2",
         "restRoot": "TA_opencti_add_on",
         "_uccVersion": "5.39.0",
         "schemaVersion": "0.0.3"

TA-opencti-add-on/appserver/static/openapi.json

@@ -2,7 +2,7 @@
     "openapi": "3.0.0",
     "info": {
         "title": "TA-opencti-add-on",
-        "version": "1.1.1",
+        "version": "1.1.2",
         "description": "OpenCTI Add-on",
         "contact": {
             "name": "Filigran"

TA-opencti-add-on/bin/input_module_opencti_indicators.py

@@ -1,6 +1,7 @@
 # encoding = utf-8
 import json
 import sys
+import time
 from datetime import datetime, timezone, timedelta
 
 import six
@@ -141,7 +142,6 @@ def enrich_payload(splunk_helper, payload):
         return None
     payload["type"] = parsed_stix["type"]
     payload["value"] = parsed_stix["value"]
-    payload["value"] = parsed_stix["value"]
 
     if "extensions" in payload:
         for extension_definition in payload["extensions"].values():
@@ -160,6 +160,10 @@ def enrich_payload(splunk_helper, payload):
                         payload["_key"] = attribute_value
                     else:
                         payload[attribute_name] = attribute_value
+
+        if "detection" not in payload:
+            payload["detection"] = False
+
         # remove extensions
         del payload["extensions"]
 
@@ -324,44 +328,54 @@ def collect_events(helper, ew):
         )
 
         for msg in messages:
-            if msg.event in ["create", "update", "delete"]:
-                data = json.loads(msg.data)["data"]
-                if data['type'] == "indicator" and data['pattern_type'] == "stix":
-                    parsed_stix = enrich_payload(helper, data)
-                    if parsed_stix is None:
-                        helper.log_error(f"Unable to process indicator: {data['name']} - {data['pattern']}")
-                        continue
-                    helper.log_info("processing msg: " + msg.event + " - " + msg.id + " - " + parsed_stix['name']
-                                    + " - " + parsed_stix['pattern'])
-                    if msg.event == "create" or msg.event == "update":
-                        exist = exist_in_kvstore(kv_store, parsed_stix["_key"])
-                        if exist:
-                            kv_store.update(parsed_stix["_key"], parsed_stix)
-                        else:
+            try:
+                if msg.event in ["create", "update", "delete"]:
+                    data = json.loads(msg.data)["data"]
+                    if data['type'] == "indicator" and data['pattern_type'] == "stix":
+                        parsed_stix = enrich_payload(helper, data)
+                        if parsed_stix is None:
+                            helper.log_error(f"Unsupported indicator pattern: {data['name']} - {data['pattern']}")
+                            continue
+                        helper.log_info("processing msg: " + msg.event + " - " + msg.id + " - " + parsed_stix['name']
+                                        + " - " + parsed_stix['pattern'])
+                        if msg.event == "create" or msg.event == "update":
+                            # update code to use bach_save
                             parsed_stix['added_at'] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-                            kv_store.insert(parsed_stix)
-                    if msg.event == "delete":
-                        exist = exist_in_kvstore(kv_store, parsed_stix["_key"])
-                        if exist:
-                            kv_store.delete_by_id(parsed_stix["_key"])
-
-                if data['type'] == "marking-definition":
-                    helper.log_info("processing msg: " + msg.event + " - " + msg.id +" - "
-                                    + data['name'] + " - " + data['id'])
-                    if msg.event == "create" or msg.event == "update":
-                        if data['id'] not in MARKING_DEFs:
-                            MARKING_DEFs[data['id']] = data['name']
-
-                if data['type'] == "identity":
-                    helper.log_info("processing msg: " + msg.event + " - " + msg.id + " - "
-                                    + data['name'] + " - " + data['id'])
-                    if msg.event == "create" or msg.event == "update":
-                        if data['id'] not in IDENTITY_DEFs:
-                            IDENTITY_DEFs[data['id']] = data['name']
-
-                # update checkpoint (take 0:00:00.005544 to update)
-                state["start_from"] = msg.id
-                helper.save_check_point(input_name, json.dumps(state))
+                            kv_store.batch_save(*[parsed_stix])
+                            """
+                            exist = exist_in_kvstore(kv_store, parsed_stix["_key"])
+                            if exist:
+                                parsed_stix['added_at'] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+                                kv_store.update(parsed_stix["_key"], parsed_stix)
+                            else:
+                                parsed_stix['added_at'] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+                                kv_store.insert(parsed_stix)
+                            """
+                        if msg.event == "delete":
+                            exist = exist_in_kvstore(kv_store, parsed_stix["_key"])
+                            if exist:
+                                kv_store.delete_by_id(parsed_stix["_key"])
+
+                    if data['type'] == "marking-definition":
+                        helper.log_info("processing msg: " + msg.event + " - " + msg.id +" - "
+                                        + data['name'] + " - " + data['id'])
+                        if msg.event == "create" or msg.event == "update":
+                            if data['id'] not in MARKING_DEFs:
+                                MARKING_DEFs[data['id']] = data['name']
+
+                    if data['type'] == "identity":
+                        helper.log_info("processing msg: " + msg.event + " - " + msg.id + " - "
+                                        + data['name'] + " - " + data['id'])
+                        if msg.event == "create" or msg.event == "update":
+                            if data['id'] not in IDENTITY_DEFs:
+                                IDENTITY_DEFs[data['id']] = data['name']
+
+                    # update checkpoint (take 0:00:00.005544 to update)
+                    state["start_from"] = msg.id
+                    helper.save_check_point(input_name, json.dumps(state))
+            except Exception as ex:
+                helper.log_debug(f"Error when processing message, reason: {ex}, msg: {msg}")
+
     except Exception as ex:
         helper.log_error(f"Error in ListenStream loop, exit, reason: {ex}")
         sys.excepthook(*sys.exc_info())

TA-opencti-add-on/bin/ta_opencti_add_on/aob_py3/filigran_sseclient-1.0.1.dist-info/RECORD

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: filigran-sseclient
-Version: 1.0.1
+Name: filigran_sseclient
+Version: 1.0.2
 Summary: Python API client for OpenCTI.
 Home-page: https://github.com/FiligranHQ/filigran-sseclient
 Author: Filigran
@@ -15,24 +15,24 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests (>=2.9)
+Requires-Dist: requests>=2.9
 Requires-Dist: six
 Provides-Extra: dev
-Requires-Dist: black (~=23.1.0) ; extra == 'dev'
-Requires-Dist: build (~=0.8.0) ; extra == 'dev'
-Requires-Dist: isort (~=5.10.1) ; extra == 'dev'
-Requires-Dist: types-pytz (~=2022.2.1.0) ; extra == 'dev'
-Requires-Dist: pre-commit (~=2.20.0) ; extra == 'dev'
-Requires-Dist: pytest-cases (~=3.6.13) ; extra == 'dev'
-Requires-Dist: pytest-cov (~=3.0.0) ; extra == 'dev'
-Requires-Dist: pytest-randomly (~=3.12.0) ; extra == 'dev'
-Requires-Dist: pytest (~=7.1.2) ; extra == 'dev'
-Requires-Dist: types-python-dateutil (~=2.8.19) ; extra == 'dev'
-Requires-Dist: wheel (~=0.37.1) ; extra == 'dev'
+Requires-Dist: black~=23.1.0; extra == "dev"
+Requires-Dist: build~=0.8.0; extra == "dev"
+Requires-Dist: isort~=5.10.1; extra == "dev"
+Requires-Dist: types-pytz~=2022.2.1.0; extra == "dev"
+Requires-Dist: pre-commit~=2.20.0; extra == "dev"
+Requires-Dist: pytest-cases~=3.6.13; extra == "dev"
+Requires-Dist: pytest-cov~=3.0.0; extra == "dev"
+Requires-Dist: pytest_randomly~=3.12.0; extra == "dev"
+Requires-Dist: pytest~=7.1.2; extra == "dev"
+Requires-Dist: types-python-dateutil~=2.8.19; extra == "dev"
+Requires-Dist: wheel~=0.37.1; extra == "dev"
 Provides-Extra: doc
-Requires-Dist: autoapi (~=2.0.1) ; extra == 'doc'
-Requires-Dist: sphinx-autodoc-typehints (~=1.19.2) ; extra == 'doc'
-Requires-Dist: sphinx-rtd-theme (~=1.0.0) ; extra == 'doc'
+Requires-Dist: autoapi~=2.0.1; extra == "doc"
+Requires-Dist: sphinx-autodoc-typehints~=1.19.2; extra == "doc"
+Requires-Dist: sphinx-rtd-theme~=1.0.0; extra == "doc"
 
 # Filigran Python SSE Client
 

TA-opencti-add-on/bin/ta_opencti_add_on/aob_py3/filigran_sseclient-1.0.1.dist-info/INSTALLER renamed to TA-opencti-add-on/bin/ta_opencti_add_on/aob_py3/filigran_sseclient-1.0.2.dist-info/INSTALLER

@@ -0,0 +1,11 @@
+filigran_sseclient-1.0.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+filigran_sseclient-1.0.2.dist-info/LICENSE,sha256=JG5WIzhHLJEXmX111sTkQJuE9v2uaeXJYJTZrr4yC04,1088
+filigran_sseclient-1.0.2.dist-info/METADATA,sha256=thig25fwpl0q-_1dOHx5gaDsyO_cFDuWZbw-NdijLRo,1855
+filigran_sseclient-1.0.2.dist-info/RECORD,,
+filigran_sseclient-1.0.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+filigran_sseclient-1.0.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+filigran_sseclient-1.0.2.dist-info/top_level.txt,sha256=EwGXWyTELjIDm1mFTyFaYbegkrIeIX2aFdj4_hrdOWU,19
+filigran_sseclient/__init__.py,sha256=q8VnSpp7lBMfkUxCzf6hsuMSfqNM365xFgwC-xdR2m4,120
+filigran_sseclient/__pycache__/__init__.cpython-312.pyc,,
+filigran_sseclient/__pycache__/sseclient.cpython-312.pyc,,
+filigran_sseclient/sseclient.py,sha256=Mlp6p4N5wP5T6eIAlfINw_JzDW1J42HzltflQ5lQh-U,7886

TA-opencti-add-on/bin/ta_opencti_add_on/aob_py3/filigran_sseclient-1.0.1.dist-info/LICENSE renamed to TA-opencti-add-on/bin/ta_opencti_add_on/aob_py3/filigran_sseclient-1.0.2.dist-info/LICENSE

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.38.4)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

TA-opencti-add-on/bin/ta_opencti_add_on/aob_py3/filigran_sseclient-1.0.1.dist-info/METADATA renamed to TA-opencti-add-on/bin/ta_opencti_add_on/aob_py3/filigran_sseclient-1.0.2.dist-info/METADATA

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = "1.0.1"
+__version__ = "1.0.2"
 
 from .sseclient import SSEClient
 

TA-opencti-add-on/bin/ta_opencti_add_on/aob_py3/filigran_sseclient-1.0.2.dist-info/RECORD

@@ -20,6 +20,19 @@
 # however, assumes that a system will provide consistent line endings.
 end_of_field = re.compile(r"\r\n\r\n|\r\r|\n\n")
 
+# We only supports this newlines char to split message. Formatting could be present in data.
+# b"\xc2\x85", # NEL
+# b"\xe2\x80\xa8", # LineSeparator
+# b"\xe2\x80\xa9", # ParagraphSeparator
+# b"\v", # Line Tabulation
+# b"\f", # Form Feed
+# b"\x1c", File Separator
+# b"\x1d", Group Separator
+# b"\x1e", Record Separator
+# see https://docs.python.org/3.11/library/stdtypes.html#str.splitlines
+NEWLINE_MESSAGE_CHARS = b'\r\n', b'\n', b'\r',
+DECODED_NEWLINE_MESSAGE_CHARS = [nc.decode('utf-8') for nc in NEWLINE_MESSAGE_CHARS]
+SPLIT_PATTERN = re.compile('|'.join(map(re.escape, DECODED_NEWLINE_MESSAGE_CHARS)))
 
 class SSEClient(object):
     def __init__(
@@ -164,14 +177,23 @@ def dump(self):
         lines.extend("data: %s" % d for d in self.data.split("\n"))
         return "\n".join(lines) + "\n\n"
 
+    @staticmethod
+    def splitlines(raw):
+        """
+        Yield each line from the input string, split by the precompiled newline pattern.
+        """
+        for line in SPLIT_PATTERN.split(raw):
+            if line:  # Skip any empty results
+                yield line
+
     @classmethod
     def parse(cls, raw):
         """
         Given a possibly-multiline string representing an SSE message, parse it
         and return a Event object.
         """
         msg = cls()
-        for line in raw.splitlines():
+        for line in cls.splitlines(raw):
             m = cls.sse_line_pattern.match(line)
             if m is None:
                 # Malformed line.  Discard but warn.

TA-opencti-add-on/bin/ta_opencti_add_on/aob_py3/filigran_sseclient-1.0.2.dist-info/REQUESTED

@@ -7,7 +7,7 @@ build = 1
 
 [launcher]
 author = Filigran
-version = 1.1.1
+version = 1.1.2
 description = Add-on for OpenCTI
 
 [ui]