Merge branch 'master' into depr-coremods

Author: Cisphyx

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.203.0
+current_version = 2.205.0
 commit = True
 tag = True
 tag_message = 

.circleci/config.yml

@@ -103,10 +103,12 @@ commands:
           key: v5-docvenv-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}-{{ checksum "pyproject.toml" }}-{{ checksum "/tmp/python.version" }}
 
       - run:
-          name: executing docs jupyter notebooks
+          name: executing docs test / build
           command: |
             . venv/bin/activate
             ./scripts/doctests.py
+            cd docs
+            make html
 
   test_steps_python:
     description: "Python test steps"

CHANGELOG.rst

@@ -6,6 +6,55 @@
 Synapse Changelog
 *****************
 
+v2.205.0 - 2025-03-28
+=====================
+
+Model Changes
+-------------
+- Added a ``uses`` light edge between ``it:prod:soft`` and ``risk:vuln`` forms.
+  (`#4198 <https://github.com/vertexproject/synapse/pull/4198>`_)
+- Added a ``targets`` light edge between ``risk:compromise`` and
+  ``ou:industry`` forms.
+  (`#4198 <https://github.com/vertexproject/synapse/pull/4198>`_)
+- See :ref:`userguide_model_v2_205_0` for more detailed model changes.
+
+v2.204.1 - 2025-03-25
+=====================
+
+Bugfixes
+--------
+- Fixed a regression in the Storm ``parallel`` command where variables
+  containing certain heavy Storm object types were not passed into the parallel
+  runtimes.
+  (`#4197 <https://github.com/vertexproject/synapse/pull/4197>`_)
+
+v2.204.0 - 2025-03-21
+=====================
+
+Model Changes
+-------------
+- See :ref:`userguide_model_v2_204_0` for more detailed model changes.
+
+Bugfixes
+--------
+- Fixed an issue where locked users could still access HTTP endpoints with an
+  existing session cookie. User lock status now invalidates existing sessions
+  across all authenticated endpoints.
+  (`#4180 <https://github.com/vertexproject/synapse/pull/4180>`_)
+- Fixed an issue in Storm where the ``(`` and ``$`` control characters were
+  allowed in unquoted strings.
+  (`#4187 <https://github.com/vertexproject/synapse/pull/4187>`_)
+- Fixed a regression where the Storm ``not`` operator was incorrectly
+  whitespace sensitive.
+  (`#4187 <https://github.com/vertexproject/synapse/pull/4187>`_)
+- Fixed an issue with URL sanitizing where incorrect data was being removed
+  from the URL string.
+  (`#4190 <https://github.com/vertexproject/synapse/pull/4190>`_)
+- Fixed an issue with the Storm ``parallel`` command where variables
+  initialized within a parallel runtime were not properly isolated to that
+  specific runtime.
+  (`#4194 <https://github.com/vertexproject/synapse/pull/4194>`_)
+
 v2.203.0 - 2025-03-14
 =====================
 

changes/modelrefs/model_2.204.0_6b616fcd2d70a036d9853355fcb6fd491e84f015.yaml.gz

@@ -13,12 +13,28 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
+import re
 import sys
 import datetime
 sys.path.insert(0, os.path.abspath('..'))
 
 import synapse
 
+# -- Warning Filter Configuration --------------------------------------------
+
+# List of warning patterns to ignore
+WARNINGS_IGNORE = [
+    r'Detected \d+ deprecated properties unlocked and not in use',
+    r'Sysctl values different than expected',
+    r'The form edge:refs is deprecated or using a deprecated type',
+    r'The form edge:has is deprecated or using a deprecated type',
+    r'The property media:news:author is deprecated or using a deprecated type', # storm_ref_automation Cron Example
+]
+
+# List of warning patterns to convert to errors
+WARNINGS_ERROR = [
+    r'.* is deprecated or using a deprecated type',
+]
 
 # -- Project information -----------------------------------------------------
 
@@ -265,6 +281,20 @@ def convert_rstorm(_):
     synpd = os.path.split(synbd)[0]  # split off the synapse module directory
     env = {**os.environ, 'SYN_LOG_LEVEL': 'DEBUG'}
 
+    def check_output_for_warnings(output):
+        for line in output.splitlines():
+            if '[WARNING]' in line:
+                msg = line.split('[WARNING]', 1)[1].strip()
+                for pattern in WARNINGS_IGNORE:
+                    if re.search(pattern, msg):
+                        break
+                else:
+                    for pattern in WARNINGS_ERROR:
+                        if re.search(pattern, msg):
+                            raise RuntimeError(f"Warning converted to error: {msg}")
+                        else: # Unhandled warnings
+                            raise RuntimeError(f"Unhandled warning found: {msg}")
+
     cwd = os.getcwd()
     for fdir, dirs, fns in os.walk(cwd):
         for fn in fns:
@@ -278,8 +308,18 @@ def convert_rstorm(_):
                 tick = s_common.now()
 
                 args = ['python', '-m', 'synapse.tools.rstorm', '--save', ofile, sfile]
-                r = subprocess.run(args, cwd=synpd, env=env)
-                assert r.returncode == 0, f'Failed to convert {sfile}'
+                result = subprocess.run(args, cwd=synpd, env=env, capture_output=True, text=True)
+
+                if result.stdout:
+                    print(result.stdout, end='')
+                    check_output_for_warnings(result.stdout)
+
+                if result.stderr:
+                    print(result.stderr, end='')
+                    check_output_for_warnings(result.stderr)
+
+                if result.returncode != 0:
+                    raise RuntimeError(f'Failed to convert {sfile}: {result.stderr}')
 
                 tock = s_common.now()
                 took = (tock - tick) / 1000

changes/modelrefs/model_2.205.0_d985f8b93669529b2d90895baabb1dc289e287df.yaml.gz

@@ -0,0 +1,20 @@
+
+
+.. _userguide_model_v2_204_0:
+
+######################
+v2.204.0 Model Updates
+######################
+
+The following model updates were made during the ``v2.204.0`` Synapse release.
+
+**************
+New Properties
+**************
+
+``ps:contact``
+  The form had the following property added to it:
+
+  ``id``
+    A type or source specific unique ID for the contact.
+

docs/conf.py

@@ -0,0 +1,69 @@
+
+
+.. _userguide_model_v2_205_0:
+
+######################
+v2.205.0 Model Updates
+######################
+
+The following model updates were made during the ``v2.205.0`` Synapse release.
+
+*********
+New Forms
+*********
+
+``tel:phone:type:taxonomy``
+  A taxonomy of phone number types.
+
+
+
+**************
+New Properties
+**************
+
+``econ:acct:balance``
+  The form had the following property added to it:
+
+  ``instrument``
+    The financial instrument holding the balance.
+
+
+``tel:phone``
+  The form had the following property added to it:
+
+  ``type``
+    The type of phone number.
+
+
+
+***********
+Light Edges
+***********
+
+``targets``
+    When used with a ``risk:compromise`` and an ``ou:industry`` node, the edge
+    indicates the compromise was assessed to be based on the victim's role in
+    the industry.
+
+
+``uses``
+    When used with an ``it:prod:soft`` and a ``risk:vuln`` node, the edge
+    indicates the software uses the vulnerability.
+
+
+
+*********************
+Deprecated Properties
+*********************
+
+``econ:acct:balance``
+  The form had the following properties deprecated:
+
+
+  ``crypto:address``
+    Deprecated. Please use ``:instrument``.
+
+
+  ``pay:card``
+    Deprecated. Please use ``:instrument``.
+

docs/synapse/userguides/model_updates/update_v2_204_0.rst

@@ -342,16 +342,16 @@ You can assign an explicit, unchanging value to a variable.
 
 *Example:*
 
-- Tag ``file:bytes`` nodes that have a number of AV signature hits higher than a given threshold for review:
+- Tag ``file:bytes`` nodes that have a number of AV scanner results higher than a given threshold for review:
 
-.. storm-pre:: [file:bytes=sha256:0000746c55336cd8d34885545f9347d96607d0391fbd3e76dae7f2b3447775b4 it:av:filehit=(sha256:0000746c55336cd8d34885545f9347d96607d0391fbd3e76dae7f2b3447775b4, (0bfef0179bf358f3fe7bad67fa529c77, trojan.gen.2)) it:av:filehit=(sha256:0000746c55336cd8d34885545f9347d96607d0391fbd3e76dae7f2b3447775b4, (325cd5a01724fa0c63907eac044f4961, trojan.agent/gen-onlinegames)) it:av:filehit=(sha256:0000746c55336cd8d34885545f9347d96607d0391fbd3e76dae7f2b3447775b4, (ac8d9645c6cdf123683a73a02e231052, w32/imestartup.a.gen!eldorado))]
-.. storm-pre:: [file:bytes=sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f it:av:filehit=(sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f, (be9793d772d23269ab0c165af819e74a, troj_gen.r002c0gkj17)) it:av:filehit=(sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f, (eef2ccb70945fb28a45c7f14f2a0f11d, malicious.1b8fb7)) it:av:filehit=(sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f, (ce4e34d2f9207095aa7351986bbad357, trojan-ddos.win32.stormattack.c)) it:av:filehit=(sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f, (ed344310e3203ec4348c4ee549a3b188, "trojan ( 00073eb11 )")) it:av:filehit=(sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f, (f5b5daeda10e487fccc07463d9df6b47, tool.stormattack.win32.10)) it:av:filehit=(sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f, (a0f25a5ba637d5c8e7c42911c4336085, trojan/w32.agent.61440.eii))]
-.. storm-cli:: $threshold=5 file:bytes +{ -> it:av:filehit } >= $threshold [ +#review ]
+.. storm-pre:: [file:bytes=sha256:0000746c55336cd8d34885545f9347d96607d0391fbd3e76dae7f2b3447775b4 (it:av:scan:result=(sha256:0000746c55336cd8d34885545f9347d96607d0391fbd3e76dae7f2b3447775b4, trojan.gen.2) :signame="trojan.gen.2" :target:file="sha256:0000746c55336cd8d34885545f9347d96607d0391fbd3e76dae7f2b3447775b4") (it:av:scan:result=(sha256:0000746c55336cd8d34885545f9347d96607d0391fbd3e76dae7f2b3447775b4, trojan.agent/gen-onlinegames) :signame="trojan.agent/gen-onlinegames" :target:file="sha256:0000746c55336cd8d34885545f9347d96607d0391fbd3e76dae7f2b3447775b4") (it:av:scan:result=(sha256:0000746c55336cd8d34885545f9347d96607d0391fbd3e76dae7f2b3447775b4, w32/imestartup.a.gen!eldorado) :signame="w32/imestartup.a.gen!eldorado" :target:file="sha256:0000746c55336cd8d34885545f9347d96607d0391fbd3e76dae7f2b3447775b4")]
+.. storm-pre:: [file:bytes=sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f (it:av:scan:result=(sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f, troj_gen.r002c0gkj17) :signame="" :target:file="sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f") (it:av:scan:result=(sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f, malicious.1b8fb7) :signame="malicious.1b8fb7" :target:file="sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f") (it:av:scan:result=(sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f, trojan-ddos.win32.stormattack.c) :signame="trojan-ddos.win32.stormattack.c" :target:file="sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f") (it:av:scan:result=(sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f, "trojan ( 00073eb11 )") :signame="trojan ( 00073eb11 )" :target:file="sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f") (it:av:scan:result=(sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f, tool.stormattack.win32.10) :signame="tool.stormattack.win32.10" :target:file="sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f") (it:av:scan:result=(sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f, trojan/w32.agent.61440.eii) :signame="trojan/w32.agent.61440.eii" :target:file="sha256:00007694135237ec8dc5234007043814608f239befdfc8a61b992e4d09e0cf3f")]
+.. storm-cli:: $threshold=5 file:bytes +{ -> it:av:scan:result } >= $threshold [ +#review ]
 
 .. TIP::
 
-  The example above uses a subquery filter (:ref:`filter-subquery`) to pivot to the ``it:av:filehit`` nodes
-  associated with the ``file:bytes`` node, and compares the number of AV hits to the value of the ``$threshold``
+  The example above uses a subquery filter (:ref:`filter-subquery`) to pivot to the ``it:av:scan:result`` nodes
+  associated with the ``file:bytes`` node, and compares the number of AV results to the value of the ``$threshold``
   variable.
 
 

docs/synapse/userguides/model_updates/update_v2_205_0.rst

@@ -409,16 +409,16 @@ a more "human friendly" method.
 
 *Use a subquery to assign an organization's (ou:org) guid as the secondary property of a ps:contact node:*
 
-.. storm-pre:: [ ou:org=0fa690c06970d2d2ae74e43a18f46c2a :alias=usgovdoj :url=https://www.justice.gov/ :name="U.S. Department of Justice" ]
+.. storm-pre:: [ ou:org=0fa690c06970d2d2ae74e43a18f46c2a :names=(usgovdoj,) :url=https://www.justice.gov/ :name="U.S. Department of Justice" ]
 .. storm-pre:: [ ps:contact=d41d8cd98f00b204e9800998ecf8427e :orgname="U.S. Department of Justice" :address="950 Pennsylvania Avenue NW, Washington, DC, 20530-0001" :phone="+1 202-514-2000" :loc="us.dc.washington" ]
-.. storm-cli:: ps:contact:orgname="U.S. Department of Justice" [ :org={ ou:org:alias=usgovdoj } ]
+.. storm-cli:: ps:contact:orgname="U.S. Department of Justice" [ :org={ ou:org:names*[=usgovdoj] } ]
 
-In the example above, the subquery ``ou:org:alias=usgovdoj`` is used to lift the organization node with that ``:alias``
+In the example above, the subquery ``ou:org:names*[=usgovdoj]`` is used to lift the organization node with that ``:names``
 property value and assign the ``ou:org`` node's guid value to the ``:org`` property of the ``ps:contact`` node.
 
 *Use a subquery to assign one or more industries (ou:industry) to an organization (ou:org):*
 
-.. storm-pre:: [ ou:org=2848b564bf1e68563e3fea4ce27299f3 :alias=apple :name=apple :names=(apple, "apple, inc.") :phone="+1 408-996-1010" :loc=us.ca.cupertino]
+.. storm-pre:: [ ou:org=2848b564bf1e68563e3fea4ce27299f3 :name=apple :names=(apple, "apple, inc.") :phone="+1 408-996-1010" :loc=us.ca.cupertino]
 .. storm-pre:: [ps:contact="*" :orgname="Apple" :address="1 Apple Park Way, Cupertino, CA 95014" :phone="+1 202-514-2000" :loc="us.ca.cupertino"]
 .. storm-pre:: [ ou:industry="*" :name="Computers and Electronics" ]
 .. storm-pre:: [ ou:industry="*" :name="Telecommunications" ]

docs/synapse/userguides/storm_adv_vars.rstorm

@@ -643,12 +643,12 @@ The extended comparator ``~=`` is used to filter nodes based on PCRE-compatible
 
 *Filter the current working set to exclude organizations (ou:org nodes) whose name contains a string that starts with "v", followed by 0 or more characters, followed by "x":*
 
-.. storm-pre:: [ (ou:org=4b0c2c5671874922ce001d69215d032f :name="the vertex project" :alias=vertex) (ou:org=ad8de4b5da0fccb2caadb0d425e35847 :name=vxunderground) ] -ou:org:name ~= '^v.*x'
+.. storm-pre:: [ (ou:org=4b0c2c5671874922ce001d69215d032f :name="the vertex project" :names=(vertex,)) (ou:org=ad8de4b5da0fccb2caadb0d425e35847 :name=vxunderground) ] -ou:org:name ~= '^v.*x'
 ::
 
   <query> -ou:org:name ~= '^v.*x'
 
-.. storm-pre:: [ (ou:org=4b0c2c5671874922ce001d69215d032f :name="the vertex project" :alias=vertex) (ou:org=ad8de4b5da0fccb2caadb0d425e35847 :name=vxunderground) ] -:name ~= '^v.*x'
+.. storm-pre:: [ (ou:org=4b0c2c5671874922ce001d69215d032f :name="the vertex project" :names=(vertex,)) (ou:org=ad8de4b5da0fccb2caadb0d425e35847 :name=vxunderground) ] -:name ~= '^v.*x'
 ::
 
   <query> -:name ~= '^v.*x'

docs/synapse/userguides/storm_ref_data_mod.rstorm

@@ -390,7 +390,7 @@ Lift by Primary Property Value
 
 *Lift the organization node whose primary property is the specified guid (globally unique identifier):*
 
-.. storm-pre:: [ ou:org=4b0c2c5671874922ce001d69215d032f :name="the vertex project" :alias=vertex ]
+.. storm-pre:: [ ou:org=4b0c2c5671874922ce001d69215d032f :name="the vertex project" :names=(vertex,) ]
 .. storm-pre:: ou:org = 4b0c2c5671874922ce001d69215d032f
 ::
 
@@ -416,13 +416,13 @@ Lift by Secondary Property Value
 
 **Examples:**
 
-*Lift the organization node with the alias "vertex":*
+*Lift the organization node with "vertex" in the names property:*
 
-.. storm-pre:: [ ou:org=4b0c2c5671874922ce001d69215d032f :name="the vertex project" :alias=vertex ]
-.. storm-pre:: ou:org:alias = vertex
+.. storm-pre:: [ ou:org=4b0c2c5671874922ce001d69215d032f :name="the vertex project" :names=(vertex,) ]
+.. storm-pre:: ou:org:names *[ = vertex ]
 ::
 
-  ou:org:alias = vertex
+  ou:org:names *[ = vertex ]
 
 *Lift all DNS A records for the FQDN "hugesoft.org":*
 
@@ -671,7 +671,7 @@ The extended comparator ``~=`` is used to lift nodes based on PCRE-compatible re
 
 *Lift all organizations whose name contains a string that starts with "v", followed by 0 or more characters, followed by "x":*
 
-.. storm-pre:: [ (ou:org=4b0c2c5671874922ce001d69215d032f :name="the vertex project" :alias=vertex) (ou:org=ad8de4b5da0fccb2caadb0d425e35847 :name=vxunderground) ]
+.. storm-pre:: [ (ou:org=4b0c2c5671874922ce001d69215d032f :name="the vertex project" :names=(vertex,)) (ou:org=ad8de4b5da0fccb2caadb0d425e35847 :name=vxunderground) ]
 .. storm-pre:: ou:org:name ~= '^v.*x'
 ::
 

docs/synapse/userguides/storm_ref_filter.rstorm

@@ -1679,7 +1679,7 @@ or with a wildcard to represent any / all edges (e.g., ``-(*)+>`` ).
 *Join a threat cluster (risk:threat node) with any nodes used or targeted by the cluster:*
 
 .. storm-pre:: [ ( risk:threat=* :org:name='sparkling unicorn' ) (risk:vuln=* :cve=cve-2012-0158) ]
-.. storm-pre:: risk:threat:org:name='sparkling unicorn' [ +(uses)> { risk:vuln:cve=cve-2012-0158 } ] [ +(targets)> { ou:org:alias=vertex } ]
+.. storm-pre:: risk:threat:org:name='sparkling unicorn' [ +(uses)> { risk:vuln:cve=cve-2012-0158 } ] [ +(targets)> { ou:org:names=(vertex,) } ]
 .. storm-pre:: risk:threat -( (uses, targets) )+> *
 
 ::

docs/synapse/userguides/storm_ref_lift.rstorm

@@ -769,10 +769,10 @@ is often easier to lift guid nodes by a unique secondary property.
 
 **Examples:**
 
-Lift an org node by its alias:
+Lift an org node by a single name in the names property:
 
-.. storm-pre:: [ ou:org = * :alias = choam :name = 'combine honnete ober advancer mercantiles' ]
-.. storm-cli:: ou:org:alias = choam
+.. storm-pre:: [ ou:org = * :names = (choam,) :name = 'combine honnete ober advancer mercantiles' ]
+.. storm-cli:: ou:org:names *[ = choam ]
 
 Lift a DNS request node by the name used in the DNS query:
 
@@ -1490,7 +1490,7 @@ Synapse will automatically normalize the value by:
 
 **Examples:**
 
-.. storm-cli:: [ syn:tag = rep.us-cisa.LAPSUS$ ]
+.. storm-cli:: [ syn:tag = "rep.us-cisa.LAPSUS$" ]
 
 In the above example:
 

docs/synapse/userguides/storm_ref_pivot.rstorm

@@ -4,17 +4,16 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.203.0'
+version = '2.205.0'
 authors = [
     { name = 'The Vertex Project LLC', email = '[email protected]'},
 ]
 description = 'Synapse Intelligence Analysis Framework'
 readme = 'README.rst'
 requires-python = '>=3.11'
-license = { text = 'Apache License 2.0' }
+license = 'Apache-2.0'
 classifiers = [
     'Development Status :: 5 - Production/Stable',
-    'License :: OSI Approved :: Apache Software License',
     'Topic :: Database :: Database Engines/Servers',
     'Topic :: System :: Clustering',
     'Topic :: System :: Distributed Computing',