Merge pull request #900 from nautobot/release-v3.9.1

Release v3.9.1

Author: jdrew82

docs/admin/integrations/servicenow_setup.md

@@ -40,6 +40,23 @@ PLUGINS_CONFIG = {
 !!! note
     All integration settings are defined in the block above as an example. Only some will be needed as described below.
 
+## Duplicates
+
+When duplicates records are encountered in ServiceNow this is problematic for Nautobot to identify the correct record to update. The ServiceNow SSOT sync logic will warn you about these duplicate instances but it is up to the end-user to reconcile them for accurate data syncronization.
+
+At the end of an SSOT run, for every ServiceNow table where duplicates were found - a corresponding `duplicate_${table}.txt` file will be present in the results. This is in the format of a CSV file with the top row containing the attribute name and each subsequent row being the element that was found in duplicate.
+
+For example, if multiple product models were discovered you'll see a log warning and a file called `duplicate_product_model.txt` in the SSOT run output with contents such as:
+
+```
+manufacturer_name,model_name,model_number
+Cisco,Catalyst 9300,C9300-48P
+Dell,PowerEdge R740,R740-8SFF
+HP,ProLiant DL360,DL360-G10
+Juniper,EX4300,EX4300-48P
+Arista,7050X3,DCS-7050X3-32S
+```
+
 ## Upgrading from `nautobot-plugin-ssot-servicenow` App
 
 !!! warning

docs/admin/release_notes/version_3.9.md

@@ -13,7 +13,7 @@ This release brings several significant additions and changes:
 - The example Jobs now support synchronizing Tags on appropriate objects between Nautobot instances.
 - All integrations that utilize the contrib pattern will automatically support [Object Metadata](https://docs.nautobot.com/projects/core/en/stable/user-guide/platform-functionality/objectmetadata/) being added to their models.
 
-## [v3.9.0 (2025-06-20)](https://github.com/nautobot/nautobot-app-ssot/releases/tag/v3.9.0)
+## [v3.9.0 (2025-06-25)](https://github.com/nautobot/nautobot-app-ssot/releases/tag/v3.9.0)
 
 ### Added
 
@@ -50,3 +50,21 @@ This release brings several significant additions and changes:
 
 - [#856](https://github.com/nautobot/nautobot-app-ssot/issues/856) - Added a note to the developer upgrade documentation to explain the default value for text fields declared with `blank=True, null=False`.
 - [#870](https://github.com/nautobot/nautobot-app-ssot/issues/870) - Updated installation steps for vSphere integration.
+
+## [v3.9.1 (2025-07-09)](https://github.com/nautobot/nautobot-app-ssot/releases/tag/v3.9.1)
+
+### Release Overview
+
+Please note that the behavior in the SNOW integration now is to swallow and log an overview of how many duplicates encountered, and provide file output outlining what duplicates were encountered.
+
+### Changed
+
+- [#874](https://github.com/nautobot/nautobot-app-ssot/issues/874) - Reverted changes in `NautobotModel` to be backwards compatible with other integrations.
+- [#874](https://github.com/nautobot/nautobot-app-ssot/issues/874) - Reverted removal of `invalidate_cache` method in `NautobotAdapter`.
+
+### Fixed
+
+- [#844](https://github.com/nautobot/nautobot-app-ssot/issues/844) - Fixed job failure if there are duplicate devices in LibreNMS. Will skip device instead.
+- [#867](https://github.com/nautobot/nautobot-app-ssot/issues/867) - Gracefully swallow ServiceNow exceptions
+- [#867](https://github.com/nautobot/nautobot-app-ssot/issues/867) - Adds ServiceNow duplicate file reports
+- [#867](https://github.com/nautobot/nautobot-app-ssot/issues/867) - Fixes ServiceNow comparison filters to only compare against company names with Manufacturer set to True

nautobot_ssot/contrib/adapter.py

@@ -50,6 +50,14 @@ def _get_parameter_names(diffsync_model):
         """Ignore the differences between identifiers and attributes, because at this point they don't matter to us."""
         return list(diffsync_model._identifiers) + list(diffsync_model._attributes)  # pylint: disable=protected-access
 
+    def invalidate_cache(self, zero_out_hits=True):
+        """Deprecated, kept for backwards compatibility."""
+        self.job.logger.warning(
+            "Adapter class method `self.invalidate_cache()` is deprecated and will be removed in a future version. "
+            "Use `self.cache.invalidate_cache()` instead."
+        )
+        self.cache.invalidate_cache(zero_out_hits=zero_out_hits)
+
     def _load_objects(self, diffsync_model):
         """Given a diffsync model class, load a list of models from the database and return them."""
         parameter_names = self._get_parameter_names(diffsync_model)

nautobot_ssot/contrib/model.py

@@ -67,7 +67,7 @@ def _check_field(cls, name):
     def get_from_db(self):
         """Get the ORM object for this diffsync object from the database using the primary key."""
         try:
-            return self.adapter.cache.get_from_orm(self._model, {"pk": self.pk})
+            return self.adapter.get_from_orm_cache({"pk": self.pk}, self._model)
         except self._model.DoesNotExist as error:
             raise ObjectCrudException(f"No such {self._model._meta.verbose_name} instance with PK {self.pk}") from error
 
@@ -172,7 +172,7 @@ def _handle_single_field(cls, field, obj, value, relationship_fields, adapter):
 
         # Prepare handling of custom relationship many-to-many fields.
         if custom_relationship_annotation:
-            relationship = adapter.cache.get_from_orm(Relationship, {"label": custom_relationship_annotation.name})
+            relationship = adapter.get_from_orm_cache({"label": custom_relationship_annotation.name}, Relationship)
             if custom_relationship_annotation.side == RelationshipSideEnum.DESTINATION:
                 related_object_content_type = relationship.source_type
             else:
@@ -188,8 +188,8 @@ def _handle_single_field(cls, field, obj, value, relationship_fields, adapter):
                 }
             else:
                 relationship_fields["custom_relationship_many_to_many_fields"][field] = {
-                    "objects": [adapter.cache.get_from_orm(related_model_class, parameters) for parameters in value],
                     "annotation": custom_relationship_annotation,
+                    "objects": [adapter.get_from_orm_cache(parameters, related_model_class) for parameters in value],
                 }
 
             return
@@ -201,7 +201,7 @@ def _handle_single_field(cls, field, obj, value, relationship_fields, adapter):
         if django_field.many_to_many or django_field.one_to_many:
             try:
                 relationship_fields["many_to_many_fields"][field] = [
-                    adapter.cache.get_from_orm(django_field.related_model, parameters) for parameters in value
+                    adapter.get_from_orm_cache(parameters, django_field.related_model) for parameters in value
                 ]
             except django_field.related_model.DoesNotExist as error:
                 raise ObjectCrudException(
@@ -261,7 +261,7 @@ def _set_custom_relationship_to_many_fields(cls, custom_relationship_many_to_man
             annotation = dictionary.pop("annotation")
             objects = dictionary.pop("objects")
             # TODO: Deduplicate this code
-            relationship = adapter.cache.get_from_orm(Relationship, {"label": annotation.name})
+            relationship = adapter.get_from_orm_cache({"label": annotation.name}, Relationship)
             parameters = {
                 "relationship": relationship,
                 "source_type": relationship.source_type,
@@ -274,7 +274,7 @@ def _set_custom_relationship_to_many_fields(cls, custom_relationship_many_to_man
                     association_parameters = parameters.copy()
                     association_parameters["destination_id"] = object_to_relate.id
                     try:
-                        association = adapter.cache.get_from_orm(RelationshipAssociation, association_parameters)
+                        association = adapter.get_from_orm_cache(association_parameters, RelationshipAssociation)
                     except RelationshipAssociation.DoesNotExist:
                         association = RelationshipAssociation(**parameters, destination_id=object_to_relate.id)
                         association.validated_save()
@@ -285,7 +285,7 @@ def _set_custom_relationship_to_many_fields(cls, custom_relationship_many_to_man
                     association_parameters = parameters.copy()
                     association_parameters["source_id"] = object_to_relate.id
                     try:
-                        association = adapter.cache.get_from_orm(RelationshipAssociation, association_parameters)
+                        association = adapter.get_from_orm_cache(association_parameters, RelationshipAssociation)
                     except RelationshipAssociation.DoesNotExist:
                         association = RelationshipAssociation(**parameters, source_id=object_to_relate.id)
                         association.validated_save()
@@ -321,7 +321,7 @@ def _lookup_and_set_custom_relationship_foreign_keys(cls, custom_relationship_fo
             annotation = related_model_dict.pop("_annotation")
             # TODO: Deduplicate this code
             try:
-                relationship = adapter.cache.get_from_orm(Relationship, {"label": annotation.name})
+                relationship = adapter.get_from_orm_cache({"label": annotation.name}, Relationship)
             except Relationship.DoesNotExist as error:
                 raise ObjectCrudException(f"No such relationship with label '{annotation.name}'") from error
             parameters = {
@@ -333,7 +333,7 @@ def _lookup_and_set_custom_relationship_foreign_keys(cls, custom_relationship_fo
                 parameters["source_id"] = obj.id
                 related_model_class = relationship.destination_type.model_class()
                 try:
-                    destination_object = adapter.cache.get_from_orm(related_model_class, related_model_dict)
+                    destination_object = adapter.get_from_orm_cache(related_model_dict, related_model_class)
                 except related_model_class.DoesNotExist as error:
                     raise ObjectCrudException(
                         f"Couldn't resolve custom relationship {relationship.name}, no such {related_model_class._meta.verbose_name} object with parameters {related_model_dict}."
@@ -348,7 +348,7 @@ def _lookup_and_set_custom_relationship_foreign_keys(cls, custom_relationship_fo
                 )
             else:
                 parameters["destination_id"] = obj.id
-                source_object = adapter.cache.get_from_orm(relationship.source_type.model_class(), related_model_dict)
+                source_object = adapter.get_from_orm_cache(related_model_dict, relationship.source_type.model_class())
                 RelationshipAssociation.objects.update_or_create(**parameters, defaults={"source_id": source_object.id})
 
     @classmethod
@@ -377,8 +377,8 @@ def _lookup_and_set_foreign_keys(cls, foreign_keys, obj, adapter):
                         f"for generic foreign keys."
                     ) from error
                 try:
-                    related_model_content_type = adapter.cache.get_from_orm(
-                        ContentType, {"app_label": app_label, "model": model}
+                    related_model_content_type = adapter.get_from_orm_cache(
+                        {"app_label": app_label, "model": model}, ContentType
                     )
                     related_model = related_model_content_type.model_class()
                 except ContentType.DoesNotExist as error:
@@ -388,7 +388,7 @@ def _lookup_and_set_foreign_keys(cls, foreign_keys, obj, adapter):
                 setattr(obj, field_name, None)
                 continue
             try:
-                related_object = adapter.cache.get_from_orm(related_model, related_model_dict)
+                related_object = adapter.get_from_orm_cache(related_model_dict, related_model)
             except related_model.DoesNotExist as error:
                 raise ObjectCrudException(
                     f"Couldn't find '{related_model._meta.verbose_name}' instance behind '{field_name}' with: {related_model_dict}."

nautobot_ssot/integrations/librenms/diffsync/adapters/librenms.py

@@ -3,7 +3,7 @@
 import os
 
 from diffsync import DiffSync
-from diffsync.exceptions import ObjectNotFound
+from diffsync.exceptions import ObjectAlreadyExists, ObjectNotFound
 from django.contrib.contenttypes.models import ContentType
 from nautobot.dcim.models import Location, LocationType
 from nautobot.extras.models import Status
@@ -99,7 +99,10 @@ def load_device(self, device: dict):
                     ip_address=device["ip"],
                     system_of_record=os.getenv("NAUTOBOT_SSOT_LIBRENMS_SYSTEM_OF_RECORD", "LibreNMS"),
                 )
-                self.add(new_device)
+                try:
+                    self.add(new_device)
+                except ObjectAlreadyExists:
+                    self.job.logger.warning(f"Device {device[self.hostname_field]} already exists. Skipping.")
         else:
             self.job.logger.info(f'Device {device[self.hostname_field]} is "ping-only". Skipping.')
 

nautobot_ssot/integrations/servicenow/diffsync/adapter_servicenow.py

@@ -12,10 +12,12 @@
 from diffsync.exceptions import ObjectAlreadyExists, ObjectNotFound
 from jinja2 import Environment, FileSystemLoader
 
+from nautobot_ssot.contrib.model import DiffSyncModel
+
 from . import models
 
 
-class ServiceNowDiffSync(Adapter):
+class ServiceNowDiffSync(Adapter):  # pylint: disable=too-many-instance-attributes
     """DiffSync adapter using pysnow to communicate with a ServiceNow server."""
 
     # create defaultdict object to store objects that should be deleted from ServiceNow if they do not
@@ -50,6 +52,8 @@ def __init__(self, *args, client=None, job=None, sync=None, site_filter=None, **
         # create all of these interfaces in a single API call.
         self.interfaces_to_create_per_device = {}
 
+        self.duplicate_records = defaultdict(list)  # Store duplicate records for user notification
+
     def load(self):
         """Load data via pysnow."""
         self.mapping_data = self.load_yaml_datafile("mappings.yaml")
@@ -102,6 +106,9 @@ def load(self):
             else:
                 self.load_table(modelname, **entry)
 
+        for modelname, duplicate_uids in self.duplicate_records.items():
+            self.job.create_file(f"duplicate_{modelname}.txt", "\n".join(duplicate_uids))
+
     @classmethod
     def load_yaml_datafile(cls, filename, config=None):
         """Get the contents of the given YAML data file.
@@ -133,23 +140,35 @@ def load_table(self, modelname, table, mappings, **kwargs):
         """
         model_cls = getattr(self, modelname)
         self.job.logger.info(f"Loading ServiceNow table `{table}` into {modelname} instances...")
+        table_query_filter = kwargs.get("table_query", {})
 
         if "parent" not in kwargs:
             # Load the entire table
-            for record in self.client.all_table_entries(table):
+            for record in self.client.all_table_entries(table, table_query_filter):
                 self.load_record(table, record, model_cls, mappings, **kwargs)
         else:
             # Load items per parent object that we know/care about
             # This is necessary because, for example, the cmdb_ci_network_adapter table contains network interfaces
             # for ALL types of devices (servers, switches, firewalls, etc.) but we only have switches as parent objects
             for parent in self.get_all(kwargs["parent"]["modelname"]):
-                for record in self.client.all_table_entries(table, {kwargs["parent"]["column"]: parent.sys_id}):
+                table_query_filter[kwargs["parent"]["column"]] = parent.sys_id
+                for record in self.client.all_table_entries(table, table_query_filter):
                     self.load_record(table, record, model_cls, mappings, **kwargs)
+        if self.duplicate_records.get(modelname, False):
+            self.job.logger.warning(f"Found {len(self.duplicate_records[modelname])} duplicate {modelname} record(s).")
 
         self.job.logger.info(
             f"Loaded {len(self.get_all(modelname))} {modelname} records from ServiceNow table `{table}`."
         )
 
+    def log_duplicate_records(self, model_cls: DiffSyncModel) -> None:
+        """Capture duplicate records in CSV format that were found during the ServiceNow record load."""
+        model_name = model_cls._modelname  # pylint: disable=protected-access
+        model_identifiers = model_cls._identifiers  # pylint: disable=protected-access
+        if not self.duplicate_records.get(model_name, False):
+            self.duplicate_records[model_name] = [",".join(model_identifiers)]
+        self.duplicate_records[model_name].append(",".join([getattr(model_cls, attr) for attr in model_identifiers]))
+
     def load_record(self, table, record, model_cls, mappings, **kwargs):
         """Helper method to load_table()."""
         self.sys_ids.setdefault(table, {})[record["sys_id"]] = record
@@ -163,7 +182,7 @@ def load_record(self, table, record, model_cls, mappings, **kwargs):
         except ObjectAlreadyExists:
             # The baseline data in a standard ServiceNow developer instance has a number of duplicate Location entries.
             # For now, ignore the duplicate entry and continue
-            self.job.logger.warning(f'Ignoring apparent duplicate record for {modelname} "{model.get_unique_id()}".')
+            self.log_duplicate_records(model)
 
         if "parent" in kwargs:
             parent_uid = getattr(model, kwargs["parent"]["field"])
@@ -174,7 +193,10 @@ def load_record(self, table, record, model_cls, mappings, **kwargs):
                 )
             else:
                 parent_model = self.get(kwargs["parent"]["modelname"], parent_uid)
-                parent_model.add_child(model)
+                try:
+                    parent_model.add_child(model)
+                except ObjectAlreadyExists:
+                    pass
 
         return model
 

nautobot_ssot/tests/servicenow/test_adapter_servicenow.py

@@ -1,5 +1,7 @@
 """Unit tests for the ServiceNowDiffSync adapter class."""
 
+from collections import defaultdict
+
 from nautobot.core.testing import TransactionTestCase
 from nautobot.extras.models import JobResult
 
@@ -10,13 +12,18 @@
 class MockServiceNowClient:
     """Mock version of the ServiceNowClient class using canned data."""
 
+    def __init__(self):
+        self.query_params = defaultdict(list)
+
     def get_by_sys_id(self, table, sys_id):  # pylint: disable=unused-argument
         """Get a record with a given sys_id from a given table."""
         return None
 
-    def all_table_entries(self, table, query=None):
+    def all_table_entries(self, table, query={}):  # pylint: disable=dangerous-default-value
         """Iterator over all records in a given table."""
 
+        self.query_params[table].append(query)
+
         if table == "cmn_location":
             yield from [
                 {
@@ -611,3 +618,16 @@ def test_data_loading(self):
             ["hkg-leaf-01__Ethernet1", "hkg-leaf-01__Ethernet2"],
             sorted(intf.get_unique_id() for intf in snds.get_all("interface")),
         )
+
+    def test_filtering(self):
+        """Want to ensure our table query filtering is passed through correctly.
+
+        In the mappings yaml, we have a table filter for company to only grab records with manufacturer=True.
+        """
+        job = self.job_class()
+        job.job_result = JobResult.objects.create(name=job.class_path, task_name="fake task", worker="default")
+        mock_snow_client = MockServiceNowClient()
+        snds = ServiceNowDiffSync(job=job, sync=None, client=mock_snow_client)
+
+        snds.load()
+        self.assertEqual(mock_snow_client.query_params["core_company"], [{"manufacturer": True}])

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "nautobot-ssot"
-version = "3.9.0"
+version = "3.9.1"
 description = "Nautobot Single Source of Truth"
 authors = ["Network to Code, LLC <opensource@networktocode.com>"]
 license = "Apache-2.0"