diff --git a/AGENTS.md b/AGENTS.md index 6e8d9377..9171d3d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,7 +37,7 @@ Defer all version pins to `pyproject.toml` and `netbox_custom_objects/__init__.p │ ├── tables.py — django-tables2 tables for list views. │ ├── template_content.py — PluginTemplateExtension registrations. │ ├── urls.py — Web UI URL routing (80+ routes). -│ ├── utilities.py — AppsProxy, generate_model(), get_viewname(), is_in_branch(). +│ ├── utilities.py — AppsProxy, generate_model(), get_viewname(). │ ├── views.py — All UI views. │ ├── api/ │ │ ├── serializers.py — get_serializer_class() + static serializers. @@ -134,7 +134,7 @@ Multi-object fields create a separate through table (`custom_objects__ [!NOTE] -> If you are using NetBox Custom Objects with NetBox Branching, you need to insert the following into your `configuration.py`. See the docs for a full description of how NetBox Custom Objects currently works with NetBox Branching. - -``` -PLUGINS_CONFIG = { - 'netbox_branching': { - 'exempt_models': [ - 'netbox_custom_objects.customobjecttype', - 'netbox_custom_objects.customobjecttypefield', - ], - }, -} -``` - ## Known Limitations NetBox Custom Objects is now Generally Available which means you can use it in production and migrations to future versions will work. There are many upcoming features including GraphQL support - the best place to see what's on the way is the [issues](https://github.com/netboxlabs/netbox-custom-objects/issues) list on the GitHub repository. diff --git a/docs/branching.md b/docs/branching.md index eb8d51f5..a260abc0 100644 --- a/docs/branching.md +++ b/docs/branching.md @@ -1,5 +1,12 @@ # Using NetBox Custom Objects with NetBox Branching +When using Custom Objects together with NetBox Branching, the following minimum versions are required: + +- NetBox >= 4.6.2 +- netbox-branching >= 1.0.4 + +These requirements are only enforced when `netbox_branching` is present in `PLUGINS`. If you do not use branching, the standard compatibility matrix in `COMPATIBILITY.md` applies. A Django system check (`netbox_custom_objects.E001` / `E002`) will fail at startup if the combination is misconfigured. + As of version 0.4.0, Custom Objects is _compatible_ with [NetBox Branching](https://netboxlabs.com/docs/extensions/branching/), but not yet fully supported. Users can safely run both plugins together, but there are some caveats to be aware of. See below for how each Custom Objects model interacts with NetBox Branching. !!! note diff --git a/docs/field-attributes.md b/docs/field-attributes.md index 2050780b..075e0d3f 100644 --- a/docs/field-attributes.md +++ b/docs/field-attributes.md @@ -19,6 +19,7 @@ The following attributes are available when creating or editing a Custom Object | `multiselect` | Multiple selections from a choice set | | `object` | Reference to a single object (built-in NetBox object or Custom Object) | | `multiobject` | Reference to multiple objects of the same type | +| `coordinates` | A geographic latitude/longitude pair (with a map link) | ## Common Attributes @@ -85,3 +86,24 @@ Field types: `object`, `multiobject` !!! note To reference another Custom Object Type, choose `Custom Objects > ` in the **Related object type** dropdown. To create a polymorphic field that may reference objects of multiple types, enable **Polymorphic** and select the allowed types under **Related object types**. + +## Coordinates Fields + +Field type: `coordinates` + +A single `coordinates` field stores a geographic latitude/longitude pair, mirroring NetBox's +native Site/Device coordinates (plain decimal columns — PostGIS is not required). Adding one +`coordinates` field named `location` creates two backing columns and two form inputs: + +- `location_latitude` — decimal, range −90 to 90, up to 6 decimal places +- `location_longitude` — decimal, range −180 to 180, up to 6 decimal places + +Behaviour: + +- **Both-or-neither.** Latitude and longitude must either both be set or both be empty; setting + only one is rejected in forms and via the REST API. +- **REST API.** The pair is exposed as two flat fields, `_latitude` and `_longitude` + (matching NetBox core's serializers), not as a nested object. +- **Map link.** Detail views render the coordinates with a **Map** button that opens the location + using NetBox's `MAPS_URL` configuration parameter (Google Maps by default). +- `Must be unique` and `Default` are not supported for `coordinates` fields. diff --git a/docs/graphql.md b/docs/graphql.md new file mode 100644 index 00000000..84139cec --- /dev/null +++ b/docs/graphql.md @@ -0,0 +1,173 @@ +# GraphQL API + +The NetBox Custom Objects plugin exposes custom objects through NetBox's GraphQL +API at the standard `/graphql/` endpoint, alongside NetBox's built-in models. For +each Custom Object Type you have defined, two root query fields are generated: + +- `` — fetch a single custom object by `id`. +- `_list` — fetch a list of custom objects (paginated). + +The `` is `custom_objects_`, where `` is the Custom Object +Type's **slug** with any characters that are not valid in a GraphQL name replaced +by underscores (for example a type with slug `dhcp-scope` becomes +`custom_objects_dhcp_scope` and `custom_objects_dhcp_scope_list`). + +The `custom_objects_` prefix namespaces these fields so they can never collide +with NetBox's own (or another plugin's) root query fields — every plugin's +GraphQL query is merged into NetBox's single global `Query`, so a bare, +slug-derived name like `site` or `group` would otherwise be shadowed by a core +model's field. + +## Authentication + +GraphQL requests use the same authentication as the REST API. Pass a token in the +`Authorization` header: + +``` +Authorization: Token +``` + +Object-level permissions are enforced. The top-level `` / `_list` +queries only return custom objects the authenticated user has permission to view, +and objects reached through a relationship field are likewise filtered to those +the user may view. + +## Using the GraphiQL explorer + +NetBox's built-in GraphiQL explorer (at `/graphql/`) loads the GraphQL schema +**once when the page opens** and caches it for the life of the page. Because +custom object types and their fields can change at runtime, the explorer's +autocomplete suggestions and **Documentation** panel can go stale. + +If you **add, remove, or change a custom object type's fields**, or **add a new +custom object type**, while GraphiQL is open, those changes will not appear in the +field autocomplete or the Docs panel until GraphiQL re-reads the schema. (The +change is already live on the server — a query using the new field works +immediately — it is only GraphiQL's cached copy of the schema that lags.) + +To refresh it, click the **Re-fetch GraphQL schema** button — the circular-arrow +icon in the GraphiQL toolbar (lower left). This re-reads the schema *without* +reloading the page, so your current query is preserved: + +![The Re-fetch GraphQL schema button in the GraphiQL toolbar](media/graphql/graphiql-refresh-schema.png) + +Reloading the whole browser page has the same effect, but discards whatever you +have typed in the editor. + +!!! note + This only affects the interactive explorer. Programmatic API clients fetch the + schema via introspection whenever they choose, and always query against the + current server-side schema. + +## Querying a list + +The fields you can select depend on how the Custom Object Type is defined. Every +type exposes `id` and `display`; the examples below also assume the type defines +custom fields named `name` and `description`. + +```graphql +query { + custom_objects_dhcp_scope_list { + id + display + name + description + } +} +``` + +> **Tip:** `display` is the only human-readable field guaranteed to exist on every +> type — it is the value of the type's primary field. `name` exists only if the +> type defines a custom field literally named `name`. + +## Querying a single object + +```graphql +query { + custom_objects_dhcp_scope(id: 42) { + id + display + } +} +``` + +## Available fields + +Each generated type exposes: + +| Field | Description | +|-------|-------------| +| `id` | The object's primary key. | +| `display` | The object's display string (its primary field value). | +| `created` / `last_updated` | Change-logging timestamps. | +| `tags` | The object's tags. | +| One field per custom field | Named exactly as the field's `name`. | + +### Scalar field types + +Scalar custom fields map to their natural GraphQL types: + +| Custom field type | GraphQL type | +|-------------------|--------------| +| text, long text, URL, select | `String` | +| integer | `Int` | +| decimal | `Decimal` | +| boolean | `Boolean` | +| date | `Date` | +| datetime | `DateTime` | +| JSON | `JSON` | +| multi-select | `[String]` | + +### Relationship field types + +Object and multi-object fields resolve to the **native GraphQL type** of their +target, so a related object is fully traversable exactly as it would be when +queried directly. A field pointing at a Site resolves to NetBox's `SiteType`: + +```graphql +query { + custom_objects_server_list { + name + primary_site { # an object (single) field → SiteType + id + name + region { name } # traverse into the related object's own fields + } + interfaces { # a multi-object (list) field → [InterfaceType] + id + name + } + } +} +``` + +#### Polymorphic fields + +A polymorphic relationship field may point at several different model types. It +resolves to a GraphQL **union** of those types; select fields per type with inline +fragments: + +```graphql +query { + custom_objects_binding_list { + name + target { # polymorphic object field + ... on SiteType { id name } + ... on DeviceType { id name } + } + } +} +``` + +#### Fallback for targets without a GraphQL type + +A small number of NetBox models are not exposed in GraphQL. When a relationship +field's target has no native GraphQL type, that field falls back to a uniform +`CustomObjectRelatedObjectType` so it is never silently dropped: + +| Field | Description | +|-------|-------------| +| `id` | The referenced object's primary key. | +| `object_type` | The referenced object's type, as `.`. | +| `display` | The referenced object's display string. | +| `url` | The referenced object's absolute URL, if resolvable. | diff --git a/docs/index.md b/docs/index.md index febc7945..eb175fac 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,6 +41,7 @@ This documentation covers: * Full-text search * Change logging * Bookmarks + * Contacts * Custom Links * Cloning * Import/Export @@ -126,6 +127,39 @@ As with core NetBox objects, Custom Objects have their own list views. To see al You will see a standard NetBox list view for your new Custom Objects with the standard options including `Configure Table`, `+ Add`, `Import`, `Export`, and others. +### Config Context + +A Custom Object Type can opt in to NetBox's [config context](https://netboxlabs.com/docs/netbox/models/extras/configcontext/) support by checking **Config context support** when the type is created. (This can only be set at creation time — it adds a `local_context_data` column to the type's table, so it cannot be toggled afterwards.) + +When enabled, every object of that type gains: + +- A **Local Context Data** field (arbitrary JSON), stored per object and exposed in the REST API — useful for building configurations (e.g. with Ansible). +- A **Config Context** tab on the object's detail view, showing the rendered context, the local context, and any source contexts. + +#### Aggregating source config contexts + +Custom objects can also inherit the ConfigContexts that apply to NetBox objects they reference, exactly as a Device or VM would. This works **by field-naming convention** — no extra configuration: + +Give the Custom Object Type a single (non-polymorphic) **object** field named one of the following, pointing at the matching NetBox model: + +| Field name | Points at | +|------------|-----------| +| `site` | Site | +| `tenant` | Tenant | +| `role` | Device Role | +| `platform` | Platform | +| `location` | Location | +| `device_type` | Device Type | +| `cluster` | Cluster | + +Any active ConfigContext assigned to the referenced object (or to its region, site group, tenant group, cluster type/group, or matching tags) is then aggregated into the object's rendered context, ordered by weight, with the object's own **Local Context Data** applied last (it always wins). Assigning tags to the custom object also matches tag-scoped ConfigContexts automatically. + +Notes: + +- Field **names are unique per type**, so there is never ambiguity about which field feeds a dimension. +- A field must be an object field with both the **name and target model** above; a mis-named or mis-pointed field is simply ignored. +- If a type defines **none** of these fields, aggregation is skipped entirely and its rendered context is just its Local Context Data. This is a deliberate difference from Devices/VMs: **global (unassigned) ConfigContexts are not applied** to such a type — enabling config context support alone never silently pulls in every global context. Add at least one convention field (e.g. `site`) to opt the type into source aggregation; global contexts then apply too (as they do for any object with a dimension). + ### Deletions #### Deleting Custom Object Types diff --git a/docs/media/graphql/graphiql-refresh-schema.png b/docs/media/graphql/graphiql-refresh-schema.png new file mode 100644 index 00000000..367cc382 Binary files /dev/null and b/docs/media/graphql/graphiql-refresh-schema.png differ diff --git a/docs/releases.md b/docs/releases.md index 09ad9f89..63b0dcd6 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,5 +1,37 @@ # Releases +## 0.6.0 + +### New Features + +**NetBox Branching Support** + +Custom Object Types and their instances now participate fully in NetBox branching workflows. COT schema changes (adding/removing fields, modifying field definitions) and instance creates/updates/deletes are all branch-aware. A version check ensures the minimum required NetBox and netbox-branching versions are present before enabling branching features. + +- [#404](https://github.com/netboxlabs/netbox-custom-objects/issues/404) - Full branching support for Custom Object Types and instances +- [#569](https://github.com/netboxlabs/netbox-custom-objects/issues/569) - Add version check to gate branching features on required NetBox / netbox-branching versions + +**GraphQL Support** + +Custom Object Types and their instances are now queryable and mutable via the NetBox GraphQL API. Queries, mutations, and filtering are supported for all field types. + +- [#30](https://github.com/netboxlabs/netbox-custom-objects/issues/30) - GraphQL support for Custom Objects + +### Enhancements + +- [#98](https://github.com/netboxlabs/netbox-custom-objects/issues/98) - Config context support for Custom Objects +- [#254](https://github.com/netboxlabs/netbox-custom-objects/issues/254) - Quick Add support for object/multiobject fields +- [#286](https://github.com/netboxlabs/netbox-custom-objects/issues/286) - Contacts support for Custom Objects +- [#376](https://github.com/netboxlabs/netbox-custom-objects/issues/376) - Ownership field for Custom Object instances +- [#532](https://github.com/netboxlabs/netbox-custom-objects/issues/532) - Integer fields now use 64-bit `BigIntegerField` to support values exceeding the 32-bit range +- [#551](https://github.com/netboxlabs/netbox-custom-objects/issues/551) - Add `location` field type for latitude/longitude coordinate pairs + +### Bug Fixes + +- [#572](https://github.com/netboxlabs/netbox-custom-objects/issues/572) - Crash on startup when netbox-branching is installed but not listed in `PLUGINS` + +--- + ## 0.5.3 ### Bug Fixes diff --git a/mkdocs.yml b/mkdocs.yml index 7e127c75..c9c1dd2b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -42,5 +42,6 @@ nav: - Field Attributes: 'field-attributes.md' - Branching Support: 'branching.md' - REST API: 'rest-api.md' + - GraphQL API: 'graphql.md' - Portable Schema: 'portable-schema.md' - Releases: 'releases.md' diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 690a45b1..a65e667e 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -1,4 +1,5 @@ import contextvars +import logging import sys import warnings @@ -12,6 +13,8 @@ from .constants import APP_LABEL as APP_LABEL from .utilities import extract_cot_id_from_model_name, install_clear_cache_suppressor +logger = logging.getLogger(__name__) + # Context variable to track if we're currently running migrations _is_migrating = contextvars.ContextVar('is_migrating', default=False) @@ -27,6 +30,11 @@ # access recomputes with the full set of COT models. _app_ready = False +# Guards ``super().ready()`` against the duplicate-registration check in +# NetBox's ``register_serializer_resolver`` (triggered by ``serializer_resolver`` +# on this PluginConfig). +_super_ready_called = False + def _migration_started(sender, **kwargs): """Signal handler for pre_migrate - sets the migration flag.""" @@ -40,6 +48,78 @@ def _migration_finished(sender, **kwargs): _migrations_checked = None +# Guards the netbox-branching hook setup against duplicate registration if +# ``ready()`` runs more than once (e.g. test isolation paths that reset the +# app registry). Covers signal connects and the netbox-branching +# ``register_*`` functions, which do not all dedupe internally. +_branching_hooks_registered = False + + +def _reset_deferred_co_field_data(sender, **kwargs): + """Module-level receiver so Django's ``Signal.connect`` dedupes it across + repeat ``ready()`` invocations (a closure would have a fresh id each call). + """ + from netbox_custom_objects.models import _deferred_co_field_data + _deferred_co_field_data.set(None) + + +def _register_branching_hooks_once(): + """Register netbox-branching integration hooks at most once per process. + + Wraps the branching-resolver, objectchange-field-migrator, deferred-data + reset receivers, and squash-dependency-graph receiver. Connect both pre- + and post- merge/sync/revert so the deferred-data reset runs even when the + operation raises (post-signals only fire on success). ``weak=False`` keeps + the receivers alive past the end of ``ready()``. + """ + global _branching_hooks_registered + if _branching_hooks_registered: + return + + try: + from netbox_branching.signals import ( + pre_merge, post_merge, + pre_sync, post_sync, + pre_revert, post_revert, + ) + except ImportError: + return + + for sig in (pre_merge, post_merge, pre_sync, post_sync, pre_revert, post_revert): + sig.connect(_reset_deferred_co_field_data, weak=False) + + try: + from netbox_branching.utilities import ( + register_branching_resolver, + register_objectchange_field_migrator, + ) + from .branching import ( + objectchange_field_migrator, + supports_branching_resolver, + ) + register_branching_resolver(supports_branching_resolver) + register_objectchange_field_migrator(objectchange_field_migrator) + # Subscribe to the squash dependency-graph signal so CO-specific + # edges (M2M targets, polymorphic-M2M sidecar) get added before + # topological ordering. Skipped silently on older netbox-branching + # that doesn't expose the signal yet. + try: + from netbox_branching.signals import ( + squash_dependency_graph_built, + ) + from .branching import add_custom_object_dependencies + squash_dependency_graph_built.connect( + add_custom_object_dependencies, + weak=False, + ) + except ImportError: + pass + except ImportError: + pass + + _branching_hooks_registered = True + + # Module-level flag so the heal runs at most once per process invocation even # though post_migrate fires once per installed app. _heal_ran = False @@ -64,12 +144,6 @@ def _heal_mixin_columns(sender, **kwargs): if any(cmd in sys.argv for cmd in ("makemigrations", "collectstatic")): return - # Set the flag *before* running so that subsequent post_migrate firings - # (one per installed app) are no-ops even if the first attempt raises. - # A failure here will not be retried in the same process; operators can - # run 'manage.py upgrade_custom_objects' manually if needed. - _heal_ran = True - try: from netbox_custom_objects.mixin_migration import heal_all_cots # noqa: PLC0415 heal_all_cots(verbosity=kwargs.get("verbosity", 1)) @@ -78,6 +152,13 @@ def _heal_mixin_columns(sender, **kwargs): logging.getLogger(__name__).exception( "upgrade_custom_objects: unexpected error during mixin drift check" ) + # Leave _heal_ran False so a subsequent post_migrate firing (or a + # manual 'manage.py upgrade_custom_objects') gets another attempt. + return + + # Only mark complete on success so a transient failure can be retried by + # the next post_migrate firing in this process. + _heal_ran = True def _patch_object_selector_view(): @@ -115,12 +196,62 @@ def _patched_get_filterset_class(self, model): ObjectSelectorView._get_filterset_class = _patched_get_filterset_class +_graphql_view_patched = False + + +def _patch_graphql_view(): + """ + Patch NetBox's GraphQL view so custom object types appear without a restart. + + NetBox builds its GraphQL schema once at startup and binds it to the view via + ``as_view(schema=...)``. Custom object types are created/deleted at runtime + and the app runs across multiple worker processes, so that single bound + schema goes stale. This patch swaps in a per-request, per-process schema + that is rebuilt only when the database changes (see + :mod:`netbox_custom_objects.graphql.live`). + + Like the ObjectSelectorView patch above, this monkey-patches NetBox because + the schema binding leaves no other extension point. The wrapper is + re-decorated with ``csrf_exempt`` because Django copies that marker from + ``dispatch.__dict__`` onto the view at ``as_view()`` time (which runs after + this patch), and the GraphQL endpoint must accept token-authenticated POSTs. + """ + global _graphql_view_patched + if _graphql_view_patched: + return + + from django.views.decorators.csrf import csrf_exempt + from netbox.graphql.views import NetBoxGraphQLView + + _original_dispatch = NetBoxGraphQLView.dispatch + + @csrf_exempt + def _patched_dispatch(self, request, *args, **kwargs): + from netbox_custom_objects.graphql.live import get_live_schema + + # netbox-branching has already activated the request's branch (from the + # X-NetBox-Branch header, ?_branch=, or the active_branch cookie) — the plugin + # imposes no GraphQL-specific branch policy. get_live_schema builds the + # schema for whichever branch is active, so the schema and the data the query + # returns agree, and core models resolve exactly as they do without the plugin. + try: + live_schema = get_live_schema() + if live_schema is not None: + self.schema = live_schema + except Exception: # noqa: BLE001 - fall back to the static schema + logger.warning("Failed to load live GraphQL schema; using static schema", exc_info=True) + return _original_dispatch(self, request, *args, **kwargs) + + NetBoxGraphQLView.dispatch = _patched_dispatch + _graphql_view_patched = True + + # Plugin Configuration class CustomObjectsPluginConfig(PluginConfig): name = "netbox_custom_objects" verbose_name = "Custom Objects" description = "A plugin to manage custom objects in NetBox" - version = "0.5.3" + version = "0.6.0" author = 'Netbox Labs' author_email = 'support@netboxlabs.com' base_url = "custom-objects" @@ -133,6 +264,18 @@ class CustomObjectsPluginConfig(PluginConfig): } required_settings = [] template_extensions = "template_content.template_extensions" + # Resolves dynamic CO models (table{n}model) to on-the-fly serializers — + # they have no importable path at the conventional location. + serializer_resolver = "api.serializers.serializer_resolver" + # Registers the plugin's GraphQL schema contribution with NetBox. The export + # is intentionally an empty list: custom object types are created/deleted at + # runtime, so a query class built once at startup would be immediately stale. + # The live, per-request schema is served instead by the GraphQL view patch in + # ``_patch_graphql_view`` (see the graphql/schema.py and graphql/live.py module + # docstrings). The path resolves as module ``graphql.schema`` + attribute + # ``schema`` (NetBox loads resources via ``import_string``, which treats the + # final component as an attribute). + graphql_schema = "graphql.schema.schema" @staticmethod def should_skip_dynamic_model_creation(): @@ -216,11 +359,25 @@ def should_skip_dynamic_model_creation(): # Always clear the recursion flag _checking_migrations = False + def _call_super_ready_once(self): + """Call ``super().ready()`` once; subsequent calls are no-ops. + ``register_serializer_resolver`` rejects duplicates.""" + global _super_ready_called + if _super_ready_called: + return + super().ready() + _super_ready_called = True + def ready(self): # Install the thread-safe apps.clear_cache wrapper before any dynamic # model is registered (must happen exactly once, before get_model() runs). install_clear_cache_suppressor() + # Register Django system checks (import triggers @register). These + # enforce the conditional NetBox/netbox-branching version floors that + # PluginConfig's static min_version/max_version can't express. + from . import checks # noqa: F401 + from .models import CustomObjectType from netbox_custom_objects.api.serializers import get_serializer_class @@ -234,6 +391,23 @@ def ready(self): # Patch ObjectSelectorView to support dynamically-generated custom object models _patch_object_selector_view() + # Patch the GraphQL view so custom object types added/removed at runtime + # are reflected in the schema without a NetBox restart. + _patch_graphql_view() + + # Keep the live GraphQL schema's signature cache fresh across workers + # event-driven, so the per-request hot path reads the cache instead of + # polling the database. + from .graphql.live import connect_signature_invalidation + connect_signature_invalidation() + + # Register netbox-branching integration hooks (deferred-data reset + # receivers, branchable resolver, ObjectChange field-name migrator, + # squash dependency-graph receiver). Guarded so the plugin still + # works without netbox-branching, and so repeat ready() invocations + # don't accumulate duplicate handlers. + _register_branching_hooks_once() + # Suppress warnings about database calls during app initialization. # Filter by source module rather than message content — message-pattern # matching proved unreliable on Python 3.13 (the filter silently failed @@ -253,7 +427,7 @@ def ready(self): # Skip database calls if dynamic models can't be created yet if self.should_skip_dynamic_model_creation(): - super().ready() + self._call_super_ready_once() return try: @@ -290,7 +464,7 @@ def ready(self): except (ProgrammingError, OperationalError): # DB schema is incomplete (unapplied migrations). Skip dynamic # model registration — it will happen after migrations finish. - super().ready() + self._call_super_ready_once() return # Signal that ready() has fully completed. get_models() checks this flag @@ -304,7 +478,7 @@ def ready(self): from django.apps import apps as django_apps django_apps.clear_cache() - super().ready() + self._call_super_ready_once() def get_model(self, model_name, require_ready=True): self.apps.check_apps_ready() diff --git a/netbox_custom_objects/api/serializers.py b/netbox_custom_objects/api/serializers.py index be3249ed..3d117af3 100644 --- a/netbox_custom_objects/api/serializers.py +++ b/netbox_custom_objects/api/serializers.py @@ -5,17 +5,20 @@ from core.models import ObjectType from django.contrib.contenttypes.models import ContentType from django.db import transaction -from django.db.utils import OperationalError, ProgrammingError from django.urls import NoReverseMatch from django.utils.translation import gettext_lazy as _ from extras.choices import CustomFieldTypeChoices +from extras.models import ConfigContextModel from netbox.api.serializers import NetBoxModelSerializer from rest_framework import serializers from rest_framework.exceptions import ValidationError from rest_framework.reverse import reverse from rest_framework.utils import model_meta +from users.api.serializers_.owners import OwnerSerializer + from netbox_custom_objects import constants, field_types +from netbox_custom_objects.choices import CustomObjectFieldTypeChoices from netbox_custom_objects.models import (CustomObject, CustomObjectType, CustomObjectTypeField) @@ -333,6 +336,7 @@ class Meta: "version", "group_name", "description", + "config_context_enabled", "tags", "created", "last_updated", @@ -344,6 +348,16 @@ class Meta: read_only_fields = ("schema_document",) brief_fields = ("id", "url", "name", "slug", "description") + def validate_config_context_enabled(self, value): + # Settable only at creation: the backing local_context_data column is + # created once when the type's table is built, so the flag is immutable + # afterwards (mirrors the disable-on-edit behaviour of the web form). + if self.instance is not None and value != self.instance.config_context_enabled: + raise serializers.ValidationError( + _("Config context support cannot be changed after creation.") + ) + return value + def get_table_model_name(self, obj): return obj.get_table_model_name(obj.id) @@ -458,8 +472,22 @@ def get_serializer_class(model, skip_object_fields=False): if isinstance(attr, field_types.PolymorphicM2MDescriptor) ) + # If a COT field is named 'owner', it shadows the OwnerMixin FK on the dynamic model + # (Django silently lets child attrs override abstract parent fields). The serializer + # must skip the FK owner field to avoid OwnerSerializer.to_representation() being + # called on a string value and crashing on .pk. + has_owner_field_conflict = any(f.name == 'owner' for f in model_fields) + # Create field list including all necessary fields base_fields = ["id", "url", "display", "created", "last_updated", "tags"] + if not has_owner_field_conflict: + base_fields.insert(3, "owner") + + # Expose local_context_data when the type opted in to config context support + # (the generated model mixes in ConfigContextModel via + # CustomObjectConfigContextMixin). + if issubclass(model, ConfigContextModel): + base_fields.append("local_context_data") # Include _context field when the model has designated context fields has_context_fields = bool(getattr(model, '_context_field_ids', [])) @@ -469,6 +497,11 @@ def get_serializer_class(model, skip_object_fields=False): # Only include custom field names that will actually be added to the serializer custom_field_names = [] for field in model_fields: + # Coordinates fields expand into two real columns; expose them flat, mirroring + # NetBox core's Site/Device latitude/longitude serializer fields. + if field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + custom_field_names += [f"{field.name}_latitude", f"{field.name}_longitude"] + continue if field.name not in model_field_names: continue # excluded during model generation (e.g. broken FK) if skip_object_fields and field.type in [ @@ -524,6 +557,12 @@ def get_display(self, obj): f.name for f in model_fields if f.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT and f.is_polymorphic } + # (latitude_column, longitude_column) pairs for coordinates fields. + _coordinate_fields = [ + (f"{f.name}_latitude", f"{f.name}_longitude") + for f in model_fields + if f.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES + ] def get__context(self, obj): """Return context field values as a nested display object for APISelect secondary text.""" @@ -661,6 +700,31 @@ def validate(self, data): saved[field_name] = data.pop(field_name) data = NetBoxModelSerializer.validate(self, data) data.update(saved) + + # Coordinates: latitude and longitude must both be set or both be empty. + # On partial updates, only enforce when at least one of the pair is supplied. + for lat_field, lon_field in _coordinate_fields: + lat_in_data = lat_field in data + lon_in_data = lon_field in data + if not lat_in_data and not lon_in_data: + continue + # For a PATCH that touches only one column, the other is absent from + # data; fall back to the instance's current DB value so clearing just + # one half of an already-populated pair is still rejected. + latitude = data.get(lat_field) if lat_in_data else ( + getattr(self.instance, lat_field, None) if self.instance else None + ) + longitude = data.get(lon_field) if lon_in_data else ( + getattr(self.instance, lon_field, None) if self.instance else None + ) + if (latitude is None) != (longitude is None): + # Pin the error to the empty field rather than non_field_errors, + # mirroring the UI form's add_error() behaviour. + missing_field = lat_field if latitude is None else lon_field + raise serializers.ValidationError( + {missing_field: [_("Latitude and longitude must both be set or both be empty.")]} + ) + return data # Create basic attributes for the serializer @@ -676,6 +740,9 @@ def validate(self, data): "validate": validate, } + if not has_owner_field_conflict: + attrs["owner"] = OwnerSerializer(nested=True, required=False, allow_null=True) + if has_context_fields: attrs["_context"] = serializers.SerializerMethodField() attrs["get__context"] = get__context @@ -716,7 +783,8 @@ def validate(self, data): ) # Register the FULL serializer as a module attribute so NetBox's import_string() - # and the module-level __getattr__ fallback can find it. + # can find it (the serializer_resolver below generates on demand; this keeps + # any direct import-path lookups working too). # The partial variant (skip_object_fields=True) is used only as a nested field # descriptor inside another serializer class and must NOT be stored on the module # — doing so would silently replace the full serializer with an incomplete one @@ -729,35 +797,23 @@ def validate(self, data): return serializer -def __getattr__(name): - """ - Module-level lazy resolution for Table{N}ModelSerializer attributes (PEP 562). - - NetBox's get_serializer_for_model() resolves serializers via - import_string("netbox_custom_objects.api.serializers.TableNModelSerializer"), - which ultimately calls getattr(module, name). That lookup hits this hook - when the attribute has not yet been registered — for example in a worker - whose ready() ran against an empty database, or in any worker that started - before a given COT was created. - - Generating on demand here means SerializerNotFound is never raised just - because startup-time registration was skipped or missed (issue #370). - The generated serializer is stored via setattr inside get_serializer_class(), - so subsequent lookups return it directly from __dict__ without re-entering - this hook. +def serializer_resolver(model, prefix=''): + """Resolve dynamic CO models (``table{n}model``) to on-the-fly serializers. + + Called by ``utilities.api.get_serializer_for_model`` before its default + import-path lookup. Returns ``None`` for non-CO models so the default + lookup runs (including this plugin's static CustomObjectType serializer). + + This supersedes the import-path/``__getattr__`` fallback approach: because + the resolver runs before any import path is built for a CO model, the + serializer is always generated on demand and ``SerializerNotFound`` is never + raised just because startup-time registration was skipped or missed + (issue #370). """ - match = re.match(r'^Table(\d+)ModelSerializer$', name) - if match: - cot_id = int(match.group(1)) - try: - obj = CustomObjectType.objects.get(pk=cot_id) - model = obj.get_model() - return get_serializer_class(model) - except (CustomObjectType.DoesNotExist, ProgrammingError, OperationalError, LookupError): - pass - except Exception: - logger.warning( - "Unexpected error generating serializer for %r; serializer will not be available", - name, exc_info=True, - ) - raise AttributeError(f"module '{__name__}' has no attribute {name!r}") + if ( + getattr(model, '_meta', None) + and model._meta.app_label == 'netbox_custom_objects' + and _TABLE_MODEL_PATTERN.match(model.__name__) + ): + return get_serializer_class(model) + return None diff --git a/netbox_custom_objects/api/views.py b/netbox_custom_objects/api/views.py index 58b41153..31b53cf8 100644 --- a/netbox_custom_objects/api/views.py +++ b/netbox_custom_objects/api/views.py @@ -39,8 +39,6 @@ class ETagMixin: # pragma: no cover – NetBox < 4.6 shim UnknownFieldTypeError, UnknownObjectTypeError, ) -from netbox_custom_objects.utilities import is_in_branch - from . import serializers logger = logging.getLogger(__name__) @@ -102,10 +100,6 @@ def _serialize_diff(diff) -> dict: } -# Constants -BRANCH_ACTIVE_ERROR_MESSAGE = _("Please switch to the main branch to perform this operation.") - - class RootView(APIRootView): def get_view_name(self): return "CustomObjects" @@ -158,14 +152,9 @@ def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) def create(self, request, *args, **kwargs): - if is_in_branch(): - raise ValidationError(BRANCH_ACTIVE_ERROR_MESSAGE) return super().create(request, *args, **kwargs) def update(self, request, *args, **kwargs): - if is_in_branch(): - raise ValidationError(BRANCH_ACTIVE_ERROR_MESSAGE) - # Replicate DRF's UpdateModelMixin.update() so we can snapshot the instance # before the serializer is constructed. Calling super().update() would invoke # get_object() a second time and return a fresh, un-snapshotted instance. @@ -415,13 +404,14 @@ class SchemaApplyView(APIView): permission_classes = [IsAuthenticatedOrLoginNotRequired, TokenWritePermission] def post(self, request, *args, **kwargs): - # TODO: Schema apply is blocked while in a branch context because the executor - # performs direct DDL (ALTER/DROP TABLE) that is not branch-aware. When branching - # is extended to support schema operations, remove this guard and wire up the - # appropriate branch-scoped apply path. - if is_in_branch(): - raise ValidationError(BRANCH_ACTIVE_ERROR_MESSAGE) - + # Branch context: this endpoint no longer rejects requests with an active + # branch. Schema-editor calls inside ``apply_document`` route through + # ``_get_schema_connection()`` in models.py, which selects the active + # branch's connection when one is set. The resulting DDL therefore lands + # in the branch's PostgreSQL schema, and the CustomObjectType / + # CustomObjectTypeField writes flow through netbox-branching's router. + # See ``_schema_add_field`` / ``_schema_remove_field`` / ``_schema_alter_field`` + # and ``CustomObjectType.save`` for the routing details. if not ( request.user.has_perm('netbox_custom_objects.add_customobjecttype') and request.user.has_perm('netbox_custom_objects.change_customobjecttype') diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py new file mode 100644 index 00000000..667dd1b1 --- /dev/null +++ b/netbox_custom_objects/branching.py @@ -0,0 +1,135 @@ +"""netbox-branching integration hooks. + +Registered from ``__init__.ready()`` only when netbox-branching is installed. +""" + + +def supports_branching_resolver(model): + """Mark CustomObject M2M through models as branchable. + + Through models are plain ``models.Model`` subclasses (no ChangeLoggingMixin), + so the default heuristic would route their queries to main even inside a + branch — and the FK to the parent CO would resolve against main's rows. + Returning ``True`` pulls them into the branch connection routing. + """ + meta = getattr(model, '_meta', None) + if meta is None or meta.app_label != 'netbox_custom_objects': + return None + name = meta.model_name or '' + if name.startswith('through_custom_objects_'): + return True + return None + + +def objectchange_field_migrator(model, data): + """Rewrite stale field-name keys in *data* for CustomObject models. + + Delegates to ``CustomObject.resolve_field_aliases`` (shared with + ``deserialize_object``). Returns ``None`` for non-CO models so other + plugins' migrators can run. + """ + meta = getattr(model, '_meta', None) + if meta is None or meta.app_label != 'netbox_custom_objects': + return None + resolve = getattr(model, 'resolve_field_aliases', None) + if resolve is None: + return None + return resolve(data) + + +def _collect_co_refs(model_class, data): + """Return ``(app.model, pk)`` refs from CO-specific shapes in *data*. + + Covers: + * Local M2M target lists (squash's default ``_get_fk_references`` only + walks FK / GFK fields, so M2M targets — including self-referential + ones — are invisible to it). + * Every ``CustomObjectTypeField`` on the CO's model. A CO INSERT needs + the field's column (scalar) or through table (M2M) to exist first; + without these edges squash may apply the CO CREATE before the field + CREATEs. Pulled from the model class's ``_field_objects`` plus the + polymorphic ``POLY_M2M_SIDECAR_KEY`` (which carries field PKs in the + ObjectChange payload even when ``_field_objects`` isn't available). + """ + from .constants import APP_LABEL + from .models import POLY_M2M_SIDECAR_KEY + + refs = set() + if not data: + return refs + + for field in model_class._meta.local_many_to_many: + values = data.get(field.name) + if not values: + continue + rel_meta = field.related_model._meta + label = f'{rel_meta.app_label}.{rel_meta.model_name}' + for pk in values: + if isinstance(pk, int): + refs.add((label, pk)) + + field_label = f'{APP_LABEL}.customobjecttypefield' + for fo in (getattr(model_class, '_field_objects', None) or {}).values(): + cotf = fo.get('field') if isinstance(fo, dict) else None + if cotf is not None and getattr(cotf, 'pk', None) is not None: + refs.add((field_label, cotf.pk)) + + entries = data.get(POLY_M2M_SIDECAR_KEY) or () + for entry in entries: + if isinstance(entry, dict) and entry.get('pk') is not None: + refs.add((field_label, entry['pk'])) + + return refs + + +def add_custom_object_dependencies(sender, collapsed_changes, **kwargs): + """Extend squash's dependency graph with CO-specific edges. + + Walks every collapsed change for a CO model and mirrors squash's four + edge-direction rules (UPDATE→DELETE, UPDATE→CREATE, CREATE→CREATE, + DELETE→DELETE) using ``_collect_co_refs`` instead of the FK/GFK walker. + + The signal's ``operation`` kwarg ('merge' or 'revert') is intentionally + ignored: these edges express physical "must exist before" relationships, + and revert reverses the topological order so the same edges produce the + correct undo sequence. + """ + from .constants import APP_LABEL + + deletes_map = {} + updates_map = {} + creates_map = {} + for key, cc in collapsed_changes.items(): + action = cc.final_action.value if cc.final_action else None + if action == 'create': + creates_map[key] = cc + elif action == 'update': + updates_map[key] = cc + elif action == 'delete': + deletes_map[key] = cc + + for cc in collapsed_changes.values(): + meta = getattr(cc.model_class, '_meta', None) + if meta is None or meta.app_label != APP_LABEL: + continue + action = cc.final_action.value if cc.final_action else None + + if action == 'update': + for ref in _collect_co_refs(cc.model_class, cc.prechange_data): + if ref in deletes_map: + deletes_map[ref].depends_on.add(cc.key) + cc.depended_by.add(ref) + for ref in _collect_co_refs(cc.model_class, cc.postchange_data): + if ref in creates_map: + cc.depends_on.add(ref) + creates_map[ref].depended_by.add(cc.key) + elif action == 'create': + for ref in _collect_co_refs(cc.model_class, cc.postchange_data): + if ref != cc.key and ref in creates_map: + cc.depends_on.add(ref) + creates_map[ref].depended_by.add(cc.key) + elif action == 'delete': + for ref in _collect_co_refs(cc.model_class, cc.prechange_data): + if ref != cc.key and ref in deletes_map: + deletes_map[ref].depends_on.add(cc.key) + cc.depended_by.add(ref) diff --git a/netbox_custom_objects/checks.py b/netbox_custom_objects/checks.py new file mode 100644 index 00000000..47585501 --- /dev/null +++ b/netbox_custom_objects/checks.py @@ -0,0 +1,75 @@ +"""Django system checks. + +Enforces conditional version floors that PluginConfig's static +``min_version``/``max_version`` can't express: when netbox-branching is +installed, NetBox and netbox-branching versions are pinned tighter because +the branching integration relies on APIs that only landed in those releases: + +- ``serializer_resolver`` / ``register_serializer_resolver`` — NetBox 4.6.2 +- ``register_branching_resolver``, ``register_objectchange_field_migrator``, + and the ``squash_dependency_graph_built`` signal — netbox-branching 1.0.4 + +Users who never install netbox-branching keep the broader compatibility +window declared by PluginConfig. +""" + +from importlib.metadata import PackageNotFoundError, version as _pkg_version + +from django.apps import apps +from django.conf import settings +from django.core.checks import Error, Warning, register +from packaging.version import InvalidVersion, Version + + +# Version floors enforced only when netbox-branching is installed. +REQUIRED_NETBOX_VERSION_FOR_BRANCHING = '4.6.2' +REQUIRED_BRANCHING_VERSION = '1.0.4' + + +@register() +def check_branching_compatibility(app_configs, **kwargs): + """Enforce branching-only version floors; no-op without netbox-branching.""" + if not apps.is_installed('netbox_branching'): + return [] + + errors = [] + + try: + netbox_version = Version(settings.RELEASE.version) + if netbox_version < Version(REQUIRED_NETBOX_VERSION_FOR_BRANCHING): + errors.append(Error( + f'netbox-custom-objects requires NetBox >= {REQUIRED_NETBOX_VERSION_FOR_BRANCHING} ' + f'when netbox-branching is installed (detected {netbox_version}).', + hint='Upgrade NetBox, or remove netbox-branching from PLUGINS ' + 'if you do not need branching support for custom objects.', + id='netbox_custom_objects.E001', + )) + except (AttributeError, InvalidVersion): + pass # settings.RELEASE missing/unparseable — other checks surface it + + try: + branching_version = Version(_pkg_version('netbox-branching')) + if branching_version < Version(REQUIRED_BRANCHING_VERSION): + errors.append(Error( + f'netbox-custom-objects requires netbox-branching >= ' + f'{REQUIRED_BRANCHING_VERSION} (detected {branching_version}).', + hint=f'Upgrade with: pip install "netbox-branching>={REQUIRED_BRANCHING_VERSION}"', + id='netbox_custom_objects.E002', + )) + except PackageNotFoundError: + # netbox-branching is an installed app but its distribution metadata + # isn't discoverable (e.g. an editable checkout without dist-info), so + # the version floor can't be enforced. Warn rather than silently pass + # so an incompatible checkout doesn't slip through unnoticed. + errors.append(Warning( + 'netbox-branching is installed but its version could not be ' + f'determined, so the >= {REQUIRED_BRANCHING_VERSION} requirement ' + 'cannot be verified.', + hint='If using an editable install, ensure its dist-info metadata ' + 'is present (reinstall with `pip install -e`).', + id='netbox_custom_objects.W001', + )) + except InvalidVersion: + pass # unparseable version string — skip rather than emit a confusing error + + return errors diff --git a/netbox_custom_objects/choices.py b/netbox_custom_objects/choices.py index 8f66351d..59a8f4f4 100644 --- a/netbox_custom_objects/choices.py +++ b/netbox_custom_objects/choices.py @@ -1,7 +1,21 @@ from django.utils.translation import gettext_lazy as _ +from extras.choices import CustomFieldTypeChoices from utilities.choices import ChoiceSet +class CustomObjectFieldTypeChoices(CustomFieldTypeChoices): + """ + Extends NetBox's CustomFieldTypeChoices with field types specific to custom + objects. All existing ``CustomFieldTypeChoices.TYPE_*`` members remain valid. + """ + + TYPE_COORDINATES = "coordinates" + + CHOICES = CustomFieldTypeChoices.CHOICES + ( + (TYPE_COORDINATES, _("Coordinates")), + ) + + class ObjectFieldOnDeleteChoices(ChoiceSet): """Controls what happens to a Custom Object when the referenced object is deleted.""" CASCADE = "cascade" diff --git a/netbox_custom_objects/constants.py b/netbox_custom_objects/constants.py index d7ac7fc3..15c72a42 100644 --- a/netbox_custom_objects/constants.py +++ b/netbox_custom_objects/constants.py @@ -13,6 +13,21 @@ APP_LABEL = "netbox_custom_objects" +# Convention for config-context aggregation (issue #98): a single, non-polymorphic +# OBJECT field named as the key and pointing at the given (app_label, model) feeds the +# matching ConfigContext assignment dimension read by +# ConfigContextQuerySet.get_for_object(). Region / site-group / tenant-group / +# cluster-type / cluster-group are derived from site/tenant/cluster and need no field. +CONFIG_CONTEXT_DIMENSION_FIELDS = { + "site": ("dcim", "site"), + "tenant": ("tenancy", "tenant"), + "role": ("dcim", "devicerole"), + "platform": ("dcim", "platform"), + "location": ("dcim", "location"), + "device_type": ("dcim", "devicetype"), + "cluster": ("virtualization", "cluster"), +} + # Field names that are reserved and cannot be used for custom object fields. # Keep in alphabetical order for ease of reading error message. RESERVED_FIELD_NAMES = [ @@ -36,8 +51,10 @@ "jobs", "journal_entries", "last_updated", + "local_context_data", "model", "objects", + "owner", "pk", "refresh_from_db", "save", diff --git a/netbox_custom_objects/dynamic_forms.py b/netbox_custom_objects/dynamic_forms.py index 153f6d86..6fb227a9 100644 --- a/netbox_custom_objects/dynamic_forms.py +++ b/netbox_custom_objects/dynamic_forms.py @@ -1,6 +1,7 @@ import logging from netbox.forms import NetBoxModelFilterSetForm +from netbox.forms.mixins import OwnerFilterMixin as OwnerFilterFormMixin from utilities.forms.fields import TagFilterField from . import field_types @@ -35,6 +36,6 @@ def build_filterset_form_class(model): return type( f"{model._meta.object_name}FilterForm", - (NetBoxModelFilterSetForm,), + (OwnerFilterFormMixin, NetBoxModelFilterSetForm,), attrs, ) diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index 0de2bf65..ff2acc55 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -1,19 +1,27 @@ +import datetime +import decimal import hashlib import json import logging +from decimal import Decimal +from typing import List import django_tables2 as tables +from strawberry.scalars import JSON from django import forms from django.apps import apps from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.contrib.postgres.fields import ArrayField -from django.core.exceptions import FieldDoesNotExist -from django.core.validators import RegexValidator -from django.db import connection, models +from django.core.exceptions import FieldDoesNotExist, ValidationError +from django.core.validators import (MaxValueValidator, MinValueValidator, + RegexValidator) +from django.db import models, router from django.db.utils import OperationalError, ProgrammingError from django.db.models.fields.related import ForeignKey, ManyToManyDescriptor from django.db.models.manager import Manager +from django.db.models.signals import m2m_changed +from django.urls import reverse from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ @@ -35,7 +43,8 @@ from utilities.templatetags.builtins.filters import linkify, render_markdown from netbox.tables.columns import BooleanColumn -from netbox_custom_objects.choices import ObjectFieldOnDeleteChoices +from netbox_custom_objects.choices import (CustomObjectFieldTypeChoices, + ObjectFieldOnDeleteChoices) from netbox_custom_objects.constants import APP_LABEL from netbox_custom_objects.utilities import extract_cot_id_from_model_name, generate_model @@ -178,11 +187,15 @@ def _resolve_model(self, model): try: from netbox_custom_objects.models import CustomObjectType cot = CustomObjectType.objects.get(pk=int(cot_id_str)) + branch_id = CustomObjectType._active_branch_id() # Only seed the cache if nothing is cached yet; don't overwrite # a full model that was generated by a previous get_model() call. - if not CustomObjectType.is_model_cached(cot.id): + if not CustomObjectType.is_model_cached(cot.id, branch_id): with CustomObjectType._global_lock: - CustomObjectType._model_cache[cot.id] = (actual_model, cot.cache_timestamp) + CustomObjectType._model_cache[(cot.id, branch_id)] = ( + actual_model, + cot.cache_timestamp, + ) except (CustomObjectType.DoesNotExist, OperationalError, ProgrammingError): pass @@ -217,6 +230,18 @@ def _make_lazy_cot_fk(cot, field, on_delete, **field_kwargs): class FieldType: + # The Python annotation Strawberry should use when exposing this field type + # as a scalar GraphQL field. ``None`` means the type is not exposed as a + # plain scalar — relationship types (OBJECT / MULTIOBJECT) are handled by + # dedicated resolvers in ``graphql/types.py`` instead. Co-locating the + # GraphQL mapping here (next to the model/serializer/form mappings) keeps the + # GraphQL schema in sync automatically: a new field type that needs a scalar + # annotation sets it on its own subclass. + graphql_annotation = None + + def get_graphql_annotation(self): + return self.graphql_annotation + def get_display_value(self, instance, field_name): """ This value is used as the object title in the Custom Object detail view. @@ -288,10 +313,11 @@ def _get_related_content_type(self, field): def after_model_generation(self, instance, model, field_name): ... - def create_m2m_table(self, instance, model, field_name): ... + def create_m2m_table(self, instance, model, field_name, schema_conn=None): ... class TextFieldType(FieldType): + graphql_annotation = str def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) @@ -324,6 +350,8 @@ def get_filterform_field(self, field, **kwargs): class LongTextFieldType(FieldType): + graphql_annotation = str + def get_filterform_field(self, field, **kwargs): return forms.CharField( label=field, @@ -361,12 +389,13 @@ def render_table_column(self, value): class IntegerFieldType(FieldType): + graphql_annotation = int def get_model_field(self, field, **kwargs): # TODO: handle all args for IntegerField field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) - return models.IntegerField(null=True, blank=True, **field_kwargs) + return models.BigIntegerField(null=True, blank=True, **field_kwargs) def get_filterform_field(self, field, **kwargs): return forms.IntegerField( @@ -384,6 +413,8 @@ def get_form_field(self, field, **kwargs): class DecimalFieldType(FieldType): + graphql_annotation = decimal.Decimal + def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) @@ -415,6 +446,8 @@ def get_filterform_field(self, field, **kwargs): class BooleanFieldType(FieldType): + graphql_annotation = bool + def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) @@ -449,6 +482,8 @@ def get_table_column_field(self, field, **kwargs): class DateFieldType(FieldType): + graphql_annotation = datetime.date + def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) @@ -468,6 +503,8 @@ def get_filterform_field(self, field, **kwargs): class DateTimeFieldType(FieldType): + graphql_annotation = datetime.datetime + def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) @@ -487,6 +524,8 @@ def get_filterform_field(self, field, **kwargs): class URLFieldType(FieldType): + graphql_annotation = str + def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) @@ -505,6 +544,8 @@ def get_filterform_field(self, field, **kwargs): class JSONFieldType(FieldType): + graphql_annotation = JSON + def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) @@ -524,6 +565,8 @@ def get_filterform_field(self, field, **kwargs): class SelectFieldType(FieldType): + graphql_annotation = str + def get_display_value(self, instance, field_name): value = getattr(instance, field_name) if value is None: @@ -602,6 +645,8 @@ def get_form_field(self, field, for_csv_import=False, **kwargs): class MultiSelectFieldType(FieldType): + graphql_annotation = List[str] + def get_filterform_field(self, field, **kwargs): choices = field.choice_set.choices return DynamicMultipleChoiceField( @@ -817,6 +862,17 @@ def get_form_field(self, field, for_csv_import=False, **kwargs): ) if has_context: form_field.widget.attrs['ts-parent-field'] = '_context' + # Enable quick-add for custom object targets: set quick_add_context + # directly on the widget (bypassing DynamicModelChoiceMixin.get_bound_field + # which uses get_action_url, a tag that can't resolve COT URLs). + if content_type.app_label == APP_LABEL: + form_field.widget.quick_add_context = { + 'url': reverse( + 'plugins:netbox_custom_objects:customobject_add', + kwargs={'custom_object_type': custom_object_type.slug}, + ), + 'params': {}, + } return form_field def get_filterform_field(self, field, **kwargs): @@ -911,6 +967,11 @@ def add_polymorphic_object_columns(self, field_instance, model, schema_editor): on_delete=models.SET_NULL, related_name="+", db_column=f"{field_instance.name}_content_type_id", + # No DB-level FK to django_content_type — same reasoning as the + # polymorphic-MULTIOBJECT through (avoids the AccessExclusiveLock + # branching merge needs to release + lets test teardown TRUNCATE + # django_content_type without manual CASCADE). + db_constraint=False, ) ct_field.contribute_to_class(model, ct_field_name) schema_editor.add_field(model, ct_field) @@ -994,6 +1055,14 @@ def remove_polymorphic_object_columns(self, field_instance, model, schema_editor # fields, this is the first place to check. The Django source to compare against # is django/db/models/fields/related_managers.py :: ManyRelatedManager. class CustomManyToManyManager(Manager): + """ + Many-related manager for custom M2M fields. + + m2m_changed deviation: ``post_clear`` fires with the cleared PKs instead + of ``None`` so netbox-branching's change-capture can log the removal. + Receivers that branch on ``pk_set is None`` will misread it as removes. + """ + def __init__(self, instance=None, field_name=None): super().__init__() self.instance = instance @@ -1066,20 +1135,75 @@ def get_queryset(self): # Add default ordering by pk return qs.order_by("pk") + def _fire_m2m_changed(self, action, pk_set): + """Emit ``m2m_changed`` so change-logging / branching can observe the write. + + Our direct-SQL add/remove/set/clear bypass Django's ManyRelatedManager, + which would otherwise fire this signal — without it, merge/sync replays + zero through-table rows. ``using`` reflects the alias the write + actually happened on; for ``*_clear`` callers pass ``pk_set=None`` to + match Django's contract. + """ + if pk_set is not None and not pk_set: + return + db = router.db_for_write(self.through, instance=self.instance) + m2m_changed.send( + sender=self.through, + instance=self.instance, + action=action, + reverse=False, + model=getattr(self.through._meta.get_field('target').remote_field, 'model', None), + pk_set=pk_set, + using=db, + ) + + @staticmethod + def _resolve_pk(obj): + """Accept an instance or raw PK — mirrors Django's ManyRelatedManager.""" + return obj.pk if hasattr(obj, 'pk') else obj + def add(self, *objs): - for obj in objs: - self.through.objects.get_or_create( - source_id=self.instance.pk, target_id=obj.pk + candidate_pks = {self._resolve_pk(obj) for obj in objs} + if not candidate_pks: + return + self._fire_m2m_changed('pre_add', candidate_pks) + added = set() + for pk in candidate_pks: + _, created = self.through.objects.get_or_create( + source_id=self.instance.pk, target_id=pk ) + if created: + added.add(pk) + if added: + self._fire_m2m_changed('post_add', added) def remove(self, *objs): - for obj in objs: - self.through.objects.filter( - source_id=self.instance.pk, target_id=obj.pk + candidate_pks = {self._resolve_pk(obj) for obj in objs} + if not candidate_pks: + return + self._fire_m2m_changed('pre_remove', candidate_pks) + removed = set() + for pk in candidate_pks: + n, _ = self.through.objects.filter( + source_id=self.instance.pk, target_id=pk ).delete() + if n: + removed.add(pk) + if removed: + self._fire_m2m_changed('post_remove', removed) def clear(self): + existing = set( + self.through.objects.filter(source_id=self.instance.pk) + .values_list('target_id', flat=True) + ) + if not existing: + return + # pre_clear follows Django's contract; post_clear deliberately does + # not — see the class docstring. + self._fire_m2m_changed('pre_clear', None) self.through.objects.filter(source_id=self.instance.pk).delete() + self._fire_m2m_changed('post_clear', existing) def set(self, objs, clear=False): objs = tuple(objs) # force evaluation before any mutation @@ -1087,7 +1211,7 @@ def set(self, objs, clear=False): self.clear() self.add(*objs) else: - new_pks = {obj.pk for obj in objs} + new_pks = {self._resolve_pk(obj) for obj in objs} old_pks = set( self.through.objects.filter(source_id=self.instance.pk) .values_list('target_id', flat=True) @@ -1095,15 +1219,21 @@ def set(self, objs, clear=False): # Remove relationships no longer in the target set to_remove = old_pks - new_pks if to_remove: + self._fire_m2m_changed('pre_remove', to_remove) self.through.objects.filter( source_id=self.instance.pk, target_id__in=to_remove, ).delete() + self._fire_m2m_changed('post_remove', to_remove) # Add only genuinely new relationships - for pk in new_pks - old_pks: - self.through.objects.get_or_create( - source_id=self.instance.pk, target_id=pk - ) + to_add = new_pks - old_pks + if to_add: + self._fire_m2m_changed('pre_add', to_add) + for pk in to_add: + self.through.objects.get_or_create( + source_id=self.instance.pk, target_id=pk + ) + self._fire_m2m_changed('post_add', to_add) class CustomManyToManyDescriptor(ManyToManyDescriptor): @@ -1119,10 +1249,6 @@ def __get__(self, instance, cls=None): return CustomManyToManyManager(instance=instance, field_name=self.field.name) - def get_prefetch_queryset(self, instances, queryset=None): - manager = CustomManyToManyManager(instances[0], self.field.name) - return manager.get_prefetch_queryset(instances, queryset) - def is_cached(self, instance): """ Returns True if the field's value has been cached for the given instance. @@ -1332,6 +1458,14 @@ def get_form_field(self, field, for_csv_import=False, **kwargs): ) if has_context: form_field.widget.attrs['ts-parent-field'] = '_context' + if content_type.app_label == APP_LABEL: + form_field.widget.quick_add_context = { + 'url': reverse( + 'plugins:netbox_custom_objects:customobject_add', + kwargs={'custom_object_type': custom_object_type.slug}, + ), + 'params': {}, + } return form_field def get_display_value(self, instance, field_name): @@ -1422,10 +1556,16 @@ def after_model_generation(self, instance, model, field_name): field = model._meta.get_field(field_name) + # Django's JSON serializer (handle_m2m_field) skips non-auto-created + # through models; mark ours so M2M values appear in change-log output. + # Guarded by test_serialize_object_includes_m2m_values. + through_model = field.remote_field.through + if through_model is not None: + through_model._meta.auto_created = model + # Skip model resolution for self-referential fields if getattr(field, "_is_self_referential", False): field.remote_field.model = model - through_model = field.remote_field.through # Update both source and target fields to point to the same model source_field = through_model._meta.get_field("source") @@ -1484,11 +1624,12 @@ def after_model_generation(self, instance, model, field_name): target_field.remote_field.model = to_model target_field.related_model = to_model - def create_m2m_table(self, instance, model, field_name): + def create_m2m_table(self, instance, model, field_name, schema_conn=None): """ Creates the actual M2M table after models are fully generated """ - from django.db import connection + from django.db import connection as default_connection + connection = schema_conn if schema_conn is not None else default_connection # Get the field instance field = model._meta.get_field(field_name) @@ -1514,44 +1655,31 @@ def create_m2m_table(self, instance, model, field_name): else: to_model = content_type.model_class() - # Create the through model with actual model references + # Create a fresh through for THIS CO model. Django's metaclass will + # auto-register it in apps.all_models, but the M2M field itself keeps + # a direct reference — so even if a later branch generation overwrites + # the registry entry, this field's through is unaffected. through = self.get_through_model(instance, model) - # Update the through model's foreign key references source_field = through._meta.get_field("source") target_field = through._meta.get_field("target") - - # Source field should point to the current model source_field.remote_field.model = model source_field.remote_field.field_name = model._meta.pk.name source_field.related_model = model - - # Target field should point to the related model target_field.remote_field.model = to_model target_field.remote_field.field_name = to_model._meta.pk.name target_field.related_model = to_model - # Register the model with Django's app registry - apps = model._meta.apps - - try: - through_model = apps.get_model(APP_LABEL, instance.through_model_name) - except LookupError: - apps.register_model(APP_LABEL, through) - through_model = through - - # Update the M2M field's through model and target model - field.remote_field.through = through_model + field.remote_field.through = through field.remote_field.model = to_model field.remote_field.field_name = to_model._meta.pk.name - # Create the through table with connection.schema_editor() as schema_editor: - table_name = through_model._meta.db_table + table_name = through._meta.db_table with connection.cursor() as cursor: tables = connection.introspection.table_names(cursor) if table_name not in tables: - schema_editor.create_model(through_model) + schema_editor.create_model(through) def get_polymorphic_through_model(self, field_instance, source_model_string): """ @@ -1567,10 +1695,10 @@ def get_polymorphic_through_model(self, field_instance, source_model_string): "app_label": APP_LABEL, "apps": apps, "managed": True, - "unique_together": (("source", "content_type", "object_id"),), + "unique_together": (("source", "content_type_id", "object_id"),), "indexes": [ models.Index( - fields=["content_type", "object_id"], + fields=["content_type_id", "object_id"], name=_safe_index_name( f"co_{field_instance.custom_object_type_id}" f"_{field_instance.name}_pgfk" @@ -1580,6 +1708,13 @@ def get_polymorphic_through_model(self, field_instance, source_model_string): }, ) + # content_type_id is a plain integer, not a ContentType FK. An ORM FK + # would (a) install a PG trigger needing AccessExclusiveLock on + # django_content_type during CREATE TABLE → deadlocks netbox-branching + # merge (default-conn DDL vs branch-conn changed_object_type lookups), + # and (b) make Django's collector traverse the through when an + # ObjectType is deleted, querying a table revert has already dropped. + # The descriptor tolerates orphan content_type_ids in _get_objects. attrs = { "__module__": "netbox_custom_objects.models", "Meta": meta, @@ -1590,12 +1725,7 @@ def get_polymorphic_through_model(self, field_instance, source_model_string): related_name="+", db_column="source_id", ), - "content_type": models.ForeignKey( - "contenttypes.ContentType", - on_delete=models.CASCADE, - related_name="+", - db_column="content_type_id", - ), + "content_type_id": models.PositiveBigIntegerField(db_column="content_type_id"), "object_id": models.PositiveBigIntegerField(db_column="object_id"), } @@ -1614,39 +1744,41 @@ def create_polymorphic_m2m_table(self, field_instance, model, schema_editor): source_model_string = f"{APP_LABEL}.{model.__name__}" through = self.get_polymorphic_through_model(field_instance, source_model_string) - # Update source FK to point to the actual model source_field = through._meta.get_field("source") source_field.remote_field.model = model source_field.related_model = model - # Register with Django's app registry - _apps = model._meta.apps - try: - through_model = _apps.get_model(APP_LABEL, field_instance.through_model_name) - except LookupError: - _apps.register_model(APP_LABEL, through) - through_model = through - - table_name = through_model._meta.db_table - with connection.cursor() as cursor: - existing_tables = connection.introspection.table_names(cursor) + # Probe the same schema the DDL will target. schema_editor is branch-aware + # (opened via _get_schema_connection() by the caller), whereas the module-level + # ``connection`` always points at the main schema — using it here would let the + # idempotency guard diverge from where create_model() actually writes. + conn = schema_editor.connection + table_name = through._meta.db_table + with conn.cursor() as cursor: + existing_tables = conn.introspection.table_names(cursor) if table_name not in existing_tables: - schema_editor.create_model(through_model) + schema_editor.create_model(through) def drop_polymorphic_m2m_table(self, field_instance, model, schema_editor): - """ - Drops the DB table for a polymorphic MultiObject through model. + """Drops the DB table for a polymorphic MultiObject through. - ``schema_editor`` must be supplied by the caller for the same reason as - ``remove_polymorphic_object_columns``: all DDL in a single operation - should share one schema editor context. + Looks up the through on the CO model (per-context) rather than via + apps.get_model, so we drop the table this model knows about, not + whatever happens to be registered in the global app registry. + ``schema_editor`` is shared by the caller — opening a nested one would + flush deferred SQL prematurely on PostgreSQL. """ - _apps = model._meta.apps - try: - through_model = _apps.get_model(APP_LABEL, field_instance.through_model_name) - schema_editor.delete_model(through_model) - except LookupError: - pass # Already dropped or never created + through_model = None + for tm in getattr(model, '_through_models', None) or (): + if tm._meta.db_table == field_instance.through_table_name: + through_model = tm + break + if through_model is None: + try: + through_model = model._meta.apps.get_model(APP_LABEL, field_instance.through_model_name) + except LookupError: + return # already gone + schema_editor.delete_model(through_model) class PolymorphicResultList: @@ -1701,6 +1833,11 @@ class PolymorphicManyToManyManager: Manager for polymorphic many-to-many relationships. Handles objects from multiple model types via a through table with (source_id, content_type_id, object_id) columns. + + m2m_changed deviation: ``post_clear`` fires with the cleared object PKs + instead of ``None`` so netbox-branching's change-capture can log the + removal. Receivers that branch on ``pk_set is None`` will misread it as + removes. """ def __init__(self, instance, field_name, through_model_name): @@ -1728,7 +1865,12 @@ def _get_objects(self): # Build a lookup map: (ct_id, obj_id) → object, preserving row order below. obj_map: dict[tuple, object] = {} for ct_id, obj_ids in by_ct.items(): - ct = ContentType.objects.get_for_id(ct_id) + try: + ct = ContentType.objects.get_for_id(ct_id) + except ContentType.DoesNotExist: + # Orphan row — content_type_id has no DB FK (see + # get_polymorphic_through_model). Skip. + continue model_class = ct.model_class() if model_class is None: continue @@ -1752,58 +1894,117 @@ def count(self): def exists(self): return self._get_through_model().objects.filter(source_id=self.instance.pk).exists() + def _fire_m2m_changed(self, action, pk_set): + """Emit ``m2m_changed`` so change-logging records the polymorphic write. + + Polymorphic targets span multiple model classes, so pass the through + as ``model`` (NetBox's handler only checks action + pk_set truthiness). + Postchange data comes from ``CustomObject.serialize_object``. + """ + if pk_set is not None and not pk_set: + return + through = self._get_through_model() + db = router.db_for_write(through, instance=self.instance) + m2m_changed.send( + sender=through, + instance=self.instance, + action=action, + reverse=False, + model=through, + pk_set=pk_set, + using=db, + ) + def add(self, *objs): through = self._get_through_model() + candidate_keys = [] for obj in objs: ct = ContentType.objects.get_for_model(obj) - through.objects.get_or_create( + candidate_keys.append((ct.pk, obj.pk)) + if not candidate_keys: + return + self._fire_m2m_changed('pre_add', {pk for _, pk in candidate_keys}) + added = set() + for ct_pk, obj_pk in candidate_keys: + _, created = through.objects.get_or_create( source_id=self.instance.pk, - content_type_id=ct.pk, - object_id=obj.pk, + content_type_id=ct_pk, + object_id=obj_pk, ) + if created: + added.add(obj_pk) + if added: + self._fire_m2m_changed('post_add', added) def remove(self, *objs): through = self._get_through_model() + candidate_keys = [] for obj in objs: ct = ContentType.objects.get_for_model(obj) - through.objects.filter( + candidate_keys.append((ct.pk, obj.pk)) + if not candidate_keys: + return + self._fire_m2m_changed('pre_remove', {pk for _, pk in candidate_keys}) + removed = set() + for ct_pk, obj_pk in candidate_keys: + n, _ = through.objects.filter( source_id=self.instance.pk, - content_type_id=ct.pk, - object_id=obj.pk, + content_type_id=ct_pk, + object_id=obj_pk, ).delete() + if n: + removed.add(obj_pk) + if removed: + self._fire_m2m_changed('post_remove', removed) def clear(self): - self._get_through_model().objects.filter(source_id=self.instance.pk).delete() + through = self._get_through_model() + existing = set( + through.objects.filter(source_id=self.instance.pk) + .values_list('object_id', flat=True) + ) + if not existing: + return + # pre_clear follows Django's contract; post_clear deliberately does + # not — see the class docstring. + self._fire_m2m_changed('pre_clear', None) + through.objects.filter(source_id=self.instance.pk).delete() + self._fire_m2m_changed('post_clear', existing) def set(self, objs, clear=False): if clear: self.clear() self.add(*objs) - else: - # Diff-based replacement: add new, remove old. Matches Django's - # standard ManyRelatedManager.set(clear=False) behaviour. - objs = tuple(objs) - through = self._get_through_model() - existing = { - (ct_id, obj_id) - for ct_id, obj_id in through.objects.filter(source_id=self.instance.pk) - .values_list("content_type_id", "object_id") - } - # Pre-compute (ct_id, obj_pk) once per object to avoid duplicate CT lookups. - new_items = [ - (ContentType.objects.get_for_model(obj).pk, obj.pk, obj) for obj in objs - ] - new_keys = {(ct_id, obj_pk) for ct_id, obj_pk, _ in new_items} - to_add = [obj for ct_id, obj_pk, obj in new_items if (ct_id, obj_pk) not in existing] - to_remove = existing - new_keys - if to_add: - self.add(*to_add) + return + + # Diff-based replacement: add new, remove old. Matches Django's + # standard ManyRelatedManager.set(clear=False) behaviour. + objs = tuple(objs) + through = self._get_through_model() + existing = { + (ct_id, obj_id) + for ct_id, obj_id in through.objects.filter(source_id=self.instance.pk) + .values_list("content_type_id", "object_id") + } + # Pre-compute (ct_id, obj_pk) once per object to avoid duplicate CT lookups. + new_items = [ + (ContentType.objects.get_for_model(obj).pk, obj.pk, obj) for obj in objs + ] + new_keys = {(ct_id, obj_pk) for ct_id, obj_pk, _ in new_items} + to_add = [obj for ct_id, obj_pk, obj in new_items if (ct_id, obj_pk) not in existing] + to_remove = existing - new_keys + if to_add: + self.add(*to_add) + if to_remove: + remove_pks = {obj_id for _, obj_id in to_remove} + self._fire_m2m_changed('pre_remove', remove_pks) for ct_id, obj_id in to_remove: through.objects.filter( source_id=self.instance.pk, content_type_id=ct_id, object_id=obj_id, ).delete() + self._fire_m2m_changed('post_remove', remove_pks) def __iter__(self): return iter(self.all()) @@ -1980,6 +2181,147 @@ def __get__(self, instance, owner=None): return PolymorphicMultiObjectReverseManager(instance, self.cot_pk, self.through_model_name) +class CoordinatesColumn(tables.Column): + """ + Table column for a coordinates field. Renders the latitude/longitude pair stored + in the two backing columns as ``", "`` (or a placeholder when unset). + """ + + def __init__(self, latitude_accessor, longitude_accessor, *args, **kwargs): + kwargs.setdefault("accessor", latitude_accessor) + kwargs.setdefault("orderable", True) + super().__init__(*args, **kwargs) + self.latitude_accessor = latitude_accessor + self.longitude_accessor = longitude_accessor + + def render(self, record): + latitude = getattr(record, self.latitude_accessor, None) + longitude = getattr(record, self.longitude_accessor, None) + if latitude is None or longitude is None: + return self.default + return f"{latitude}, {longitude}" + + +class CoordinatesFieldType(FieldType): + """ + A geographic coordinates field. A single field of this type expands into two real + DB columns, ``_latitude`` and ``_longitude``, mirroring NetBox's native + Site/Device coordinate handling (plain ``DecimalField``s; no PostGIS dependency). + """ + + @staticmethod + def latitude_field_name(field): + return f"{field.name}_latitude" + + @staticmethod + def longitude_field_name(field): + return f"{field.name}_longitude" + + def get_model_field(self, field, **kwargs): + return { + self.latitude_field_name(field): models.DecimalField( + verbose_name=_("latitude"), + null=True, + blank=True, + max_digits=8, + decimal_places=6, + validators=[ + MinValueValidator(Decimal("-90.0")), + MaxValueValidator(Decimal("90.0")), + ], + help_text=_("GPS coordinate in decimal format (xx.yyyyyy)"), + ), + self.longitude_field_name(field): models.DecimalField( + verbose_name=_("longitude"), + null=True, + blank=True, + max_digits=9, + decimal_places=6, + validators=[ + MinValueValidator(Decimal("-180.0")), + MaxValueValidator(Decimal("180.0")), + ], + help_text=_("GPS coordinate in decimal format (xx.yyyyyy)"), + ), + } + + def get_coordinate_values(self, instance, field): + """Return the (latitude, longitude) tuple stored on an instance.""" + return ( + getattr(instance, self.latitude_field_name(field), None), + getattr(instance, self.longitude_field_name(field), None), + ) + + def get_display_value(self, instance, field_name): + latitude = getattr(instance, f"{field_name}_latitude", None) + longitude = getattr(instance, f"{field_name}_longitude", None) + if latitude is None or longitude is None: + return None + return f"{latitude}, {longitude}" + + def get_form_fields(self, field): + """ + Return the two annotated form fields (latitude, longitude) keyed by their + backing column names. The keys match real model columns, so the generated + ModelForm binds and persists them natively. + """ + base_label = field.label or field.name.replace("_", " ").title() + latitude = forms.DecimalField( + label=f"{base_label} ({_('latitude')})", + required=field.required, + max_digits=8, + decimal_places=6, + min_value=Decimal("-90.0"), + max_value=Decimal("90.0"), + help_text=_("GPS coordinate in decimal format (xx.yyyyyy)"), + ) + longitude = forms.DecimalField( + label=f"{base_label} ({_('longitude')})", + required=field.required, + max_digits=9, + decimal_places=6, + min_value=Decimal("-180.0"), + max_value=Decimal("180.0"), + help_text=_("GPS coordinate in decimal format (xx.yyyyyy)"), + ) + if field.ui_editable != CustomFieldUIEditableChoices.YES: + latitude.disabled = True + longitude.disabled = True + return { + self.latitude_field_name(field): latitude, + self.longitude_field_name(field): longitude, + } + + def get_filterform_field(self, field, **kwargs): + base_label = field.label or field.name.replace("_", " ").title() + return { + self.latitude_field_name(field): forms.DecimalField( + label=f"{base_label} ({_('latitude')})", required=False + ), + self.longitude_field_name(field): forms.DecimalField( + label=f"{base_label} ({_('longitude')})", required=False + ), + } + + def get_table_column_field(self, field, **kwargs): + return CoordinatesColumn( + self.latitude_field_name(field), + self.longitude_field_name(field), + verbose_name=str(field), + ) + + @staticmethod + def validate_pair(latitude, longitude): + """ + Enforce that latitude and longitude are either both set or both empty. + Raises ``django.core.exceptions.ValidationError`` on a half-populated pair. + """ + if (latitude is None) != (longitude is None): + raise ValidationError( + _("Latitude and longitude must both be set or both be empty.") + ) + + FIELD_TYPE_CLASS = { CustomFieldTypeChoices.TYPE_TEXT: TextFieldType, CustomFieldTypeChoices.TYPE_LONGTEXT: LongTextFieldType, @@ -1994,4 +2336,5 @@ def __get__(self, instance, owner=None): CustomFieldTypeChoices.TYPE_MULTISELECT: MultiSelectFieldType, CustomFieldTypeChoices.TYPE_OBJECT: ObjectFieldType, CustomFieldTypeChoices.TYPE_MULTIOBJECT: MultiObjectFieldType, + CustomObjectFieldTypeChoices.TYPE_COORDINATES: CoordinatesFieldType, } diff --git a/netbox_custom_objects/filtersets.py b/netbox_custom_objects/filtersets.py index 62b90173..ba319f80 100644 --- a/netbox_custom_objects/filtersets.py +++ b/netbox_custom_objects/filtersets.py @@ -7,10 +7,13 @@ from django.apps import apps as django_apps from django.db.models import QuerySet, Q from django.utils.dateparse import parse_date, parse_datetime +from django.utils.timezone import make_aware, is_aware from extras.choices import CustomFieldTypeChoices from netbox.filtersets import NetBoxModelFilterSet +from users.models import Owner, OwnerGroup +from .choices import CustomObjectFieldTypeChoices from .constants import APP_LABEL from .models import CustomObjectType @@ -339,6 +342,18 @@ def build_filter_for_field(field) -> dict: ): return _build_polymorphic_filters(field) + # Coordinates expand into two real columns; register a numeric filter for each. + if field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + base_label = field.label or field.name + return { + f"{field.name}_latitude": django_filters.NumberFilter( + field_name=f"{field.name}_latitude", label=f"{base_label} (latitude)" + ), + f"{field.name}_longitude": django_filters.NumberFilter( + field_name=f"{field.name}_longitude", label=f"{base_label} (longitude)" + ), + } + spec = FIELD_TYPE_FILTERS.get(field.type) if not spec: return {} @@ -434,15 +449,33 @@ def search(self, queryset, name, value): elif field.type == CustomFieldTypeChoices.TYPE_DATETIME: parsed = parse_datetime(value) if parsed is not None: + if not is_aware(parsed): + parsed = make_aware(parsed) q |= Q(**{f"{field.name}__exact": parsed}) if not q: return queryset.none() return queryset.filter(q) + def filter_owner_group_id(self, queryset, name, value): + if not value: + return queryset + return queryset.filter(owner__group__in=value) + attrs = { "Meta": meta, "__module__": "netbox_custom_objects.filtersets", "search": search, + "filter_owner_group_id": filter_owner_group_id, + "owner_id": django_filters.ModelMultipleChoiceFilter( + field_name='owner', + queryset=Owner.objects.all(), + label='Owner (ID)', + ), + "owner_group_id": django_filters.ModelMultipleChoiceFilter( + queryset=OwnerGroup.objects.all(), + method='filter_owner_group_id', + label='Owner Group (ID)', + ), } # For each custom field, add a corresponding filter (dict of name → Filter). diff --git a/netbox_custom_objects/forms.py b/netbox_custom_objects/forms.py index 0e560fae..3602f442 100644 --- a/netbox_custom_objects/forms.py +++ b/netbox_custom_objects/forms.py @@ -63,7 +63,7 @@ class CustomObjectTypeForm(NetBoxModelForm): fieldsets = ( FieldSet( "name", "verbose_name", "verbose_name_plural", "slug", - "version", "description", "group_name", "tags", + "version", "description", "group_name", "config_context_enabled", "tags", ), ) comments = CommentField() @@ -72,9 +72,20 @@ class Meta: model = CustomObjectType fields = ( "name", "verbose_name", "verbose_name_plural", "slug", "version", "description", - "group_name", "comments", "tags", + "group_name", "config_context_enabled", "comments", "tags", ) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # config_context_enabled controls whether the generated model carries a + # local_context_data column, which is created once at type creation. It + # cannot be toggled afterwards, so lock it when editing an existing type. + if self.instance.pk: + self.fields["config_context_enabled"].disabled = True + self.fields["config_context_enabled"].help_text = _( + "Config context support cannot be changed after creation." + ) + class CustomObjectTypeBulkEditForm(NetBoxModelBulkEditForm): description = forms.CharField( @@ -217,7 +228,7 @@ class CustomObjectTypeFieldForm(CustomFieldForm): class Meta: model = CustomObjectTypeField - fields = "__all__" + fields = '__all__' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/netbox_custom_objects/graphql/__init__.py b/netbox_custom_objects/graphql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/netbox_custom_objects/graphql/live.py b/netbox_custom_objects/graphql/live.py new file mode 100644 index 00000000..6f8ccbfe --- /dev/null +++ b/netbox_custom_objects/graphql/live.py @@ -0,0 +1,383 @@ +""" +Live (restart-free) GraphQL schema for custom objects. + +NetBox builds its GraphQL schema once at startup and binds it to the GraphQL +view. Custom object types, however, are created and deleted at runtime, and the +plugin runs across multiple worker processes — so a schema built once at boot +goes stale and a signal-based rebuild in one process can't reach the others. + +This module makes the schema reflect the current database on every request, in +every process, without a restart: + +- :func:`schema_signature` computes a cheap fingerprint of the custom object + types and their fields (counts + most-recent change timestamps). The result + is memoised in NetBox's shared cache and invalidated event-driven by the + post_save/post_delete receivers in :func:`connect_signature_invalidation`, so + the steady-state cost is one cache read rather than two DB queries per request. +- :func:`get_live_schema` caches the assembled schema per process and rebuilds + it only when the signature changes. Because each process checks the signature + independently, a type created by a request handled in one process becomes + visible to every process on its next request — no restart, no cross-process + messaging. + +GraphQL resolves against whatever branch netbox-branching activated for the +request — the ``X-NetBox-Branch`` header, the ``?_branch=`` query param, or the +``active_branch`` cookie — exactly like the REST API and the rest of the UI (the +plugin imposes no GraphQL-specific branch policy of its own); with no branch active +it is main. The schema cache is keyed per branch (see :func:`_active_branch_key`) +so each branch's custom object types are reflected independently, and +:func:`get_live_schema` builds the schema for whichever branch is active so the +schema and the data the query returns always agree. + +The view patch in ``__init__.py`` calls :func:`get_live_schema` per request and +assigns the result to the view before it executes the operation. +""" + +import logging +import threading + +from django.apps import apps as django_apps + +logger = logging.getLogger("netbox_custom_objects.graphql") + +# Single-flight rebuild lock. Two roles: +# 1. Correctness (required): build_query_classes() clears and repopulates the +# *process-global* type cache in graphql.types (clear_type_cache()). Two +# rebuilds running at once — e.g. for different branches — would race on that +# cache, one wiping the other's freshly built entries mid-build. This single +# lock (global, not per-branch) serialises ALL rebuilds so that can't happen. +# 2. Performance: concurrent requests that find a stale schema serve it instead +# of blocking on the (potentially expensive) rebuild a peer is already doing. +_rebuild_lock = threading.Lock() +# Per-branch cache of (signature, schema), keyed by branch identifier (None = main). +# GraphQL reflects whichever branch netbox-branching activated for the request, so +# each branch gets its own schema reflecting that branch's custom object types. Each +# value is an atomically swapped (signature, schema) tuple, so a lockless reader can +# never see a schema that doesn't match its signature. +_schema_cache = {} +# Signature cache keys this process has populated, so reset_cache (tests) can clear +# every per-branch entry. +_signature_keys_seen = set() + + +def _active_branch_key(): + """ + Identifier for the active branch (``None`` for main), used to key the per-branch + schema and signature caches. ``None`` when netbox-branching is not installed. + """ + try: + from netbox_branching.contextvars import active_branch + except ImportError: + return None + branch = active_branch.get() + return branch.pk if branch is not None else None + + +def schema_signature(): + """ + Return a cheap, comparable fingerprint of the custom object type schema. + + Captures both custom object types and their fields so that creating, + deleting, or editing either invalidates the cached GraphQL schema: + + - count detects additions and removals, + - max timestamp detects edits (``cache_timestamp`` / ``last_updated`` are + ``auto_now`` fields). + """ + from django.db.models import Count, Max + + from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField + + cot = CustomObjectType.objects.aggregate(n=Count("id"), t=Max("cache_timestamp")) + fields = CustomObjectTypeField.objects.aggregate(n=Count("id"), t=Max("last_updated")) + return (cot["n"], cot["t"], fields["n"], fields["t"]) + + +# Sentinel signature for a best-effort first build made when the signature query +# itself failed (see get_live_schema): it compares unequal to every real +# signature, so the next request with a working DB rebuilds. +_UNKNOWN_SIGNATURE = object() + +_SIGNATURE_CACHE_KEY = "netbox_custom_objects.graphql.schema_signature" +# Backstop TTL only — invalidation is event-driven (connect_signature_invalidation). +# It bounds staleness if an invalidation is ever lost (e.g. a cache blip during a +# write) without making the steady state poll the database. +_SIGNATURE_CACHE_TIMEOUT = 300 + + +def _signature_cache_key(branch_key): + """Per-branch cache key for the schema signature (``None`` = main).""" + if branch_key is None: + return _SIGNATURE_CACHE_KEY + return f"{_SIGNATURE_CACHE_KEY}:{branch_key}" + + +def cached_schema_signature(branch_key=None): + """ + Return the active branch's schema signature, memoised in NetBox's shared cache. + + :func:`schema_signature` is two aggregate queries; running them on every + GraphQL request is a needless per-request DB tax when the schema almost never + changes. The cached value is invalidated event-driven by the receivers in + :func:`connect_signature_invalidation`, so a change in any worker is reflected + everywhere on the next request — the same freshness guarantee as polling, at + one cache read instead of two DB round-trips. The key includes the branch + identifier so a branch's signature never shadows main's, and the aggregates run + under the caller's active branch context. Falls back to a direct DB read + whenever the cache is unavailable. + """ + from django.core.cache import cache + + key = _signature_cache_key(branch_key) + _signature_keys_seen.add(key) + try: + cached = cache.get(key) + except Exception: # noqa: BLE001 - cache down: fall back to the DB + cached = None + if cached is not None: + # Cache backends may round-trip the tuple as a list; normalise so the + # equality check against the stored signature stays type-stable. + return tuple(cached) + + signature = schema_signature() + try: + cache.set(key, signature, _SIGNATURE_CACHE_TIMEOUT) + except Exception: # noqa: BLE001 - cache down: just skip memoisation + pass + return signature + + +def _invalidate_signature_cache(**kwargs): + """ + Drop the memoised schema signature for the branch the change occurred in, so the + next request for that branch recomputes it. The receiver fires inside whatever + branch context performed the save/delete, so the active branch is the one to + invalidate. + """ + from django.core.cache import cache + + try: + cache.delete(_signature_cache_key(_active_branch_key())) + except Exception: # noqa: BLE001 - cache down: the TTL backstop still bounds staleness + logger.debug("Could not invalidate GraphQL schema signature cache", exc_info=True) + + +def _evict_branch_schema(sender, instance, **kwargs): + """ + Drop a deleted branch's cached schema and signature. + + A branch's schema is cached under its pk in :data:`_schema_cache`; once the + branch is gone that entry can never be served again (a request can't reference a + deleted branch), so evict it rather than leak it for the life of the process. + """ + branch_key = instance.pk + _schema_cache.pop(branch_key, None) # atomic dict op; no lock needed + + from django.core.cache import cache + + key = _signature_cache_key(branch_key) + _signature_keys_seen.discard(key) + try: + cache.delete(key) + except Exception: # noqa: BLE001 - cache down: the TTL backstop still bounds it + logger.debug("Could not evict deleted branch's schema signature", exc_info=True) + + +def connect_signature_invalidation(): + """ + Connect the receivers that invalidate the cached schema signature. + + Called once from ``CustomObjectsPluginConfig.ready()``. Creating, deleting, + or editing any custom object type or field changes the signature; deleting the + cache key on those events keeps :func:`cached_schema_signature` correct without + polling the database per request. Deleting a branch evicts that branch's cached + schema so it can't leak. ``dispatch_uid`` makes repeat ``ready()`` calls + idempotent. + """ + from django.db.models.signals import post_delete, post_save + + from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField + + for signal, label in ((post_save, "save"), (post_delete, "delete")): + for model in (CustomObjectType, CustomObjectTypeField): + signal.connect( + _invalidate_signature_cache, + sender=model, + dispatch_uid=f"nco_graphql_sig_{label}_{model.__name__}", + weak=False, + ) + + # Evict a branch's cached schema when the branch itself is deleted. No-op when + # netbox-branching is not installed or not in INSTALLED_APPS. + if not django_apps.is_installed('netbox_branching'): + return + from netbox_branching.models import Branch + post_delete.connect( + _evict_branch_schema, + sender=Branch, + dispatch_uid="nco_graphql_evict_branch", + weak=False, + ) + + +# NetBox assembles its GraphQL schema once at import and never changes it for the +# life of the process — only our custom-object slice does. Capture NetBox's +# static contribution (its per-app/plugin Query bases and its own +# ``StrawberryConfig``) once; every rebuild then regenerates only our slice and +# reassembles. Reusing NetBox's real config — rather than a hand-copied one — +# keeps the rebuilt schema in lock-step with NetBox's own (e.g. its stored +# ``auto_camel_case`` is ``None``, not the ``False`` a copy would assume) and +# removes a source of silent drift across supported NetBox versions. +_static_query_parts = None + + +def _get_static_query_parts(): + """Return ``(query_bases, config)`` captured once from NetBox's startup schema.""" + global _static_query_parts + if _static_query_parts is None: + import netbox.graphql.schema as ngs + + # These bases never include our own contribution (our startup + # ``graphql_schema`` export is empty and we never write back to + # ``ngs.Query``); the ``_nco_query`` guard is belt-and-braces. + bases = tuple(b for b in ngs.Query.__bases__ if not getattr(b, "_nco_query", False)) + _static_query_parts = (bases, ngs.schema.config) + return _static_query_parts + + +def build_full_schema(): + """ + Assemble a complete NetBox GraphQL schema with the current custom object types. + + A ``strawberry.Schema`` is immutable once compiled, so adding/removing a root + query field requires building a new schema — but only our custom-object slice + (:func:`build_query_classes`) is rebuilt here. NetBox's Query bases and config + are captured once (:func:`_get_static_query_parts`); the result is identical to + NetBox's startup schema except for the live custom-object query. + """ + import strawberry + + import netbox.graphql.schema as ngs + + from .schema import build_query_classes + + static_bases, config = _get_static_query_parts() + bases = static_bases + tuple(build_query_classes()) + + # Every rebuild creates fresh classes with names ("Query", "TableModelType", + # …) that were used by previous rebuilds' schemas. This relies on Strawberry + # building a *per-schema* type map at strawberry.Schema(...) time — same-named + # types in different Schema objects don't collide; only duplicates *within* one + # schema do (which build_query_classes avoids by uniquifying field/type names). + # That behaviour is version-dependent and Strawberry isn't pinned here (it's + # whatever NetBox ships); it's exercised by the multi-rebuild tests + # (test_live_schema_drops_deleted_type, test_get_live_schema_rebuilds_on_new_type). + # If a future Strawberry raised on cross-schema name reuse, get_live_schema would + # log the rebuild failure and keep serving the prior schema (not silently break). + query_cls = strawberry.type(type("Query", bases, {})) + return strawberry.Schema( + query=query_cls, + config=config, + # Built fresh per rebuild on purpose — unlike ``config`` (a read-only + # settings object), get_schema_extensions() returns extension *instances* + # that strawberry mutates per request (``extension.execution_context = …``). + # Each Schema must own its own set; sharing one set across our several live + # schemas (main + per branch, plus an old schema still serving in-flight + # requests during a rebuild) would let concurrent requests on different + # schemas clobber each other's execution context. This mirrors what NetBox + # gets for its own schema, and rebuilds are rare, so the cost is irrelevant. + extensions=ngs.get_schema_extensions(), + ) + + +def get_live_schema(): + """ + Return the schema for the current request's active branch, rebuilding it if the + database has changed since this process last built it for that branch. + + netbox-branching has already scoped the active branch for the request (from the + X-NetBox-Branch header, the ``?_branch=`` query param, or the active_branch + cookie; main if none). The signature check, the rebuild, and the query's data + resolution therefore all run against that same branch. + + Returns ``None`` when dynamic models are unavailable (migrations/tests) or if + the very first build fails — the caller then falls back to NetBox's static + schema. + """ + from netbox_custom_objects import CustomObjectsPluginConfig + + if CustomObjectsPluginConfig.should_skip_dynamic_model_creation(): + return None + + branch_key = _active_branch_key() + + try: + signature = cached_schema_signature(branch_key) + except Exception: # noqa: BLE001 - DB hiccup + logger.warning("Could not compute GraphQL schema signature", exc_info=True) + cached = _schema_cache.get(branch_key) + if cached is not None and cached[1] is not None: + # We already have a (possibly slightly stale) schema for this branch. + return cached[1] + # First build for this branch and even the signature query failed. Rather + # than fall back to NetBox's static schema (which has no custom_objects_* + # fields and would reject otherwise-valid queries), make a best-effort first + # build under a sentinel signature so the next request re-checks once the DB + # recovers. + signature = _UNKNOWN_SIGNATURE + + # Hot path: structure unchanged since this process last built for this branch — + # no lock, no rebuild, just return the cached schema. + entry = _schema_cache.get(branch_key) + sig, schema = entry if entry is not None else (None, None) + if schema is not None and sig == signature: + return schema + + # Structure changed (or first build). Single-flight: one thread rebuilds while + # concurrent requests keep serving the existing schema rather than blocking on + # the (potentially expensive) rebuild. On the *first* build there is no schema + # to serve, so a loser must block until the rebuild completes — otherwise it + # would fall back to the custom-object-less static schema and spuriously reject + # custom_objects_* queries that do resolve. + blocking = schema is None + if not _rebuild_lock.acquire(blocking=blocking): + # Another thread is already rebuilding and we have a valid (one signature + # behind) schema to serve in the meantime. + return schema + + try: + # Re-check: a prior holder may have just published a matching schema for + # this branch (always true for the first-build blocker that just waited). + entry = _schema_cache.get(branch_key) + sig, schema = entry if entry is not None else (None, None) + if schema is not None and sig == signature: + return schema + try: + new_schema = build_full_schema() + except Exception: # noqa: BLE001 - never break the endpoint + logger.exception("Failed to rebuild live GraphQL schema") + return schema + _schema_cache[branch_key] = (signature, new_schema) + return new_schema + finally: + _rebuild_lock.release() + + +def reset_cache(): + """Clear the cached schemas, signatures, per-type cache, and build state (tests).""" + global _schema_cache + _schema_cache = {} + + from django.core.cache import cache + + for key in list(_signature_keys_seen): + try: + cache.delete(key) + except Exception: # noqa: BLE001 - cache down: nothing to clear + pass + _signature_keys_seen.clear() + + from .types import clear_type_cache, reset_build_state + + clear_type_cache() + reset_build_state() diff --git a/netbox_custom_objects/graphql/schema.py b/netbox_custom_objects/graphql/schema.py new file mode 100644 index 00000000..b1b627ae --- /dev/null +++ b/netbox_custom_objects/graphql/schema.py @@ -0,0 +1,186 @@ +""" +GraphQL schema contribution for the custom-objects plugin. + +NetBox discovers this module via the plugin's ``graphql_schema`` resource path +and extends its global ``Query`` with every class in the exported ``schema`` +list. :func:`build_query_classes` builds a single ``Query`` class exposing two +fields per custom object type — ```` (single object by ``id``) and +``_list`` (filtered, paginated list) — mirroring NetBox's own per-model +GraphQL fields. + +The module-level ``schema`` export is intentionally empty: the live schema in +:mod:`netbox_custom_objects.graphql.live` (installed via the view patch in +``__init__.py``) rebuilds the custom-object query per request from the current +database, so runtime-created types appear without a restart. A statically built +query here would be immediately shadowed by that live rebuild. ``live`` reuses +:func:`build_query_classes`, so both paths share one implementation. +""" + +import logging +from typing import List + +import strawberry +import strawberry_django + +from .types import build_object_type, clear_type_cache, graphql_safe_name, reset_build_state, set_cot_map + +logger = logging.getLogger("netbox_custom_objects.graphql") + +# All custom-object root query fields are namespaced with this prefix so they +# cannot collide with NetBox's own (or another plugin's) root query fields — every +# plugin's query class is mixed into the single global ``Query`` type, so bare, +# slug-derived names like ``site``/``group`` would otherwise be shadowed by core. +# The prefix mirrors the ``custom_objects_`` table-naming convention. +QUERY_FIELD_PREFIX = "custom_objects_" + + +def _query_field_name(custom_object_type, used_names): + """ + Derive a GraphQL-safe, unique field name from a custom object type's slug. + + The name is namespaced with :data:`QUERY_FIELD_PREFIX` (see above) so it never + collides with core/plugin root query fields. GraphQL names must match + ``[_A-Za-z][_0-9A-Za-z]*``; slugs may contain hyphens. Collisions among custom + object types (after sanitisation) are disambiguated with the type id. + """ + slug = graphql_safe_name((custom_object_type.slug or "").lower()) + # The prefix guarantees a valid leading character, so no digit/empty guard is + # needed on the slug portion. + base = f"{QUERY_FIELD_PREFIX}{slug}" + + def _taken(candidate): + # Reserve the singular field name *and* its ``_list`` companion together. + # Checking both prevents one type's list field from silently colliding + # with another type's singular field — e.g. slug 'foo' yields foo/foo_list + # while slug 'foo-list' sanitises to foo_list/foo_list_list, and the bare + # 'foo_list' would otherwise clobber the first type's list field. + return candidate in used_names or f"{candidate}_list" in used_names + + name = base + if _taken(name): + # Disambiguate with the (unique) type id. The disambiguated name can + # itself collide with one already reserved — another type whose slug + # happens to end in this id — so keep extending until both the singular + # name and its ``_list`` companion are genuinely free. Without this loop + # the colliding field would silently overwrite the earlier type's field. + name = f"{base}_{custom_object_type.id}" + counter = 2 + while _taken(name): + name = f"{base}_{custom_object_type.id}_{counter}" + counter += 1 + used_names.add(name) + used_names.add(f"{name}_list") + return name + + +def build_query_classes(): + """ + Build the list of Strawberry query classes contributed to NetBox's schema. + + Returns an empty list when dynamic models are unavailable (during + migrations, tests, or before migrations have been applied) or when no custom + object types are defined — extending NetBox's Query with an empty/invalid + class is avoided. + + Called on every live rebuild (:mod:`netbox_custom_objects.graphql.live`); the + module-level ``schema`` export below deliberately does not call it (see the + module docstring). The returned class carries a ``_nco_query`` marker so live + rebuilds can identify and replace a previously contributed instance. + """ + # Import lazily to avoid import-time side effects and circular imports. + # The skip-check must run *before* importing models: during migrations and + # the test run the app registry may not be ready, and importing models then + # raises "model isn't in an application in INSTALLED_APPS". + from netbox_custom_objects import CustomObjectsPluginConfig + + if CustomObjectsPluginConfig.should_skip_dynamic_model_creation(): + return [] + + from django.db.models import Prefetch + + from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField + + # Start each rebuild from an empty per-type cache. The cache is keyed by + # (cot id, cache_timestamp), but a type that embeds another COT's type via a + # relationship field does NOT get its own cache_timestamp bumped when the + # referenced COT changes — so a cached entry could embed a stale child type. + # Clearing here makes every type fresh per rebuild (rebuilds only happen on an + # actual structural change), while the cache still memoizes within this single + # rebuild pass so shared and recursive references reuse one built type. + # NOTE: this clear+repopulate of the process-global cache is why rebuilds must + # be serialised — concurrent rebuilds (e.g. different branches) would race here. + # live.get_live_schema holds _rebuild_lock around the whole rebuild to enforce it. + clear_type_cache() + # Also drop any in-progress build state (stack / cycle taint) leaked by an + # exception during a previous rebuild on this pooled thread, so it can't + # suppress caching or corrupt cycle detection on this rebuild. + reset_build_state() + + try: + # Preload every type's fields once, with each field's related type(s) + # joined/prefetched, so the per-type build issues no per-field queries: + # without this the inner loop hits the DB for each type's fields and again + # for each field's related object type(s) — O(N + N·M) queries per rebuild. + fields_qs = ( + CustomObjectTypeField.objects + .select_related("related_object_type") + .prefetch_related("related_object_types") + ) + custom_object_types = list( + CustomObjectType.objects.prefetch_related(Prefetch("fields", queryset=fields_qs)) + ) + except Exception: # noqa: BLE001 - DB may be unavailable at import time + logger.debug("Could not load custom object types for GraphQL schema", exc_info=True) + return [] + + # Register the preloaded types by pk so a relationship field pointing at another + # custom object resolves its target from the prefetched instance instead of + # re-querying it (build_object_type then reuses the per-rebuild type cache). + set_cot_map({cot.pk: cot for cot in custom_object_types}) + + annotations = {} + attrs = {} + used_names = set() + + for cot in custom_object_types: + try: + gql_type = build_object_type(cot) + except Exception: # noqa: BLE001 - never break the whole schema for one type + logger.warning( + "Failed to build GraphQL type for custom object type %r (id=%s); skipping", + cot.name, + cot.id, + exc_info=True, + ) + continue + if gql_type is None: + continue + + field_name = _query_field_name(cot, used_names) + list_name = f"{field_name}_list" + + annotations[field_name] = gql_type + attrs[field_name] = strawberry_django.field() + annotations[list_name] = List[gql_type] + attrs[list_name] = strawberry_django.field() + + if not attrs: + return [] + + attrs["__annotations__"] = annotations + # The GraphQL type name must be "Query": strawberry-django only attaches the + # single-object ``id`` lookup argument to fields whose origin type is named + # "Query" (see strawberry_django.filters: ``is_root_query``). NetBox names + # every per-app query class "Query" for the same reason; our contributed class + # is mixed into NetBox's real Query as a base, so it must do likewise. + query_cls = strawberry.type(type("CustomObjectsQuery", (), attrs), name="Query") + # Marker so live schema rebuilds can find and replace a stale instance of + # this class among NetBox's Query bases. + query_cls._nco_query = True + return [query_cls] + + +# Empty by design — the live schema (graphql/live.py) owns per-request assembly so +# runtime-created types appear without a restart. Must remain a list: NetBox calls +# ``.extend`` on it when registering plugin GraphQL schemas. +schema = [] diff --git a/netbox_custom_objects/graphql/types.py b/netbox_custom_objects/graphql/types.py new file mode 100644 index 00000000..33e89cd0 --- /dev/null +++ b/netbox_custom_objects/graphql/types.py @@ -0,0 +1,565 @@ +""" +Dynamic GraphQL type generation for custom object models. + +Each ``CustomObjectType`` is backed by a real, runtime-generated Django model +(``TableModel``). This module builds a Strawberry GraphQL type for each of +those models so that custom objects can be queried through NetBox's GraphQL API +exactly like first-class NetBox models. + +Because the underlying models are generated at runtime, the GraphQL types are +also generated at runtime. They are rebuilt per-request by +:mod:`netbox_custom_objects.graphql.live` (installed via the view patch in +``__init__.py``) whenever the set of custom object types or their fields changes, +so a type created *after* startup appears without a NetBox restart. + +Scalar fields are mapped to their natural GraphQL scalar types (the mapping +lives on each ``FieldType`` in ``field_types.py``). Object and multi-object +(relationship) fields resolve to the *native* NetBox GraphQL type of their +target — a field pointing at a Site resolves to NetBox's ``SiteType`` and is +fully traversable. Polymorphic relationship fields, which may point at several +model types, resolve to a Strawberry union of those native types (mirroring how +NetBox exposes ``assigned_object``/cable terminations). When a target model has +no registered GraphQL type, the field falls back to a lightweight, uniform +``CustomObjectRelatedObjectType`` stub so the field is never silently dropped. +""" + +import logging +import re +import threading +from typing import Annotated, List, Optional, Union + +import strawberry +import strawberry_django +from core.graphql.mixins import ChangelogMixin +from extras.choices import CustomFieldTypeChoices +from extras.graphql.mixins import TagsMixin +from netbox.graphql.scalars import BigInt +from netbox.graphql.types import BaseObjectType +from strawberry.types import Info + +from netbox_custom_objects.constants import APP_LABEL +from netbox_custom_objects.utilities import extract_cot_id_from_model_name + +logger = logging.getLogger("netbox_custom_objects.graphql") + +__all__ = ( + "CustomObjectObjectType", + "CustomObjectRelatedObjectType", + "build_object_type", + "clear_type_cache", + "graphql_safe_name", + "reset_build_state", + "set_cot_map", +) + +# Per-rebuild memoization of built GraphQL types, keyed by (cot id, +# cache_timestamp). It lets a single schema rebuild reuse one built type across +# the many places that reference it (shared targets and recursive relationships) +# instead of re-running build_object_type for each. It is cleared at the start of +# every rebuild (see schema.build_query_classes): a type that embeds another COT's +# type does not get its own cache_timestamp bumped when that referenced COT +# changes, so persisting entries across rebuilds could serve a stale embedded +# type. clear_type_cache() also lets tests reset it explicitly. +_type_cache = {} +_type_cache_lock = threading.RLock() + +# Per-thread build state. ``cot_stack`` is the stack of COT ids whose GraphQL +# type is being built on the current thread, so that a relationship between two +# custom objects (A -> B -> A) does not recurse forever: a back-reference to a +# type still under construction falls back to the flat stub instead of rebuilding +# it. ``cycle_tainted`` records the COT ids whose build had to use that stub +# fallback for a cyclic edge — those types are intentionally not cached (see +# build_object_type) so the next top-level query rebuilds them and resolves the +# related type fully from that entry point. +_building = threading.local() + +# Lazily-built map of Django model class -> its registered NetBox strawberry +# GraphQL type. The app-defined types are static for the life of the process, so +# this is computed once. +_model_type_registry = None +_registry_lock = threading.RLock() + + +def clear_type_cache(): + """Drop all cached GraphQL types (used by tests).""" + with _type_cache_lock: + _type_cache.clear() + + +def reset_build_state(): + """ + Clear this thread's in-progress build stack and cycle-taint set. + + Called at the start of each schema rebuild (and by tests) so that a stack + frame or taint leaked by an exception during a previous rebuild on this + (pooled) thread cannot suppress caching or corrupt cycle detection on the + next one. ``build_object_type`` only clears a type's taint on the success + path, so a build that raises after a cyclic edge tainted an ancestor would + otherwise leave that taint set on the thread indefinitely. Also drops the + preloaded COT map (see :func:`set_cot_map`). + """ + _building.cot_stack = [] + _building.cycle_tainted = set() + _building.cot_map = None + + +def set_cot_map(cot_map): + """ + Register a ``{pk: CustomObjectType}`` map for the current rebuild. + + ``build_query_classes`` preloads every custom object type once with its fields + (and their related types) prefetched, then registers them here so a relationship + field pointing at another custom object resolves its target from the prefetched + instance — via :func:`_custom_object_graphql_type` — instead of issuing a fresh + query per reference. Cleared by :func:`reset_build_state`. + """ + _building.cot_map = cot_map + + +RELATIONSHIP_TYPES = ( + CustomFieldTypeChoices.TYPE_OBJECT, + CustomFieldTypeChoices.TYPE_MULTIOBJECT, +) + + +@strawberry.type +class CustomObjectRelatedObjectType: + """ + Fallback representation of an object referenced by a relationship field whose + target model has no registered NetBox GraphQL type. + + Most relationship targets resolve to their native GraphQL type and are fully + traversable; this uniform stub is only used when no such type exists, so the + field still exposes the basics rather than disappearing from the schema. + """ + + # BigInt (not int/Int): NetBox primary keys are BigAutoField and can exceed + # the signed 32-bit range of GraphQL's Int. Native relationship types already + # expose their id as BigInt; the fallback stub must match. + id: BigInt + object_type: str + display: str + url: Optional[str] + + +@strawberry.type +class CustomObjectObjectType(ChangelogMixin, TagsMixin, BaseObjectType): + """ + Base GraphQL type for all custom object models. + + ``BaseObjectType`` provides ``display``/``class_type`` and, crucially, + ``get_queryset()`` which enforces NetBox object-level view permissions. + ``ChangelogMixin`` and ``TagsMixin`` add change-log and tag access — both are + supported by the ``CustomObject`` base model. Custom fields are added per + type by :func:`build_object_type`. + """ + + pass + + +def graphql_safe_name(value): + """Replace any character not valid in a GraphQL name with an underscore.""" + return re.sub(r"[^0-9a-zA-Z_]", "_", value or "") + + +def _in_progress_stack(): + stack = getattr(_building, "cot_stack", None) + if stack is None: + stack = [] + _building.cot_stack = stack + return stack + + +def _cycle_tainted_set(): + tainted = getattr(_building, "cycle_tainted", None) + if tainted is None: + tainted = set() + _building.cycle_tainted = tainted + return tainted + + +def _request_user(info): + """Best-effort extraction of the requesting user from the GraphQL info context.""" + request = getattr(getattr(info, "context", None), "request", None) + return getattr(request, "user", None) + + +def _filter_viewable(user, objects): + """ + Return the subset of ``objects`` the user may view, preserving order. + + The top-level query restricts the custom objects themselves, but the objects + reached through their relationship fields are *not* covered by that check, so + each one must be gated here or the field would leak objects the user cannot + see. Used for both single- and multi-object fields so the permission rule + lives in one place; batches the check to one query per distinct model rather + than one ``.exists()`` per object (an N+1 explosion on multi-object fields): + the related objects are grouped by model and each model's permission-restricted + queryset is evaluated once with ``pk__in``. + + The user (anonymous or ``None`` included) is passed straight to the model + manager's ``restrict(user, "view")``, mirroring NetBox's + ``BaseObjectType.get_queryset`` — superuser bypass, ``EXEMPT_VIEW_PERMISSIONS`` + and anonymous handling are all left to ``restrict``. + """ + objects = [obj for obj in objects if obj is not None] + if not objects: + return [] + if getattr(user, "is_superuser", False): + return objects + + # sentinel meaning "model isn't permission-aware → all allowed". + allowed_by_model = {} + by_model = {} + for obj in objects: + by_model.setdefault(type(obj), []).append(obj) + for model, model_objs in by_model.items(): + manager = getattr(model, "_default_manager", None) + if manager is None or not hasattr(manager, "restrict"): + allowed_by_model[model] = None + continue + pks = [obj.pk for obj in model_objs] + allowed_by_model[model] = set( + manager.restrict(user, "view").filter(pk__in=pks).values_list("pk", flat=True) + ) + + return [ + obj + for obj in objects + if (allowed_by_model[type(obj)] is None or obj.pk in allowed_by_model[type(obj)]) + ] + + +def _related_repr(obj): + """Convert a referenced model instance into a ``CustomObjectRelatedObjectType``.""" + if obj is None: + return None + try: + url = obj.get_absolute_url() + except Exception: # noqa: BLE001 - URL resolution is best-effort + url = None + try: + display = str(obj) + except Exception: # noqa: BLE001 - a broken __str__ must not fail the whole field + # Degrade to a stable identifier so one unrenderable object becomes a + # placeholder rather than erroring the entire (possibly multi-object) field; + # pk and _meta are safe even when __str__ raises. + display = f"{obj._meta.label_lower}:{obj.pk}" + return CustomObjectRelatedObjectType( + id=obj.pk, + object_type=obj._meta.label_lower, + display=display, + url=url, + ) + + +def _build_model_type_registry(): + """ + Map every Django model to its registered NetBox strawberry GraphQL type. + + NetBox (and other plugins) declare their per-model types in + ``.graphql.types``. Each such type carries a strawberry-django + definition naming its model, so importing those modules and indexing by model + gives a model -> type lookup that relationship fields use to resolve their + target to a native, traversable type. + """ + from importlib import import_module + + from django.apps import apps as django_apps + from strawberry_django.utils.typing import get_django_definition + + registry = {} + for app_config in django_apps.get_app_configs(): + if app_config.label == APP_LABEL: + # Custom-object types are resolved dynamically, not from a module. + continue + module_name = f"{app_config.name}.graphql.types" + try: + module = import_module(module_name) + except ModuleNotFoundError: + continue + except Exception: # noqa: BLE001 - a broken app module must not break the schema + logger.debug("Could not import %s for the GraphQL type registry", module_name, exc_info=True) + continue + for value in vars(module).values(): + if not isinstance(value, type): + continue + definition = get_django_definition(value) + if definition is None or definition.model is None: + continue + # Only index GraphQL *output* object types. Filters and inputs also + # carry a django definition (and a model), but using one as a field's + # type would break the schema. + sb_definition = getattr(value, "__strawberry_definition__", None) + if sb_definition is None or sb_definition.is_input or sb_definition.is_interface: + continue + registry.setdefault(definition.model, value) + return registry + + +def _get_model_type_registry(): + global _model_type_registry + if _model_type_registry is not None: + return _model_type_registry + with _registry_lock: + if _model_type_registry is None: + _model_type_registry = _build_model_type_registry() + return _model_type_registry + + +def _custom_object_graphql_type(model_name): + """Resolve a custom-object target (``tablemodel``) to its GraphQL type.""" + from netbox_custom_objects.models import CustomObjectType + + cot_id = extract_cot_id_from_model_name(model_name) + if cot_id is None: + return None + # extract_cot_id_from_model_name returns the id as a str; the in-progress + # stack holds ints, so coerce before the membership test or it never matches. + cot_id = int(cot_id) + stack = _in_progress_stack() + if cot_id in stack: + # Back-reference to a type still being built — fall back to the flat stub + # for this edge to avoid infinite recursion, and taint every type currently + # under construction so none of them is cached with this temporary stub + # frozen in (see build_object_type). Tainting only the immediate parent + # (stack[-1]) would still cache the outer types of a cycle longer than two + # (A -> B -> C -> A), permanently freezing the stub into their subtree. + _cycle_tainted_set().update(stack) + return None + # Resolve from the rebuild's preloaded (prefetched) map when available, so a + # cross-COT reference doesn't issue a query per edge; fall back to a lookup for + # direct callers (e.g. tests) that didn't register a map. + cot_map = getattr(_building, "cot_map", None) + cot = cot_map.get(cot_id) if cot_map else None + if cot is None: + cot = CustomObjectType.objects.filter(pk=cot_id).first() + if cot is None: + return None + try: + return build_object_type(cot) + except Exception: # noqa: BLE001 + logger.warning( + "Failed to build related custom-object GraphQL type for %r", model_name, exc_info=True + ) + return None + + +def _graphql_type_for_content_type(content_type): + """Return the native strawberry GraphQL type for ``content_type``, or ``None``.""" + model = content_type.model_class() + if model is None: + return None + if content_type.app_label == APP_LABEL: + return _custom_object_graphql_type(content_type.model) + return _get_model_type_registry().get(model) + + +def _field_target_content_types(field): + """Return the ContentType(s) a relationship field may point at.""" + if field.is_polymorphic: + return list(field.related_object_types.all()) + if field.related_object_type_id: + return [field.related_object_type] + return [] + + +def _resolve_relationship_members(field): + """ + Return ``(members, native_models)`` for a relationship field. + + ``members`` is the list of GraphQL types the field can resolve to — the + native type of each target that has one, plus the flat stub when at least one + target has no native type (or there are no targets at all). ``native_models`` + is the set of Django model classes that resolve to a native type, used at + resolve time to decide whether to return the raw instance or wrap it. + """ + members = [] + native_models = set() + needs_stub = False + for content_type in _field_target_content_types(field): + gql_type = _graphql_type_for_content_type(content_type) + model = content_type.model_class() + if gql_type is not None and model is not None: + if gql_type not in members: + members.append(gql_type) + native_models.add(model) + else: + needs_stub = True + if needs_stub or not members: + if CustomObjectRelatedObjectType not in members: + members.append(CustomObjectRelatedObjectType) + return members, native_models + + +def _relationship_union_name(field): + """A schema-unique, GraphQL-safe name for a polymorphic field's union type.""" + base = graphql_safe_name(field.name) + return f"CustomObject{field.custom_object_type_id}_{base}_Related" + + +def _relationship_annotation(members, is_list, union_name): + """Build the resolver return annotation from the resolved member types.""" + if len(members) == 1: + inner = members[0] + else: + inner = Annotated[Union[tuple(members)], strawberry.union(union_name)] + return List[inner] if is_list else Optional[inner] + + +def _coerce_related(obj, native_models): + """Return ``obj`` itself if its model resolves natively, else the flat stub.""" + if type(obj) in native_models or any(isinstance(obj, model) for model in native_models): + return obj + return _related_repr(obj) + + +def _make_relationship_resolver(field): + """ + Build a resolver for an OBJECT or MULTIOBJECT relationship field. + + The resolver returns the referenced object(s) as their native GraphQL + type(s) (or the flat stub for targets without one), filtered to those the + requesting user may view. + """ + field_name = field.name + is_list = field.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT + + members, native_models = _resolve_relationship_members(field) + if not members: + return None + annotation = _relationship_annotation(members, is_list, _relationship_union_name(field)) + + # Query-optimisation hints read by NetBox's DjangoOptimizerExtension. A + # non-polymorphic OBJECT field is a ForeignKey (select_related); a polymorphic + # one is a GenericForeignKey (prefetch_related only). A non-polymorphic + # MULTIOBJECT field is a real M2M (prefetch_related); a polymorphic one is a + # custom descriptor that can't be prefetched. + if is_list: + hint = {} if field.is_polymorphic else {"prefetch_related": field_name} + description = f"Related objects referenced by '{field_name}'" + else: + hint = {"prefetch_related": field_name} if field.is_polymorphic else {"select_related": field_name} + description = f"Related object referenced by '{field_name}'" + + def resolver(self, info: Info): + user = _request_user(info) + value = getattr(self, field_name, None) + if is_list: + if value is None: + return [] + try: + related = list(value.all()) + except Exception: # noqa: BLE001 - never let one field break the query + logger.warning( + "Failed to resolve multi-object GraphQL field %r", field_name, exc_info=True + ) + return [] + return [ + _coerce_related(obj, native_models) + for obj in _filter_viewable(user, related) + ] + if value is None: + return None + viewable = _filter_viewable(user, [value]) + if not viewable: + return None + return _coerce_related(viewable[0], native_models) + + resolver.__annotations__ = {"info": Info, "return": annotation} + return strawberry_django.field(description=description, **hint)(resolver) + + +def _scalar_annotation_for(field_type): + """Return the GraphQL scalar annotation for a field type, or ``None``.""" + from netbox_custom_objects.field_types import FIELD_TYPE_CLASS + + field_type_cls = FIELD_TYPE_CLASS.get(field_type) + if field_type_cls is None: + return None + return field_type_cls().get_graphql_annotation() + + +def build_object_type(custom_object_type): + """ + Build (or return ``None`` on failure) a Strawberry type for a single + ``CustomObjectType``. + + The returned class is a ``strawberry_django.type`` bound to the runtime + model, with one GraphQL field per custom field plus the inherited base + fields (id, display, tags, changelog, created, last_updated). + """ + model = custom_object_type.get_model() + if model is None: + return None + + # Reuse the already-built type when this COT's structure is unchanged (see + # _type_cache). cache_timestamp moves on every structural change, so this can + # never serve a type that no longer matches the model. + cache_key = (custom_object_type.id, custom_object_type.cache_timestamp) + with _type_cache_lock: + cached = _type_cache.get(cache_key) + if cached is not None: + return cached + + stack = _in_progress_stack() + stack.append(custom_object_type.id) + try: + gql_type = _build_object_type(custom_object_type, model) + finally: + stack.pop() + + # A type whose build had to break a relationship cycle with the flat stub (a + # related custom object was still under construction) must not be cached: the + # stub edge is an artefact of *this* build order, and caching it would freeze + # that degraded edge forever. Leaving it uncached lets a later top-level + # query rebuild it and resolve the related type fully from that entry point. + tainted = _cycle_tainted_set() + if custom_object_type.id in tainted: + tainted.discard(custom_object_type.id) + return gql_type + + with _type_cache_lock: + _type_cache[cache_key] = gql_type + # Bound growth: drop now-superseded entries for this COT. + for key in [k for k in _type_cache if k[0] == cache_key[0] and k != cache_key]: + del _type_cache[key] + return gql_type + + +def _build_object_type(custom_object_type, model): + """Construct a fresh Strawberry type for ``custom_object_type`` (uncached).""" + type_name = f"{model.__name__}Type" + + namespace = { + "__doc__": f"Custom object type '{custom_object_type.name}'.", + "__annotations__": {}, + } + + for field in custom_object_type.fields.all(): + field_name = field.name + if field.type in RELATIONSHIP_TYPES: + resolver = _make_relationship_resolver(field) + if resolver is not None: + namespace[field_name] = resolver + continue + + annotation = _scalar_annotation_for(field.type) + if annotation is None: + logger.debug( + "Skipping custom field %r of unsupported GraphQL type %r", + field_name, + field.type, + ) + continue + # Every custom field is nullable at the database level. + namespace["__annotations__"][field_name] = Optional[annotation] + + cls = type(type_name, (CustomObjectObjectType,), namespace) + + return strawberry_django.type( + model, + name=type_name, + fields=["id", "created", "last_updated"], + pagination=True, + )(cls) diff --git a/netbox_custom_objects/migrations/0015_customobjecttype_config_context_enabled.py b/netbox_custom_objects/migrations/0015_customobjecttype_config_context_enabled.py new file mode 100644 index 00000000..e3a3b15b --- /dev/null +++ b/netbox_custom_objects/migrations/0015_customobjecttype_config_context_enabled.py @@ -0,0 +1,18 @@ +# Generated by Django 6.0.6 on 2026-06-30 22:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("netbox_custom_objects", "0014_fix_mixed_case_field_names"), + ] + + operations = [ + migrations.AddField( + model_name="customobjecttype", + name="config_context_enabled", + field=models.BooleanField(default=False), + ), + ] diff --git a/netbox_custom_objects/migrations/0016_widen_integer_columns.py b/netbox_custom_objects/migrations/0016_widen_integer_columns.py new file mode 100644 index 00000000..9e208eb5 --- /dev/null +++ b/netbox_custom_objects/migrations/0016_widen_integer_columns.py @@ -0,0 +1,67 @@ +""" +Widen existing integer custom-object columns from 32-bit to 64-bit (bigint). + +Integer fields previously mapped to Django's IntegerField, i.e. a 32-bit signed +PostgreSQL ``integer`` column (max 2_147_483_647). They now map to BigIntegerField +(``bigint``). New columns are created as bigint automatically by the schema editor, +but columns on already-created custom_objects_* tables must be widened in place. + +``integer -> bigint`` is a lossless widening (every int32 value fits in int64), so +no USING clause or data transformation is needed. The conversion rewrites the table +under an ACCESS EXCLUSIVE lock; this is fine for typical custom-object table sizes. + +The reverse is intentionally a no-op: narrowing bigint back to integer could fail or +lose data for any value outside the 32-bit range. + +See issue #532. +""" + +from django.db import migrations + + +def widen_integer_columns(apps, schema_editor): + """ALTER every integer-typed custom-object column to bigint, in place.""" + CustomObjectTypeField = apps.get_model("netbox_custom_objects", "CustomObjectTypeField") + + # Drive off field metadata (not blind introspection of every custom_objects_* + # column) so we only touch user integer fields, never base-model columns + # inherited from NetBox mixins. After migration 0014 all field names are + # lowercase and the scalar column name equals the field name exactly. + integer_fields = CustomObjectTypeField.objects.filter(type="integer") + + with schema_editor.connection.cursor() as cursor: + for field in integer_fields: + table_name = f"custom_objects_{field.custom_object_type_id}" + column_name = field.name + + # Idempotent + safe: only act on a column that exists and is still a + # 32-bit integer. A no-op on fresh installs (already bigint) and on + # partial re-runs. + cursor.execute( + """ + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = %s + AND column_name = %s + AND data_type = 'integer' + """, + [table_name, column_name], + ) + if cursor.fetchone(): + # quote_name() both identifiers (consistent with the %s-parameterised + # check above) so a field name containing a quote can't break out. + cursor.execute( + f"ALTER TABLE {schema_editor.quote_name(table_name)} " + f"ALTER COLUMN {schema_editor.quote_name(column_name)} TYPE bigint" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("netbox_custom_objects", "0015_customobjecttype_config_context_enabled"), + ] + + operations = [ + migrations.RunPython(widen_integer_columns, migrations.RunPython.noop), + ] diff --git a/netbox_custom_objects/mixin_migration.py b/netbox_custom_objects/mixin_migration.py index 14e99a8f..fe46d336 100644 --- a/netbox_custom_objects/mixin_migration.py +++ b/netbox_custom_objects/mixin_migration.py @@ -157,6 +157,9 @@ def heal_cot(cot, verbosity=1, dry_run=False): try: with connection.schema_editor() as editor: + # Flush pending DEFERRABLE FK trigger events before ALTER TABLE; + # PostgreSQL rejects ADD COLUMN when deferred triggers are pending. + editor.execute('SET CONSTRAINTS ALL IMMEDIATE') editor.add_field(model, field) added.append(col_name) if verbosity >= 1: diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index a31bc4f8..847f14bf 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1,3 +1,4 @@ +import contextvars import decimal import logging import re @@ -7,6 +8,7 @@ from packaging.version import Version, InvalidVersion import django_filters +from core.choices import ObjectChangeActionChoices from core.models import ObjectType, ObjectChange from core.models.object_types import ObjectTypeManager from django.apps import apps @@ -16,9 +18,10 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import FieldDoesNotExist from django.core.validators import RegexValidator, ValidationError -from django.db import connection, IntegrityError, models, transaction +from django.db import DEFAULT_DB_ALIAS, connection, connections, IntegrityError, models, transaction from django.db.utils import OperationalError, ProgrammingError from django.db.models import Q +from django.db.models.fields.related import ForeignKey, ManyToManyField from django.db.models.functions import Lower from django.db.models.signals import m2m_changed, pre_delete, post_save from django.dispatch import receiver @@ -31,15 +34,17 @@ CustomFieldUIEditableChoices, CustomFieldUIVisibleChoices, ) -from extras.models import CustomField +from extras.models import ConfigContext, ConfigContextModel, CustomField from extras.models.customfields import SEARCH_TYPES -from extras.utils import run_validators +from extras.utils import is_taggable, run_validators from netbox.config import get_config from netbox.models import ChangeLoggedModel, NetBoxModel +from netbox.models.mixins import OwnerMixin from netbox.models.features import ( BookmarksMixin, ChangeLoggingMixin, CloningMixin, + ContactsMixin, CustomLinksMixin, CustomValidationMixin, EventRulesMixin, @@ -53,21 +58,31 @@ from netbox.registry import registry from netbox.search import SearchIndex from utilities import filters -from utilities.data import get_config_value_ci +from utilities.data import deepmerge, get_config_value_ci from utilities.datetime import datetime_from_timestamp from utilities.object_types import object_type_name from utilities.querysets import RestrictedQuerySet +from utilities.serialization import deserialize_object as _deserialize_object from utilities.string import title from utilities.validators import validate_regex -from netbox_custom_objects.choices import ObjectFieldOnDeleteChoices -from netbox_custom_objects.constants import APP_LABEL, RESERVED_FIELD_NAMES +from netbox_custom_objects.choices import (CustomObjectFieldTypeChoices, + ObjectFieldOnDeleteChoices) +from netbox_custom_objects.constants import ( + APP_LABEL, + CONFIG_CONTEXT_DIMENSION_FIELDS, + RESERVED_FIELD_NAMES, +) from netbox_custom_objects.field_types import ( FIELD_TYPE_CLASS, LazyForeignKey, safe_table_name, PolymorphicObjectReverseDescriptor, PolymorphicMultiObjectReverseDescriptor, ) from netbox_custom_objects.jobs import ReindexCustomObjectTypeJob -from netbox_custom_objects.utilities import _suppress_clear_cache, extract_cot_id_from_model_name, generate_model +from netbox_custom_objects.utilities import ( + _suppress_clear_cache, + extract_cot_id_from_model_name, + generate_model, +) logger = logging.getLogger(__name__) @@ -78,18 +93,502 @@ class UniquenessConstraintTestError(Exception): pass -def _table_exists(table_name): - """Return True if *table_name* exists in the current database.""" - return table_name in connection.introspection.table_names() +def _table_exists(table_name, conn=None): + """Return True if *table_name* exists in the database reachable via *conn*. + + Defaults to the global ``connection`` (main schema). When the caller is + operating inside a branch context, pass the branch's connection so the + lookup runs against the active branch's PostgreSQL schema. + """ + if conn is None: + conn = connection + return table_name in conn.introspection.table_names() USER_TABLE_DATABASE_NAME_PREFIX = "custom_objects_" +# Per-context storage for CO field values deferred during squash merge. +# Using ContextVar instead of a class-level dict so that concurrent merges +# (different threads or coroutines) each get an isolated copy. +# Shape: {db_table: {co_pk: {'data': {field_name: value}}}} +_deferred_co_field_data: contextvars.ContextVar[dict | None] = contextvars.ContextVar( + '_deferred_co_field_data', default=None +) + +# Sidecar key listing polymorphic-M2M fields in a serialized CO dict, as +# ``[{'name': ..., 'pk': ...}, ...]``. ``pk`` lets the squash dependency +# resolver in ``branching.py`` produce a CO → field CREATE edge without +# looking the field up in main (it isn't there yet during a branch-only +# merge). ``name`` lets ``deserialize_object`` map rows to through tables. +POLY_M2M_SIDECAR_KEY = '__nco_poly_m2m_fields__' + +# Serializes the save/restore of TM.post_through_setup in get_model(). The +# patch needs to be in place during type() class creation, so the lock has to +# span the whole generate_model() — that serialises unrelated COT generations. +# Acceptable for now: the bottleneck is bounded to startup and squash merges. +_taggable_manager_patch_lock = threading.Lock() + + +def _apply_poly_m2m_rows(schema_conn, through_table, co_pk, rows): + """Insert polymorphic M2M *rows* into *through_table* on *schema_conn*, + set-style (clear existing for *co_pk* first). ContentType resolved by + natural key, also via *schema_conn*. + + *through_table* originates from ``CustomObjectTypeField.through_table_name``, + which is derived from validated identifiers (the COT id and a field name + matching ``^[a-z0-9]+(_[a-z0-9]+)*$``) — safe to interpolate directly + into SQL. + """ + alias = schema_conn.alias + inserted = 0 + dropped = 0 + with schema_conn.cursor() as cursor: + cursor.execute( + f'DELETE FROM "{through_table}" WHERE source_id = %s', [co_pk], + ) + for row in rows: + ct_label = row.get('content_type') + obj_id = row.get('object_id') + if not ct_label or obj_id is None: + dropped += 1 + continue + try: + app_label, model_name = ct_label.split('.', 1) + ct = ContentType.objects.using(alias).get( + app_label=app_label, model=model_name, + ) + except (ValueError, ContentType.DoesNotExist) as exc: + logger.warning( + 'poly M2M replay: ct %r unresolved (%s) for %s pk=%s', + ct_label, exc, through_table, co_pk, + ) + dropped += 1 + continue + cursor.execute( + f'INSERT INTO "{through_table}" ' + '(source_id, content_type_id, object_id) VALUES (%s, %s, %s)', + [co_pk, ct.pk, obj_id], + ) + inserted += 1 + # All rows dropped — flag it. A CO with poly-M2M data should land with at + # least one row; zero inserts means the replay silently lost data. + if rows and inserted == 0 and dropped > 0: + logger.warning( + 'poly M2M replay: all %d row(s) dropped for %s pk=%s — ' + 'CO will land with empty %s', dropped, through_table, co_pk, through_table, + ) + + +def _get_schema_connection(): + """Active branch's connection if any, else the default — so DDL targets the right schema.""" + try: + from netbox_branching.contextvars import active_branch + branch = active_branch.get() + if branch is not None: + return connections[branch.connection_name] + except ImportError: + pass + return connection + + +def _historical_names_for_field(field_pk): + """All names this field has ever held, from its ObjectChange UPDATE history. + + Used so a deferred CO entry recorded under the field's old name still + matches when the field is created in the target schema under its new name. + """ + names = set() + rows = ObjectChange.objects.filter( + changed_object_type__app_label='netbox_custom_objects', + changed_object_type__model='customobjecttypefield', + changed_object_id=field_pk, + action=ObjectChangeActionChoices.ACTION_UPDATE, + ).values_list('prechange_data', 'postchange_data') + for pre, post in rows: + for blob in (pre, post): + if not blob: + continue + n = blob.get('name') + if n: + names.add(n) + return names + + +def _apply_deferred_co_field(field_instance): + """Apply deferred CO field values via raw UPDATE after the column is added. + + Squash merge fix: when a CO CREATE replays before its field's CREATE, the + field values stash in ``_deferred_co_field_data`` (shape: + ``{db_table: {co_pk: {'data': {field_name: value}}}}``); when the field + finally lands we UPDATE those rows. TYPE_OBJECT data key is ``{name}`` + but the column is ``{name}_id``; TYPE_MULTIOBJECT has no parent-table + column so it's skipped. Historical names (rename history) are accepted + as matches so a renamed field still picks up its pre-rename data. + """ + # No deferred data at all — fast path. + deferred = _deferred_co_field_data.get() + if not deferred: + return + + cot = field_instance.custom_object_type + table_name = cot.get_database_table_name() + per_table = deferred.get(table_name) + if not per_table: + return + + # M2M has no column on the main table — nothing to UPDATE. + if field_instance.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + return + + # For TYPE_OBJECT the data key is the field name but the DB column ends with _id. + if field_instance.type == CustomFieldTypeChoices.TYPE_OBJECT: + col_name = f'{field_instance.name}_id' + else: + col_name = field_instance.name + + # Candidate data keys: current name + all historical names. pk may be None + # when called before the field row is persisted; in that case we skip the + # history lookup and only match by current name. + candidate_keys = {field_instance.name} + if field_instance.pk is not None: + candidate_keys.update(_historical_names_for_field(field_instance.pk)) + + schema_conn = _get_schema_connection() + + with schema_conn.cursor() as cursor: + for co_pk, entry in per_table.items(): + data = entry['data'] + # Distinguish "key absent" from "key present with NULL" — explicit + # None is a legitimate write and must reach the column. + matched = next((k for k in candidate_keys if k in data), None) + if matched is None: + continue + value = data[matched] + # table_name / col_name come from validated identifiers + # (^[a-z0-9_]+$ on field.name) — safe to interpolate. + cursor.execute( + f'UPDATE "{table_name}" SET "{col_name}" = %s WHERE id = %s', + [value, co_pk], + ) + # Pop consumed keys immediately so a mid-loop failure leaves + # un-applied rows intact for retry but doesn't re-apply + # rows that already succeeded. + for k in candidate_keys: + data.pop(k, None) + + exhausted = [pk for pk, entry in per_table.items() if not entry['data']] + for pk in exhausted: + del per_table[pk] + if not per_table: + del deferred[table_name] + if not deferred: + _deferred_co_field_data.set(None) + + +def _schema_add_field(fi, model, schema_editor, schema_conn): + """``add_field`` against *schema_conn*; idempotent (skips if column exists). + + Creates the through table for MULTIOBJECT. Deferred CO field data is NOT + applied here — call ``_apply_deferred_co_field`` separately after. + """ + ft = FIELD_TYPE_CLASS[fi.type]() + mf = ft.get_model_field(fi) + mf.contribute_to_class(model, fi.name) + + with schema_conn.cursor() as cursor: + existing_cols = { + col.name + for col in schema_conn.introspection.get_table_description(cursor, model._meta.db_table) + } + if mf.column in existing_cols: + logger.debug('_schema_add_field: %r already exists on %s, skipping', mf.column, model._meta.db_table) + return + + # LazyForeignKey starts with a string remote_field.model. Django's + # lazy_related_operation fires immediately when the target is in + # apps.all_models, but tearDown() cleanup between tests can remove + # the target model from the registry. Resolve it directly here — + # bypassing the app-config's skip guard — so that schema_editor + # .add_field() always sees a model class, not a string. + if isinstance(mf, LazyForeignKey) and isinstance(mf.remote_field.model, str): + _app_label, _model_name = mf._to_model_name.rsplit('.', 1) + _cot_id_str = extract_cot_id_from_model_name(_model_name.lower()) + if _cot_id_str is not None: + try: + _cot = CustomObjectType.objects.get(pk=int(_cot_id_str)) + _actual = _cot.get_model() + mf.remote_field.model = _actual + mf.to = _actual + except (CustomObjectType.DoesNotExist, OperationalError, ProgrammingError): + logger.warning( + "Could not resolve LazyForeignKey target %r before add_field; " + "schema_editor.add_field may fail", + mf._to_model_name, + ) + + # Flush any pending DEFERRABLE FK trigger events (e.g. owner_id from OwnerMixin) + # before ALTER TABLE; PostgreSQL rejects ADD COLUMN when deferred triggers are pending. + schema_editor.execute('SET CONSTRAINTS ALL IMMEDIATE') + schema_editor.add_field(model, mf) + if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + ft.create_m2m_table(fi, model, fi.name, schema_conn=schema_conn) + + +def _schema_remove_field(fi, model, schema_editor, schema_conn=None, existing_tables=None): + """``remove_field`` against *schema_conn*; idempotent. + + For MULTIOBJECT, drops the through table (skipped if already gone). + For scalar fields, flushes DEFERRABLE FK triggers before ALTER TABLE so + PostgreSQL doesn't reject the call with "pending trigger events". + *existing_tables* optionally short-circuits the per-call introspection. + """ + ft = FIELD_TYPE_CLASS[fi.type]() + mf = ft.get_model_field(fi) + mf.contribute_to_class(model, fi.name) + + if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + through_table = fi.through_table_name + if existing_tables is None: + conn = schema_conn if schema_conn is not None else connection + with conn.cursor() as cursor: + existing_tables = set(conn.introspection.table_names(cursor)) + if through_table in existing_tables: + through_meta = type( + 'Meta', (), + {'db_table': through_table, 'app_label': APP_LABEL, 'managed': True}, + ) + temp_name = f'_TempThrough_{through_table}' + through_model = type( + temp_name, + (models.Model,), + {'Meta': through_meta, '__module__': 'netbox_custom_objects.models'}, + ) + try: + schema_editor.delete_model(through_model) + finally: + # ModelBase.__new__ registered the temp class in apps.all_models; + # drop it so repeated remove/re-add cycles don't leak entries. + apps.all_models.get(APP_LABEL, {}).pop(temp_name.lower(), None) + # M2M has no column on the parent table — nothing further to remove. + return + + # Flush any pending DEFERRABLE FK trigger events before ALTER TABLE; + # otherwise PostgreSQL raises "pending trigger events" when removing a FK field. + schema_editor.execute('SET CONSTRAINTS ALL IMMEDIATE') + schema_editor.remove_field(model, mf) + + +def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, existing_tables=None): + """``alter_field`` from *old_fi* to *new_fi*; idempotent across replays. + + M2M renames go through ``_rename_or_create_m2m_through`` first. When + neither old nor new column exists (rename conflict — branch A→X vs main + A→Y), looks up the live field record to find the actual current column. + MULTIOBJECT↔scalar type changes are unsupported — caller must remove+add. + """ + old_is_m2m = old_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT + new_is_m2m = new_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT + + if old_is_m2m != new_is_m2m: + logger.warning( + '_schema_alter_field: skipping unsupported type change %r→%r on %s ' + '(MULTIOBJECT ↔ scalar changes require remove+add, not alter)', + old_fi.type, new_fi.type, model._meta.db_table, + ) + return + + old_mf = FIELD_TYPE_CLASS[old_fi.type]().get_model_field(old_fi) + new_mf = FIELD_TYPE_CLASS[new_fi.type]().get_model_field(new_fi) + old_mf.contribute_to_class(model, old_fi.name) + new_mf.contribute_to_class(model, new_fi.name) + + # M2M has no parent-table column — schema work happens on the through table. + if new_is_m2m: + if old_fi.name != new_fi.name: + _rename_or_create_m2m_through( + old_fi, new_fi, model, schema_editor, schema_conn, existing_tables, + ) + return + + with schema_conn.cursor() as cursor: + existing_cols = { + col.name + for col in schema_conn.introspection.get_table_description(cursor, model._meta.db_table) + } + if old_mf.column not in existing_cols: + if new_mf.column in existing_cols: + logger.debug( + '_schema_alter_field: %r already renamed to %r on %s, skipping', + old_mf.column, new_mf.column, model._meta.db_table, + ) + return + # Both source and target columns absent → independent rename in this + # schema; look up the live column and converge on the merge target. + logger.warning( + '_schema_alter_field: rename conflict on %s — neither %r nor %r ' + 'exists; resolving via live field pk=%d', + model._meta.db_table, old_mf.column, new_mf.column, new_fi.pk, + ) + try: + live_fi = CustomObjectTypeField.objects.using(schema_conn.alias).get(pk=new_fi.pk) + except CustomObjectTypeField.DoesNotExist: + logger.debug( + '_schema_alter_field: field pk=%d not found in %s; skipping', + new_fi.pk, schema_conn.alias, + ) + return + live_mf = FIELD_TYPE_CLASS[live_fi.type]().get_model_field(live_fi) + live_mf.contribute_to_class(model, live_fi.name) + if live_mf.column not in existing_cols: + logger.debug( + '_schema_alter_field: live column %r also absent on %s; skipping', + live_mf.column, model._meta.db_table, + ) + return + schema_editor.alter_field(model, live_mf, new_mf) + return + + schema_editor.alter_field(model, old_mf, new_mf) + + +def _rename_or_create_m2m_through(old_fi, new_fi, model, schema_editor, schema_conn, existing_tables): + """Rename the through-table for a renamed M2M field, or create the new one + if the old table is absent (sync/merge against a schema that never had it). + """ + old_through = old_fi.through_table_name + new_through = new_fi.through_table_name + + tables = existing_tables + if tables is None: + with schema_conn.cursor() as cursor: + tables = schema_conn.introspection.table_names(cursor) + + if old_through in tables: + old_through_meta = type( + 'Meta', (), + {'db_table': old_through, 'app_label': APP_LABEL, 'managed': True}, + ) + temp_name = f'_TempOldThrough_{old_through}' + old_through_model = generate_model( + temp_name, + (models.Model,), + { + '__module__': 'netbox_custom_objects.models', + 'Meta': old_through_meta, + 'id': models.AutoField(primary_key=True), + 'source': models.ForeignKey( + model, on_delete=models.CASCADE, db_column='source_id', related_name='+', + ), + 'target': models.ForeignKey( + model, on_delete=models.CASCADE, db_column='target_id', related_name='+', + ), + }, + ) + try: + schema_editor.alter_db_table(old_through_model, old_through, new_through) + finally: + # generate_model() registered the temp class in apps.all_models; + # drop it so repeated renames don't leak entries. + apps.all_models.get(APP_LABEL, {}).pop(temp_name.lower(), None) + else: + # Old through table absent — create the new one from scratch + ft = FIELD_TYPE_CLASS[new_fi.type]() + ft.create_m2m_table(new_fi, model, new_fi.name, schema_conn=schema_conn) + + +def _translate_renamed_field_name(cot, attr, rename_map=None): + """Resolve *attr* to the current name of one of *cot*'s fields via its + ObjectChange rename history. Returns ``None`` on ambiguity (caller falls + back to raw key) so we never silently overwrite the wrong column. Pass + *rename_map* (built once via ``_build_rename_map``) to skip the DB query. + """ + if rename_map is not None: + return rename_map.get(attr) + candidate_pks = set( + ObjectChange.objects.filter( + changed_object_type__app_label='netbox_custom_objects', + changed_object_type__model='customobjecttypefield', + action=ObjectChangeActionChoices.ACTION_UPDATE, + ).filter( + Q(postchange_data__name=attr) | Q(prechange_data__name=attr) + ).values_list('changed_object_id', flat=True) + ) + if not candidate_pks: + return None + fields = list( + cot.fields.filter(pk__in=candidate_pks).values_list('name', flat=True) + ) + if len(fields) == 1: + return fields[0] + return None + + +def _build_rename_map(cot, attrs): + """ + Return ``{old_name: current_name}`` for those entries in *attrs* that + resolve to exactly one of *cot*'s fields via rename history. + + One ObjectChange query covers all candidates. Ambiguous mappings (same + historical name appearing in multiple fields' history) are omitted so the + caller falls back to preserving the raw key — matching the + abstain-on-ambiguity behaviour of ``_translate_renamed_field_name``. + """ + attrs = [a for a in attrs if a] + if not attrs: + return {} + rows = ObjectChange.objects.filter( + changed_object_type__app_label='netbox_custom_objects', + changed_object_type__model='customobjecttypefield', + action=ObjectChangeActionChoices.ACTION_UPDATE, + ).filter( + Q(postchange_data__name__in=attrs) | Q(prechange_data__name__in=attrs) + ).values_list('changed_object_id', 'prechange_data', 'postchange_data') + + # attr → {field_pks that have this name anywhere in their history} + attr_to_field_pks: dict[str, set[int]] = {} + for field_pk, pre, post in rows: + for blob in (pre, post): + if not blob: + continue + name = blob.get('name') + if name in attrs: + attr_to_field_pks.setdefault(name, set()).add(field_pk) + + if not attr_to_field_pks: + return {} + + # Resolve the field pks we collected to their current names in one query. + pk_to_name = dict( + cot.fields.filter( + pk__in={pk for pks in attr_to_field_pks.values() for pk in pks} + ).values_list('pk', 'name') + ) + result = {} + for attr, field_pks in attr_to_field_pks.items(): + matched = [pk_to_name[pk] for pk in field_pks if pk in pk_to_name] + if len(matched) == 1: + result[attr] = matched[0] + return result + + +def _set_with_collision_preference(result, key, value): + """Set ``result[key] = value``; on collision prefer the non-None side. + + Squash-merge can map both the old and new name of a renamed field to the + same canonical key, with the new-side often carrying a sentinel ``None`` + from ``deep_compare_dict``. Preferring non-None keeps the real write. + """ + if key in result and value is None and result[key] is not None: + return + result[key] = value + class CustomObject( + OwnerMixin, BookmarksMixin, ChangeLoggingMixin, CloningMixin, + ContactsMixin, CustomLinksMixin, CustomValidationMixin, ExportTemplatesMixin, @@ -128,6 +627,188 @@ class CustomObject( class Meta: abstract = True + @classmethod + def resolve_field_aliases(cls, data): + """Rewrite *data* keys to this model's current field names. + + Called by netbox-branching's ``update_object()`` and + ``ChangeDiff._update_conflicts()`` when a CustomObjectTypeField has + been renamed between an ObjectChange recording and its replay. Walks + each field's rename history via ObjectChange records. Unknown keys + are preserved as-is; on collision the non-None value wins (see + ``_set_with_collision_preference``). + + Only top-level keys are rewritten. All current field-name carriers + (scalar columns, FK ``_id`` keys, M2M target lists, polymorphic + ``{content_type, object_id}`` payloads) appear at the top level, so + this is sufficient today. A future field type that stores user-field + names as nested-dict keys would need a recursive walk. + """ + if not data: + return data + + cot_id_str = extract_cot_id_from_model_name(cls.__name__.lower()) + if cot_id_str is None: + return data + cot_id = int(cot_id_str) + try: + cot = CustomObjectType.objects.get(pk=cot_id) + except CustomObjectType.DoesNotExist: + return data + + field_names = {f.name for f in cls._meta.get_fields()} + + # Collect the keys that don't match a current field name so we can do + # one batched ObjectChange query instead of one per unknown key. + unknown_keys = [] + for raw_key in data: + key = 'custom_field_data' if raw_key == 'custom_fields' else raw_key + if key not in field_names: + unknown_keys.append(key) + rename_map = _build_rename_map(cot, unknown_keys) if unknown_keys else {} + + result = {} + for raw_key, value in data.items(): + # Honour custom_fields → custom_field_data the same way update_object + # used to, so this hook is a true superset of the previous behaviour. + key = 'custom_field_data' if raw_key == 'custom_fields' else raw_key + if key in field_names: + _set_with_collision_preference(result, key, value) + continue + translated = _translate_renamed_field_name(cot, key, rename_map=rename_map) + if translated and translated in field_names: + _set_with_collision_preference(result, translated, value) + else: + # Unknown key (e.g. removed field) — preserve raw key so callers + # that inspect the dict for non-field metadata can still see it. + _set_with_collision_preference(result, raw_key, value) + return result + + @classmethod + def deserialize_object(cls, data, pk=None): + """ObjectChange.apply() hook for CREATE actions. + + Builds against the context-aware ``fresh_model`` (not Django's default + ``apps.get_model`` lookup, which would return main's class with the + wrong column set inside a branch). Stashes ``data`` in + ``_deferred_co_field_data`` so squash-ordering — a CO CREATE replayed + before its field's CREATE — can apply the values via raw UPDATE once + each column is added. + """ + cot_id_str = extract_cot_id_from_model_name(cls.__name__.lower()) + if cot_id_str is None: + return _deserialize_object(cls, data, pk=pk) + cot_id = int(cot_id_str) + + # In the squash case the cache may still point to a zero-field model. + CustomObjectType.clear_model_cache(cot_id) + try: + cot = CustomObjectType.objects.get(pk=cot_id) + fresh_model = cot.get_model() + except CustomObjectType.DoesNotExist: + fresh_model = cls + + resolved = fresh_model.resolve_field_aliases(data) + + obj = fresh_model() + if pk is not None: + obj.pk = pk + m2m_data = {} + # Polymorphic M2M data: keys named by serialize_object's sidecar + # (POLY_M2M_SIDECAR_KEY) — explicit so we don't rely on _field_objects + # (empty when squash replays the CO CREATE before its field CREATE) + # or value-shape guessing. + poly_m2m_field_names = { + entry['name'] + for entry in (resolved.get(POLY_M2M_SIDECAR_KEY) or ()) + if isinstance(entry, dict) and entry.get('name') + } + poly_m2m_data = {} + field_names = {f.name for f in fresh_model._meta.get_fields()} + + for attr, value in resolved.items(): + if attr == POLY_M2M_SIDECAR_KEY: + continue + # Tags via the standard NetBox path (Tag rows are looked up by name). + if attr == 'tags' and is_taggable(fresh_model): + tag_model = apps.get_model('extras', 'Tag') + m2m_data['tags'] = list(tag_model.objects.filter(name__in=value or [])) + continue + if attr in poly_m2m_field_names: + poly_m2m_data[attr] = value or [] + continue + if attr not in field_names: + # Unknown attribute (likely a removed field) — preserve it as a + # Python attribute so downstream code (e.g. _deferred_co_field_data) + # can still see it. + setattr(obj, attr, value) + continue + try: + f = fresh_model._meta.get_field(attr) + except FieldDoesNotExist: + setattr(obj, attr, value) + continue + if isinstance(f, ManyToManyField): + m2m_data[attr] = value + elif isinstance(f, ForeignKey): + # FK values arrive as the related PK; assign via the _id column. + setattr(obj, f.attname, value) + else: + # Coerce via to_python() for datetimes etc; fall back to raw on parse failure. + try: + setattr(obj, attr, f.to_python(value)) + except (ValidationError, ValueError, TypeError): + setattr(obj, attr, value) + + table_name = fresh_model._meta.db_table + full_data = dict(data) + + class _Deserialized: + object = obj + + def save(self, using=None, **_kwargs): + _using = using or DEFAULT_DB_ALIAS + models.Model.save_base(obj, using=_using, raw=True) + obj_pk = obj.pk # captures auto-assigned PK + # Re-apply M2M relations. Skipped quietly when the through + # table or column isn't present yet (squash ordering — the + # field's own CREATE replays later). hasattr() narrows the + # except below to genuine DB-state mismatches. + for accessor, related_pks in m2m_data.items(): + if not hasattr(obj, accessor): + logger.debug( + 'deserialize_object: deferred M2M %r on %s pk=%s (descriptor unbound)', + accessor, table_name, obj_pk, + ) + continue + manager = getattr(obj, accessor) + try: + manager.set(related_pks) + except (ProgrammingError, OperationalError): + logger.debug( + 'deserialize_object: deferred M2M %r on %s pk=%s (table absent)', + accessor, table_name, obj_pk, exc_info=True, + ) + # Replay polymorphic M2M directly via the through, pinned to + # _using. Bypasses manager.add() → m2m_changed → + # handle_changed_object, which can route across DB aliases. + schema_conn_local = connections[_using] + for field_name, rows in poly_m2m_data.items(): + through_table = ( + f'{USER_TABLE_DATABASE_NAME_PREFIX}{cot_id}_{field_name}' + ) + _apply_poly_m2m_rows(schema_conn_local, through_table, obj_pk, rows) + # Stash full data for deferred column updates (squash ordering fix). + deferred = _deferred_co_field_data.get() + if deferred is None: + deferred = {} + _deferred_co_field_data.set(deferred) + if table_name not in deferred: + deferred[table_name] = {} + deferred[table_name][obj_pk] = {'data': full_data} + + return _Deserialized() + def __str__(self): # Find the field with primary=True and return that field's "name" as the name of the object primary_field = self._field_objects.get(self._primary_field_id, None) @@ -157,11 +838,103 @@ def clean(self): if validators: run_validators(self, validators) + def serialize_object(self, exclude=None): + """Standard serialization plus polymorphic-field metadata. + + For polymorphic MULTIOBJECT fields, also appends + ``[{content_type, object_id}, ...]`` per field (Django's serializer + skips them — the descriptor isn't on ``_meta``). For both polymorphic + OBJECT and MULTIOBJECT, emits a sidecar of ``[{name, pk}, ...]`` so + the squash dependency resolver in ``branching.py`` can order the + field's CREATE before the CO's CREATE — without it, squash would + apply the CO before the columns/through exist. + """ + data = super().serialize_object(exclude=exclude) + field_objects = getattr(type(self), '_field_objects', None) or {} + poly_entries = [] + for fo in field_objects.values(): + field = fo['field'] + if not field.is_polymorphic: + continue + if field.type not in ( + CustomFieldTypeChoices.TYPE_OBJECT, + CustomFieldTypeChoices.TYPE_MULTIOBJECT, + ): + continue + if exclude and field.name in exclude: + continue + poly_entries.append({'name': field.name, 'pk': field.pk}) + if field.type != CustomFieldTypeChoices.TYPE_MULTIOBJECT: + continue + # MULTIOBJECT-only: append the through-table rows. + try: + manager = getattr(self, field.name) + through = manager._get_through_model() + except (AttributeError, LookupError): + continue + rows = list( + through.objects.filter(source_id=self.pk) + .values('content_type_id', 'object_id') + ) + if not rows: + data[field.name] = [] + continue + ct_ids = {r['content_type_id'] for r in rows} + ct_map = { + ct.id: f'{ct.app_label}.{ct.model}' + for ct in ContentType.objects.filter(id__in=ct_ids) + } + data[field.name] = [ + {'content_type': ct_map.get(r['content_type_id']), 'object_id': r['object_id']} + for r in rows + if r['content_type_id'] in ct_map + ] + if poly_entries: + data[POLY_M2M_SIDECAR_KEY] = poly_entries + return data + @property def _generated_table_model(self): # An indication that the model is a generated table model. return True + def delete(self, *args, **kwargs): + # Two prep steps before super() so the deletion collector doesn't + # raise traversing reverse FKs from through models: + # 1. Realign each through's ``source`` FK to ``type(self)`` — + # isinstance(instance, fk.related_model) otherwise sees two + # Table*Model classes and fails. + # 2. Temporarily unregister throughs whose physical table is gone + # (squash revert drops field-CREATEs before CO-CREATEs, so the + # CO's collector hits ``relation does not exist`` otherwise). + cls = type(self) + prefix = f'through_{cls._meta.db_table}' + registry = apps.all_models.get(APP_LABEL, {}) + # Branch contexts may have through tables only in the branch schema, so + # introspect via the active schema's connection, not the main one. + existing_tables = _get_schema_connection().introspection.table_names() + hidden = {} + for name, through in list(registry.items()): + if not name.startswith(prefix): + continue + if through._meta.db_table not in existing_tables: + hidden[name] = registry.pop(name) + continue + for fk_name in ('source', 'target'): + try: + field = through._meta.get_field(fk_name) + except FieldDoesNotExist: + continue + remote_meta = getattr(field.remote_field.model, '_meta', None) + if remote_meta is None or remote_meta.label != cls._meta.label: + continue + field.remote_field.model = cls + field.__dict__.pop('related_model', None) + try: + return super().delete(*args, **kwargs) + finally: + registry.update(hidden) + @property def clone_fields(self): """ @@ -172,12 +945,20 @@ def clone_fields(self): if not hasattr(self, "custom_object_type_id"): return () - # Get all field names where is_cloneable=True for this custom object type + # Get all fields where is_cloneable=True for this custom object type cloneable_fields = self.custom_object_type.fields.filter( is_cloneable=True - ).values_list("name", flat=True) + ).values_list("name", "type") - return tuple(cloneable_fields) + names = [] + for name, field_type in cloneable_fields: + # Coordinates fields have no single column; clone the two backing columns. + if field_type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + names += [f"{name}_latitude", f"{name}_longitude"] + else: + names.append(name) + + return tuple(names) def get_absolute_url(self): return reverse( @@ -208,6 +989,78 @@ def _get_action_url(cls, action=None, rest_api=False, kwargs=None): return reverse(cls._get_viewname(action, rest_api), kwargs=kwargs) +class CustomObjectConfigContextMixin(ConfigContextModel): + """ConfigContextModel variant for dynamically generated custom object models. + + Inherits ``local_context_data`` (a JSONField) and its ``clean()`` validator + from NetBox's ``ConfigContextModel``. Custom objects have none of the fixed + site/tenant/role attributes that ``ConfigContext.get_for_object()`` reads, so + we can't hand a custom object to it directly. Instead, by convention (issue + #98), a single non-polymorphic OBJECT field named ``site`` / ``tenant`` / + ``role`` / ``platform`` / ``location`` / ``device_type`` / ``cluster`` and + pointing at the matching NetBox model feeds that assignment dimension. We + build a proxy from those fields and reuse ``get_for_object`` so all of + NetBox's matching, hierarchy walking, and weight ordering apply. When no + such field exists we fall back to local-data-only. + """ + + class Meta: + abstract = True + + def _config_context_source(self): + """Proxy populated from convention-named OBJECT fields, or None if none match. + + Only honours a field whose *name* and *target model* both match the + convention, so a mistyped/mispointed field is silently ignored rather + than feeding the wrong dimension. + """ + from types import SimpleNamespace + + cot = self.custom_object_type + by_name = { + f.name: f + for f in cot.fields.filter( + type=CustomFieldTypeChoices.TYPE_OBJECT, + is_polymorphic=False, + name__in=CONFIG_CONTEXT_DIMENSION_FIELDS.keys(), + ) + } + dims = {name: None for name in CONFIG_CONTEXT_DIMENSION_FIELDS} + used = False + for name, (app_label, model_name) in CONFIG_CONTEXT_DIMENSION_FIELDS.items(): + f = by_name.get(name) + ct = getattr(f, "related_object_type", None) + if f and ct and (ct.app_label, ct.model) == (app_label, model_name): + dims[name] = getattr(self, name, None) + used = True + if not used: + return None + proxy = SimpleNamespace(**dims) + proxy.tags = self.tags # custom objects are taggable + return proxy + + def get_config_context(self): + """Merge ConfigContexts applicable to referenced dimension objects, then + overlay ``local_context_data`` (which takes precedence).""" + return self._render_config_context(self._config_context_source()) + + def _render_config_context(self, source): + """Merge the ConfigContexts for *source* (a proxy from + ``_config_context_source()``, or ``None``), then overlay + ``local_context_data``. Takes a pre-built *source* so a caller that also + needs the source-context list (the detail tab) can build the proxy once + instead of paying for the ``cot.fields`` lookup twice. + """ + data = {} + if source is not None: + contexts = ConfigContext.objects.get_for_object(source, aggregate_data=True) or [] + for context in contexts: + data = deepmerge(data, context) + if self.local_context_data: + data = deepmerge(data, self.local_context_data) + return data + + def validate_pep440(value): """Validate that *value* is a valid PEP 440 version string.""" if not value: @@ -223,12 +1076,16 @@ def validate_pep440(value): class CustomObjectType(NetBoxModel): # Class-level cache for generated models + # Branch-aware model cache keyed by (cot_id, branch_id_or_None). Only main's + # class (branch_id=None) is registered in apps.all_models so that + # content_type.model_class() resolves to a class with main's column set — + # branches may have renamed columns that don't exist in main. _model_cache = {} - _through_model_cache = ( - {} - ) # Now stores {custom_object_type_id: {through_model_name: through_model}} - _model_cache_locks = {} # Per-model locks to prevent race conditions - _global_lock = threading.RLock() # Global lock for managing per-model locks + # Per-(cot, branch) through-model registry: {(cot_id, branch_id): {name: through}}. + # Each context owns its through class so the source FK is set once at + # generation time and never mutated to follow another context's CO class. + _through_model_cache = {} + _global_lock = threading.RLock() _ON_DELETE_SQL = { ObjectFieldOnDeleteChoices.CASCADE: "CASCADE", ObjectFieldOnDeleteChoices.SET_NULL: "SET NULL", @@ -294,6 +1151,14 @@ class CustomObjectType(NetBoxModel): blank=True, editable=False ) + config_context_enabled = models.BooleanField( + default=False, + verbose_name=_("config context support"), + help_text=_( + "Enable local config context data on objects of this type. " + "Can only be set when the type is created." + ), + ) class Meta: verbose_name = "Custom Object Type" @@ -312,6 +1177,9 @@ def __str__(self): return self.display_name def clean(self): + # Guard against None (can arrive via update_object during branch revert) + if self.custom_field_data is None: + self.custom_field_data = {} super().clean() if not self.slug: @@ -319,6 +1187,24 @@ def clean(self): {"slug": _("Slug field cannot be empty.")} ) + # config_context_enabled is immutable after creation: the local_context_data + # column is created once when the type's table is built (managed=False, no + # migration), so flipping the flag would desync the generated model from the + # physical schema (False→True makes every query raise "column does not exist"). + # Enforce on validated paths (forms, full_clean()); raw QuerySet.update()/save() + # bypass clean() as with any Django model. (original is None only if the row was + # deleted concurrently — nothing to compare against, so skip.) + if self.pk: + original = CustomObjectType.objects.filter(pk=self.pk).values_list( + "config_context_enabled", flat=True + ).first() + if original is not None and self.config_context_enabled != original: + raise ValidationError({ + "config_context_enabled": _( + "Config context support cannot be changed after creation." + ) + }) + # Enforce max number of COTs that may be created (max_custom_object_types) if not self.pk: max_cots = get_plugin_config("netbox_custom_objects", "max_custom_object_types") @@ -328,88 +1214,95 @@ def clean(self): "exceeded; adjust max_custom_object_types to raise this limit" )) - @classmethod - def clear_model_cache(cls, custom_object_type_id=None): - """ - Clear the model cache for a specific CustomObjectType or all models. + @staticmethod + def _active_branch_id(): + """Active Branch id, or None for main — second component of the cache key.""" + try: + from netbox_branching.contextvars import active_branch + except ImportError: + return None + branch = active_branch.get() + return branch.id if branch is not None else None - :param custom_object_type_id: ID of the CustomObjectType to clear cache for, or None to clear all + @classmethod + def clear_model_cache(cls, custom_object_type_id=None, *, all_branches=False): + """Clear the cached generated model. + + Defaults to clearing only the current branch context's entry so the + other context's class (which is registered in ``apps.all_models``) + stays valid. ``all_branches=True`` wipes every (cot, branch) entry, + appropriate for COT deletion or full re-init. ``custom_object_type_id=None`` + clears everything. """ with cls._global_lock: if custom_object_type_id is not None: - cls._model_cache.pop(custom_object_type_id, None) - cls._through_model_cache.pop(custom_object_type_id, None) - cls._model_cache_locks.pop(custom_object_type_id, None) + if all_branches: + for key in list(cls._model_cache): + if key[0] == custom_object_type_id: + cls._model_cache.pop(key, None) + for key in list(cls._through_model_cache): + if key[0] == custom_object_type_id: + cls._through_model_cache.pop(key, None) + else: + branch_id = cls._active_branch_id() + cls._model_cache.pop((custom_object_type_id, branch_id), None) + cls._through_model_cache.pop((custom_object_type_id, branch_id), None) else: cls._model_cache.clear() cls._through_model_cache.clear() - cls._model_cache_locks.clear() # Clear Django apps registry cache to ensure newly created models are recognized apps.get_models.cache_clear() @classmethod - def get_cached_model(cls, custom_object_type_id): - """ - Get a cached model for a specific CustomObjectType if it exists. - - :param custom_object_type_id: ID of the CustomObjectType - :return: The cached model or None if not found + def _restore_main_through_registration(cls, cot_id, through_model_name): + """Restore main's through to ``apps.all_models`` after a branch + generation overwrote it (Django's metaclass auto-registers under the + same name). Keeps ``apps.get_model`` lookups returning main's class. """ - cache_entry = cls._model_cache.get(custom_object_type_id) - if cache_entry: - # Cache stores (model, timestamp) tuples - return cache_entry[0] - return None + main_throughs = cls._through_model_cache.get((cot_id, None)) + if not main_throughs: + return + main_through = main_throughs.get(through_model_name) + if main_through is None: + return + apps.all_models[APP_LABEL][through_model_name.lower()] = main_through @classmethod - def get_cached_timestamp(cls, custom_object_type_id): - """ - Get the timestamp of a cached model for a specific CustomObjectType. - - :param custom_object_type_id: ID of the CustomObjectType - :return: The cached timestamp or None if not found - """ - cache_entry = cls._model_cache.get(custom_object_type_id) - if cache_entry: - # Cache stores (model, timestamp) tuples - return cache_entry[1] - return None + def get_cached_model(cls, custom_object_type_id, branch_id=None): + """Cached model for (cot, branch), or None.""" + cache_entry = cls._model_cache.get((custom_object_type_id, branch_id)) + return cache_entry[0] if cache_entry else None @classmethod - def is_model_cached(cls, custom_object_type_id): - """ - Check if a model is cached for a specific CustomObjectType. - - :param custom_object_type_id: ID of the CustomObjectType - :return: True if the model is cached, False otherwise - """ - return custom_object_type_id in cls._model_cache + def get_cached_timestamp(cls, custom_object_type_id, branch_id=None): + """Cached timestamp for (cot, branch), or None.""" + cache_entry = cls._model_cache.get((custom_object_type_id, branch_id)) + return cache_entry[1] if cache_entry else None @classmethod - def get_cached_through_model(cls, custom_object_type_id, through_model_name): - """ - Get a specific cached through model for a CustomObjectType. + def is_model_cached(cls, custom_object_type_id, branch_id=None): + """True if a model is cached for (cot, branch).""" + return (custom_object_type_id, branch_id) in cls._model_cache - :param custom_object_type_id: ID of the CustomObjectType - :param through_model_name: Name of the through model to retrieve - :return: The cached through model or None if not found - """ - if custom_object_type_id in cls._through_model_cache: - return cls._through_model_cache[custom_object_type_id].get( - through_model_name - ) - return None + @classmethod + def get_cached_through_model(cls, custom_object_type_id, through_model_name, branch_id=None): + """Get a cached through model for a (cot, branch) context, or None.""" + return cls._through_model_cache.get((custom_object_type_id, branch_id), {}).get( + through_model_name + ) @classmethod - def get_cached_through_models(cls, custom_object_type_id): - """ - Get all cached through models for a CustomObjectType. + def get_cached_through_models(cls, custom_object_type_id, branch_id=None): + """Get all cached through models for a (cot, branch) context.""" + return cls._through_model_cache.get((custom_object_type_id, branch_id), {}) - :param custom_object_type_id: ID of the CustomObjectType - :return: Dict of through models or empty dict if not found - """ - return cls._through_model_cache.get(custom_object_type_id, {}) + def serialize_object(self, exclude=None): + # cache_timestamp is an internal cache-invalidation field; exclude it + # from ObjectChange records so it doesn't appear as a tracked change. + extra = ['cache_timestamp'] + combined = list(exclude or []) + extra + return super().serialize_object(exclude=combined) def get_absolute_url(self): return reverse("plugins:netbox_custom_objects:customobjecttype", args=[self.pk]) @@ -508,7 +1401,12 @@ def _after_model_generation(self, attrs, model): for f in list(model._meta.local_fields) + list(model._meta.local_many_to_many) } - # Collect through models during after_model_generation + # Per-(cot, branch) through models — fresh per context so their source FK + # is set once at generation time and never mutated to follow another + # context's CO class. Main's throughs stay canonical in apps.all_models; + # branch's throughs are kept private (we restore main's registration + # after Django's metaclass auto-registers ours). + branch_id = self._active_branch_id() through_models = [] for field_object in all_field_objects.values(): @@ -521,11 +1419,12 @@ def _after_model_generation(self, attrs, model): if field_instance.is_polymorphic: if field_instance.type == CustomFieldTypeChoices.TYPE_OBJECT: - # Polymorphic GFK: no through model, no after_model_generation needed. + # Polymorphic GFK: no through model. pass elif field_instance.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: - # Ensure the polymorphic through model is in the app registry. - # On server restart the registry is cleared; re-register if needed. + # Reuse the apps-registered through if present; otherwise + # create one. Avoids parallel through instances diverging + # from apps.all_models (collector class-identity mismatch). _apps = model._meta.apps try: through_model = _apps.get_model(APP_LABEL, field_instance.through_model_name) @@ -554,6 +1453,7 @@ def _after_model_generation(self, attrs, model): source_field.__dict__.pop('path_infos', None) source_field.__dict__.pop('reverse_path_infos', None) _apps.register_model(APP_LABEL, through_model) + self._register_context_through(branch_id, through_model) if through_model and through_model not in through_models: through_models.append(through_model) _wire_polymorphic_reverse_descriptors(field_instance) @@ -569,14 +1469,15 @@ def _after_model_generation(self, attrs, model): field_object["field"], model, field_name ) - # Collect through models from M2M fields + # Collect through models from M2M fields — already fresh per CO from + # MultiObjectFieldType.get_model_field → get_through_model. if hasattr(field, 'remote_field') and hasattr(field.remote_field, 'through'): through_model = field.remote_field.through - # Only collect custom through models, not auto-created Django ones if (through_model and through_model not in through_models and hasattr(through_model._meta, 'app_label') and through_model._meta.app_label == APP_LABEL): through_models.append(through_model) + self._register_context_through(branch_id, through_model) # Store through models on the model for yielding in get_models() model._through_models = through_models @@ -593,6 +1494,17 @@ def _after_model_generation(self, attrs, model): ).exclude(related_name=''): _wire_polymorphic_reverse_descriptors(inbound_field) + def _register_context_through(self, branch_id, through_model): + """Cache *through_model* under (self.pk, branch_id) and, for branch + contexts, restore main's canonical registration in apps.all_models + (Django's metaclass auto-registered this branch-side through with the + same name, overwriting main's entry).""" + key = (self.pk, branch_id) + bucket = type(self)._through_model_cache.setdefault(key, {}) + bucket[through_model.__name__] = through_model + if branch_id is not None: + self._restore_main_through_registration(self.pk, through_model.__name__) + @staticmethod def _collect_base_columns(model, user_field_names): """ @@ -671,11 +1583,9 @@ def get_content_type_label(custom_object_type_id): return f"Custom Objects > {custom_object_type.display_name}" def register_custom_object_search_index(self, model): - # model must be an instance of this CustomObjectType's get_model() generated class - # Use local_fields / local_many_to_many — plain lists populated at class-creation - # time — instead of _meta.get_field(), which triggers Django's lazy _relation_tree - # computation. _relation_tree calls apps.get_models(), which re-enters our - # get_models() override, which calls get_model() for every COT → infinite recursion. + # Use local_fields / local_many_to_many directly — calling _meta.get_field() + # triggers Django's lazy _relation_tree which re-enters get_models() and + # recurses through get_model() for every COT. present = ( {f.name for f in model._meta.local_fields} | {f.name for f in model._meta.local_many_to_many} @@ -716,14 +1626,27 @@ def get_model( :rtype: Model """ + branch_id = self._active_branch_id() + + # Lock guards the cache check, not the miss → re-cache window. Two + # threads can regenerate the same (cot_id, branch_id) in parallel; + # both produce equivalent classes, so the duplication is wasteful but + # not incorrect. Worth it to avoid serialising all generation. with self._global_lock: - if self.is_model_cached(self.id) and not no_cache: - cached_timestamp = self.get_cached_timestamp(self.id) - # Only use cache if the timestamps are available and match + if self.is_model_cached(self.id, branch_id) and not no_cache: + cached_timestamp = self.get_cached_timestamp(self.id, branch_id) if cached_timestamp and self.cache_timestamp and cached_timestamp == self.cache_timestamp: - model = self.get_cached_model(self.id) + model = self.get_cached_model(self.id, branch_id) + # registry["search"] is global, not per-branch — re-bind so + # post_save's search-cache handler sees this context's fields. + self.register_custom_object_search_index(model) return model else: + # Only clear the current (cot_id, branch_id) entry. Lazy + # invalidation: each branch context detects its own stale + # timestamp on next access — main's COT save propagates the + # bumped cache_timestamp to branches via change-capture, so + # they'll re-evaluate against their own row independently. self.clear_model_cache(self.id) # Generate the model outside the lock to avoid holding it during expensive operations @@ -771,107 +1694,123 @@ def get_model( # Wrap the existing post_through_setup method to handle ValueError exceptions from taggit.managers import TaggableManager as TM - original_post_through_setup = TM.post_through_setup + # TM.post_through_setup is class-level state; serialize concurrent + # generations so save/restore can't interleave across threads. + with _taggable_manager_patch_lock: + original_post_through_setup = TM.post_through_setup - def wrapped_post_through_setup(self, cls): - try: - return original_post_through_setup(self, cls) - except ValueError: - pass + def wrapped_post_through_setup(self, cls): + try: + return original_post_through_setup(self, cls) + except ValueError: + pass - TM.post_through_setup = wrapped_post_through_setup + TM.post_through_setup = wrapped_post_through_setup - try: - model = generate_model( - str(model_name), - (CustomObject, models.Model), - attrs, - ) - finally: - TM.post_through_setup = original_post_through_setup - - # Register the main model with Django's app registry. - # _suppress_clear_cache() is used directly here (rather than going - # through generate_model()) because we are calling apps.register_model() - # explicitly, not type(). generate_model() wraps type() and suppresses - # clear_cache only for that call; the suppression window needs to extend - # through the _model_cache write that follows so the model is safely - # cached before any re-entrant get_model() call can observe it. - # Without suppression: register_model() → clear_cache() → get_models() → - # get_model() → generate_model() → register_model() recurses infinitely. - with _suppress_clear_cache(): - if model_name.lower() in apps.all_models[APP_LABEL]: - # Remove the existing model from all_models before registering the new one - del apps.all_models[APP_LABEL][model_name.lower()] - - apps.register_model(APP_LABEL, model) - - self._after_model_generation(attrs, model) - - # When this COT's model is regenerated (cache miss), non-polymorphic through - # models owned by OTHER COTs that point to this COT as their M2M target keep - # their target FK stale (pointing at the old model class). Django's deletion - # collector finds those through FKs in the new model's related_objects and - # raises ValueError: "Cannot query X: Must be OldModel instance." - # Fix: walk all inbound non-polymorphic multiobject fields and patch the - # through model's target FK to the freshly generated model class. - # (Same pattern as the existing fix for polymorphic source FKs above at - # _after_model_generation lines 526-531.) - for inbound_field in CustomObjectTypeField.objects.filter( - related_object_type=self.object_type, - type=CustomFieldTypeChoices.TYPE_MULTIOBJECT, - is_polymorphic=False, - ).iterator(): - try: - through_model = apps.get_model(APP_LABEL, inbound_field.through_model_name) - target_field = through_model._meta.get_field('target') - except (LookupError, FieldDoesNotExist): - continue - target_field.remote_field.model = model - target_field.related_model = model - # path_infos is a @cached_property on ForeignKey (see Django's - # related.py). Clear it so the path is rebuilt using the updated - # remote_field.model; stale cached path_infos would make Django's - # deletion collector compare obj against the old model class and - # raise ValueError: "Cannot query X: Must be OldModel instance." - target_field.__dict__.pop('path_infos', None) - target_field.__dict__.pop('reverse_path_infos', None) - - # Same staleness problem exists for direct FK fields (TYPE_OBJECT): - # when this COT is regenerated, any cached model for another COT that - # holds a LazyForeignKey pointing here still references the old class. - # Walk inbound non-polymorphic object fields and patch them too. - for inbound_fk_field in CustomObjectTypeField.objects.filter( - related_object_type=self.object_type, - type=CustomFieldTypeChoices.TYPE_OBJECT, - is_polymorphic=False, - ).iterator(): - owner_model = CustomObjectType.get_cached_model(inbound_fk_field.custom_object_type_id) - if owner_model is None: - continue - # Use local_fields list — avoids _relation_tree → get_models() recursion. - fk_field = next( - (f for f in owner_model._meta.local_fields if f.name == inbound_fk_field.name), - None, + # Optionally mix in config context support (local_context_data) when + # the type opts in. The mixin contributes a concrete column, so the + # flag is only honoured at creation time (the table is built once via + # schema_editor.create_model() and is otherwise managed=False). + bases = (CustomObject, models.Model) + if self.config_context_enabled: + bases = (CustomObject, CustomObjectConfigContextMixin, models.Model) + + try: + model = generate_model( + str(model_name), + bases, + attrs, ) - if fk_field is None: - continue - fk_field.remote_field.model = model - fk_field.related_model = model - fk_field.to = model - fk_field.__dict__.pop('path_infos', None) - fk_field.__dict__.pop('reverse_path_infos', None) - - # Only cache fully-generated models. Models generated with - # skip_object_fields=True omit FK fields to other COTs; caching them - # would permanently hide those fields if a dependent COT triggers - # generation before this one in the startup loop (issue #408). + finally: + TM.post_through_setup = original_post_through_setup + + # Suppress clear_cache() through the _model_cache write so a re-entrant + # get_model() inside register_model → clear_cache → get_models() can hit + # the cache instead of recursing into another generation. + with _suppress_clear_cache(): + # Main's class is the canonical registration in apps.all_models; + # branch's class is cached only. Without this, content_type.model_class() + # would return a class with the wrong column set across contexts. + model_key = model_name.lower() + if branch_id is None: + if model_key in apps.all_models[APP_LABEL]: + del apps.all_models[APP_LABEL][model_key] + apps.register_model(APP_LABEL, model) + else: + main_class = self.get_cached_model(self.id, branch_id=None) + if main_class is not None: + apps.all_models[APP_LABEL][model_key] = main_class + # Else: branch class stays registered until main is generated — + # self-healing on the next main-context get_model() call. + + # _after_model_generation registers through models in + # _through_model_cache and mutates apps.all_models; hold the lock so + # concurrent get_model() calls for the same (cot_id, branch_id) can't + # interleave their through-model registrations. with self._global_lock: + self._after_model_generation(attrs, model) + + # When this COT's model is regenerated (cache miss), non-polymorphic through + # models owned by OTHER COTs that point to this COT as their M2M target keep + # their target FK stale (pointing at the old model class). Django's deletion + # collector finds those through FKs in the new model's related_objects and + # raises ValueError: "Cannot query X: Must be OldModel instance." + # Fix: walk all inbound non-polymorphic multiobject fields and patch the + # through model's target FK to the freshly generated model class. + # (Same pattern as the existing fix for polymorphic source FKs above at + # _after_model_generation lines 526-531.) + for inbound_field in CustomObjectTypeField.objects.filter( + related_object_type=self.object_type, + type=CustomFieldTypeChoices.TYPE_MULTIOBJECT, + is_polymorphic=False, + ).iterator(): + try: + through_model = apps.get_model(APP_LABEL, inbound_field.through_model_name) + target_field = through_model._meta.get_field('target') + except (LookupError, FieldDoesNotExist): + continue + target_field.remote_field.model = model + target_field.related_model = model + # path_infos is a @cached_property on ForeignKey (see Django's + # related.py). Clear it so the path is rebuilt using the updated + # remote_field.model; stale cached path_infos would make Django's + # deletion collector compare obj against the old model class and + # raise ValueError: "Cannot query X: Must be OldModel instance." + target_field.__dict__.pop('path_infos', None) + target_field.__dict__.pop('reverse_path_infos', None) + + # Same staleness problem exists for direct FK fields (TYPE_OBJECT): + # when this COT is regenerated, any cached model for another COT that + # holds a LazyForeignKey pointing here still references the old class. + # Walk inbound non-polymorphic object fields and patch them too. + for inbound_fk_field in CustomObjectTypeField.objects.filter( + related_object_type=self.object_type, + type=CustomFieldTypeChoices.TYPE_OBJECT, + is_polymorphic=False, + ).iterator(): + owner_model = CustomObjectType.get_cached_model(inbound_fk_field.custom_object_type_id) + if owner_model is None: + continue + # Use local_fields list — avoids _relation_tree → get_models() recursion. + fk_field = next( + (f for f in owner_model._meta.local_fields if f.name == inbound_fk_field.name), + None, + ) + if fk_field is None: + continue + fk_field.remote_field.model = model + fk_field.related_model = model + fk_field.to = model + fk_field.__dict__.pop('path_infos', None) + fk_field.__dict__.pop('reverse_path_infos', None) + + # Only cache fully-generated models. Models generated with + # skip_object_fields=True omit FK fields to other COTs; caching them + # would permanently hide those fields if a dependent COT triggers + # generation before this one in the startup loop (issue #408). if not skip_object_fields: - self._model_cache[self.id] = (model, self.cache_timestamp) + self._model_cache[(self.id, branch_id)] = (model, self.cache_timestamp) - # Now that the model is in _model_cache, clear_cache() is safe: - # re-entrant get_model() calls for this COT hit the cache immediately. apps.clear_cache() ContentType.objects.clear_cache() @@ -888,15 +1827,11 @@ def get_model_with_serializer(self): return model def _ensure_field_fk_constraint(self, model, field_name, on_delete_behavior=None): - """ - Ensure that a foreign key constraint is properly created at the database level - for a specific OBJECT type field. This is necessary because models are created - with managed=False, which may not properly create FK constraints. - - :param model: The model containing the field - :param field_name: The name of the field to ensure FK constraint for - :param on_delete_behavior: Override the ON DELETE behavior (ObjectFieldOnDeleteChoices value). - If None, the value is read from the corresponding CustomObjectTypeField record. + """Create the FK constraint for an OBJECT-type field at the DB level. + + Required because dynamic models are ``managed=False``, so Django won't + emit the FK on its own. ``on_delete_behavior`` defaults to the field's + recorded value, falling back to SET_NULL. """ table_name = self.get_database_table_name() @@ -925,8 +1860,9 @@ def _ensure_field_fk_constraint(self, model, field_name, on_delete_behavior=None related_table = related_model._meta.db_table column_name = model_field.column - q = connection.ops.quote_name - with connection.cursor() as cursor: + schema_conn = _get_schema_connection() + q = schema_conn.ops.quote_name + with schema_conn.cursor() as cursor: # Drop existing FK constraint if it exists. # Join on key_column_usage so we match by actual column name, not constraint name — # RENAME COLUMN updates kcu but leaves the constraint name unchanged. @@ -947,9 +1883,9 @@ def _ensure_field_fk_constraint(self, model, field_name, on_delete_behavior=None constraint_name = row[0] cursor.execute(f'ALTER TABLE {q(table_name)} DROP CONSTRAINT IF EXISTS {q(constraint_name)}') - # PROTECT maps to RESTRICT in SQL (raises an error on delete attempt). - # SET NULL and CASCADE map directly. - # For SET NULL the column must be nullable, which it always is for Object fields. + # PROTECT → RESTRICT; SET NULL needs a nullable column (always true + # for Object fields). NOT DEFERRABLE — deferred constraints queue + # trigger events that block subsequent ALTER TABLE calls. constraint_name = f"{table_name}_{column_name}_fk" cursor.execute(f""" ALTER TABLE {q(table_name)} @@ -978,13 +1914,11 @@ def create_model(self): # Ensure the ContentType exists and is immediately available features = get_model_features(model) - if 'branching' in features: - features.remove('branching') self.object_type.features = features self.object_type.public = True self.object_type.save() - with connection.schema_editor() as schema_editor: + with _get_schema_connection().schema_editor() as schema_editor: schema_editor.create_model(model) self._store_base_column_snapshot(model) @@ -992,6 +1926,64 @@ def create_model(self): get_serializer_class(model) self.register_custom_object_search_index(model) + @classmethod + def deserialize_object(cls, data, pk=None): + """Branching merge/revert hook — replays through the real ``save()``. + + The default ``DeserializedObject.save()`` does ``save_base(raw=True)``, + which skips signals and our ``save()`` override, so the dynamic table + never gets created in the destination schema. This wrapper runs the + full lifecycle. ``object_type`` is nulled only when the FK doesn't + resolve (mirrors ``clean_fields``) so normal merges don't churn the + ContentType/ObjectType pair. + """ + inner = _deserialize_object(cls, data, pk=pk) + + class _SchemaAwareDeserialized: + def __init__(self, deserialized): + self._inner = deserialized + self.object = deserialized.object + + def save(self, using=None, **kwargs): + # Snapshot before modifying so that diff()['pre'] records the + # current state rather than showing all fields as None on revert. + self.object.snapshot() + # Only null the ObjectType FK when it doesn't resolve in the + # destination schema; custom_object_type_post_save_handler will + # then rebuild the link via get_or_create after INSERT. + if ( + self.object.object_type_id is not None + and not ObjectType.objects.filter(pk=self.object.object_type_id).exists() + ): + self.object.object_type = None + self.object.object_type_id = None + self.object.save() + # Re-apply any M2M data (tags, etc.) that was stripped during deserialization. + if self._inner.m2m_data: + for accessor_name, object_list in self._inner.m2m_data.items(): + getattr(self.object, accessor_name).set(object_list) + self._inner.m2m_data = None + + return _SchemaAwareDeserialized(inner) + + def clean_fields(self, exclude=None): + """Tolerate a stale ``object_type`` FK on revert (DELETE-undo path). + + ``delete()`` destroys the core_objecttype row to satisfy ChangeDiff's + PROTECT FK; revert then re-inserts the COT, but full_clean would fail + validating the dangling FK. We null it here so validation passes; + ``custom_object_type_post_save_handler`` rebuilds the ContentType/ + ObjectType pair via get_or_create on the resulting INSERT. The new + pk is intentional — the old one's audit refs were already invalidated. + """ + if ( + self.object_type_id is not None + and not ObjectType.objects.filter(pk=self.object_type_id).exists() + ): + self.object_type = None + self.object_type_id = None + super().clean_fields(exclude=exclude) + def save(self, *args, **kwargs): needs_db_create = self._state.adding @@ -1004,10 +1996,15 @@ def save(self, *args, **kwargs): self.clear_model_cache(self.id) def delete(self, *args, **kwargs): - # Clear the model cache for this CustomObjectType - self.clear_model_cache(self.id) + # COT is going away — every branch's cached class is stale. + self.clear_model_cache(self.id, all_branches=True) + # Regenerate against the current context so the model used for the + # DDL drop below reflects this branch's column set, not whatever + # stale class an earlier context cached. model = self.get_model() + schema_conn = _get_schema_connection() + in_branch = schema_conn is not connection # Delete all CustomObjectTypeFields that reference this CustomObjectType (non-polymorphic) for field in CustomObjectTypeField.objects.filter(related_object_type=self.object_type): @@ -1022,33 +2019,71 @@ def delete(self, *args, **kwargs): field.delete() object_type = ObjectType.objects.get_for_model(model) - ObjectChange.objects.filter(changed_object_type=object_type).delete() - # Delete any NetBox CustomField records (extras) with related_object_type pointing - # to this COT's ObjectType. CustomField.related_object_type uses on_delete=PROTECT, - # so these must be removed before object_type.delete() is called below. - CustomField.objects.filter(related_object_type=object_type).delete() + # ObjectChange and ObjectType records live in the main schema. Only clean + # them up when operating outside a branch; inside a branch they belong to + # main and must not be touched until the deletion is merged. + if not in_branch: + ObjectChange.objects.filter(changed_object_type=object_type).delete() + + # Delete any NetBox CustomField records (extras) with related_object_type pointing + # to this COT's ObjectType. CustomField.related_object_type uses on_delete=PROTECT, + # so these must be removed before object_type.delete() is called below. + CustomField.objects.filter(related_object_type=object_type).delete() super().delete(*args, **kwargs) - # Temporarily disconnect the pre_delete handler to skip the ObjectType deletion - # TODO: Remove this disconnect/reconnect after ObjectType has been exempted from handle_deleted_object - pre_delete.disconnect(handle_deleted_object) - object_type.delete() - with connection.schema_editor() as schema_editor: - # Drop polymorphic through tables first (they have FKs to django_content_type - # and to the main table, so they must be dropped before the main table). + if not in_branch: + # ChangeDiff has a PROTECT FK to ContentType/ObjectType — delete those + # records first so object_type.delete() is not blocked. + try: + from netbox_branching.models import ChangeDiff + ChangeDiff.objects.filter(object_type=object_type).delete() + except ImportError: + pass + # Temporarily disconnect the pre_delete handler to skip the ObjectType deletion + # TODO: Remove this disconnect/reconnect after ObjectType has been exempted from handle_deleted_object + pre_delete.disconnect(handle_deleted_object) + try: + object_type.delete() + finally: + pre_delete.connect(handle_deleted_object) + + with schema_conn.schema_editor() as schema_editor: + # Drop through tables before the main table (FKs). Existence checks + # use schema_conn so they target the active branch's schema. for through_model in getattr(model, '_through_models', []): - if _table_exists(through_model._meta.db_table): + if _table_exists(through_model._meta.db_table, conn=schema_conn): schema_editor.delete_model(through_model) - schema_editor.delete_model(model) - - # Unregister the model and its through-models from Django's app registry so - # that subsequent ORM operations (e.g. deleting a related device) do not try - # to query the now-dropped table and receive a - # "relation 'custom_objects_' does not exist" error. - # Use _global_lock to prevent a concurrent get_model() call from racing - # against this de-registration and re-adding the model mid-cleanup. + # Django's schema_editor.delete_model(parent) auto-recurses into + # M2M fields' through models when their _meta.auto_created is + # truthy — which we set deliberately in + # MultiObjectFieldType.after_model_generation so Django's JSON + # serializer includes M2M values. That recursion would attempt a + # DROP TABLE for through tables whose actual DB tables are absent + # (e.g. when this delete runs on a COT whose branch-side through + # was already dropped by a revert). Clear auto_created on those + # missing-table throughs for the duration of delete_model(parent) + # and restore it after. + cleared = [] + for field in model._meta.local_many_to_many: + through = field.remote_field.through + if through._meta.auto_created and not _table_exists( + through._meta.db_table, conn=schema_conn, + ): + cleared.append((through, through._meta.auto_created)) + through._meta.auto_created = False + try: + # Flush DEFERRABLE FK triggers so PG doesn't reject the DROP. + schema_editor.execute('SET CONSTRAINTS ALL IMMEDIATE') + schema_editor.delete_model(model) + finally: + for through, original in cleared: + through._meta.auto_created = original + + # Unregister from apps.all_models so cascade-delete doesn't query the + # dropped table. _global_lock guards against a concurrent get_model() + # racing and re-registering mid-cleanup. with self._global_lock: model_name = model.__name__.lower() if model_name in apps.all_models.get(APP_LABEL, {}): @@ -1059,15 +2094,10 @@ def delete(self, *args, **kwargs): if through_name in apps.all_models.get(APP_LABEL, {}): del apps.all_models[APP_LABEL][through_name] - # Clear Django's internal relation/field caches so the removed model is no - # longer discovered during cascade-delete collector traversal. apps.clear_cache() - # Re-clear the model cache to remove re-cached model from get_model. - self.clear_model_cache(self.id) - - # Reconnect the pre_delete handler after all cleanup is done. - pre_delete.connect(handle_deleted_object) + # Re-clear in case anything re-cached during cleanup. + self.clear_model_cache(self.id, all_branches=True) @receiver(post_save, sender=CustomObjectType) @@ -1079,10 +2109,85 @@ def custom_object_type_post_save_handler(sender, instance, created, **kwargs): app_label=APP_LABEL, model=content_type_name ) + # Snapshot for the second save below (the object_type assignment). + # Without this, its ObjectChange would mark every field as changed. + instance.snapshot() instance.object_type = ct instance.save() +def _rename_objectchange_field_key(fi, old_name, new_name): + """Rewrite *old_name* → *new_name* JSON keys in ObjectChange (and + ChangeDiff when netbox-branching is installed) for this field's COT. + + Runs inside ``CustomObjectTypeField.save()``'s atomic so it rolls back + cleanly. JSON column names are literals and field names are validated + against ``^[a-z0-9_]+$`` — safe to interpolate. + """ + cot = fi.custom_object_type + model = cot.get_model() + ct = ContentType.objects.get_for_model(model) + # core.ObjectChange is branched by netbox-branching (migrations are allowed + # on the branch schema, and read routing sends ObjectChange queries to the + # active branch — see netbox_branching.database.BranchAwareRouter). Using + # _get_schema_connection() therefore updates the branch's copy of + # core_objectchange, keeping branch-context history consistent with the + # rename. Main's copy is updated when the rename is later merged and this + # function runs again in main context. + conn = _get_schema_connection() + + oc_sql = ( + 'UPDATE core_objectchange ' + 'SET {col} = ({col} - %s) || jsonb_build_object(%s, {col}->%s) ' + 'WHERE changed_object_type_id = %s AND {col} ? %s' + ) + # Savepoint contains the failure cleanly, then we re-raise so the outer + # field save aborts — silently logging would leave audit data inconsistent. + try: + with transaction.atomic(using=conn.alias): + with conn.cursor() as cursor: + for json_col in ('prechange_data', 'postchange_data'): + cursor.execute( + oc_sql.format(col=json_col), + [old_name, new_name, old_name, ct.id, old_name], + ) + except ProgrammingError: + logger.error( + '_rename_objectchange_field_key: ObjectChange schema mismatch ' + 'rewriting %r -> %r; aborting rename', + old_name, new_name, exc_info=True, + ) + raise + + logger.debug('_rename_objectchange_field_key: %r -> %r for %s', old_name, new_name, ct) + + try: + from netbox_branching.models import ChangeDiff # noqa: F401 + except ImportError: + return + + cd_sql = ( + 'UPDATE netbox_branching_changediff ' + 'SET {col} = ({col} - %s) || jsonb_build_object(%s, {col}->%s) ' + 'WHERE object_type_id = %s AND {col} IS NOT NULL AND {col} ? %s' + ) + try: + with transaction.atomic(using=conn.alias): + with conn.cursor() as cursor: + for json_col in ('original', 'modified', 'current'): + cursor.execute( + cd_sql.format(col=json_col), + [old_name, new_name, old_name, ct.id, old_name], + ) + except ProgrammingError: + logger.error( + '_rename_objectchange_field_key: ChangeDiff schema mismatch ' + 'rewriting %r -> %r; aborting rename', + old_name, new_name, exc_info=True, + ) + raise + + class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedModel): custom_object_type = models.ForeignKey( CustomObjectType, on_delete=models.CASCADE, related_name="fields" @@ -1090,7 +2195,7 @@ class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedMode type = models.CharField( verbose_name=_("type"), max_length=50, - choices=CustomFieldTypeChoices, + choices=CustomObjectFieldTypeChoices, default=CustomFieldTypeChoices.TYPE_TEXT, help_text=_("The type of data this custom object field holds"), ) @@ -1365,6 +2470,13 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + # Two distinct "original" mechanisms exist on this model: + # * ``_original_name`` / ``_original_type`` / ``_original_*`` (here) are + # scalar snapshots set on every instance (including freshly constructed + # ones) for cheap before/after comparisons in save(). + # * ``_original`` (set in ``from_db``) is a full instance clone from the + # DB row, used when more than a single attribute is needed; only + # DB-loaded objects have it — see ``original`` property. self._name = self.__dict__.get("name") self._original_name = self.name self._original_type = self.type @@ -1529,12 +2641,31 @@ def clean(self): {"unique": _("Uniqueness cannot be enforced for boolean or multiobject fields")} ) - # Check if uniqueness constraint can be applied when changing from non-unique to unique + # Coordinates fields expand into two columns and have no single value, so a + # number of single-value options do not apply. + if self.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + if self.unique: + raise ValidationError( + {"unique": _("Uniqueness cannot be enforced for coordinates fields")} + ) + if self.default is not None: + raise ValidationError( + {"default": _("A default value cannot be set for coordinates fields")} + ) + # A coordinates field has no single column matching its name, so it can + # never be added to the search index. Force the weight to 0 rather than + # silently honouring a non-zero value that has no effect. + self.search_weight = 0 + + # Check if uniqueness constraint can be applied when changing from non-unique to unique. + # _original is set by from_db only; deserialized objects (branch merge/revert) + # never load from DB and won't have it — guard before touching self.original. if ( self.pk and self.unique - and not self.original.unique and not self._state.adding + and hasattr(self, '_original') + and not self.original.unique ): field_type = FIELD_TYPE_CLASS[self.type]() model_field = field_type.get_model_field(self) @@ -1544,9 +2675,14 @@ def clean(self): old_field = field_type.get_model_field(self.original) old_field.contribute_to_class(model, self._original_name) + # Route the probe through the branch's connection so the ALTER + # TABLE runs in the active schema. Using the default connection + # here would either probe main's table from a branch context or + # fail outright if the table only exists in the branch schema. + probe_conn = _get_schema_connection() try: - with transaction.atomic(): - with connection.schema_editor() as test_schema_editor: + with transaction.atomic(using=probe_conn.alias): + with probe_conn.schema_editor() as test_schema_editor: test_schema_editor.alter_field(model, old_field, model_field) # If we get here, the constraint was applied successfully # Now raise a custom exception to rollback the test transaction @@ -1675,6 +2811,60 @@ def clean(self): {"name": _("Cannot rename a polymorphic field after creation.")} ) + # Prevent converting an existing field to or from coordinates. + # + # A coordinates field occupies two concrete columns ("{name}_latitude" and + # "{name}_longitude") while every other field type occupies a single column + # named "{name}". The save() path has no logic to migrate between those two + # shapes, so allowing the conversion would either leave the original column + # orphaned (→ coordinates) or attempt to alter a column that doesn't exist + # (coordinates → other), raising an uncaught database error. Reject it here, + # mirroring the polymorphic-flag guard above. + is_coordinates = self.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES + was_coordinates = self._original_type == CustomObjectFieldTypeChoices.TYPE_COORDINATES + if self.pk and not self._state.adding and is_coordinates != was_coordinates: + raise ValidationError( + {"type": _("Cannot change a field's type to or from coordinates after creation.")} + ) + + # Guard against backing-column name collisions. + # + # A coordinates field expands into "{name}_latitude"/"{name}_longitude". If a + # sibling field already occupies one of those column names (or vice versa: a + # plain field named "_latitude"), the schema editor would issue a + # duplicate-column ALTER and PostgreSQL would raise a ProgrammingError instead + # of a clean validation error. Detect the overlap here. + if self.custom_object_type_id: + def _occupied_columns(name, field_type): + if field_type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + return {f"{name}_latitude", f"{name}_longitude"} + return {name} + + own_columns = _occupied_columns(self.name, self.type) + # A clash can only involve a coordinates field's expanded columns. If + # this field isn't coordinates, only sibling coordinates fields can + # collide with it — so skip the full sibling scan and query just those + # (usually zero) to avoid an extra queryset on every field's save. + if self.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + siblings = self.custom_object_type.fields.all() + else: + siblings = self.custom_object_type.fields.filter( + type=CustomObjectFieldTypeChoices.TYPE_COORDINATES + ) + if self.pk: + siblings = siblings.exclude(pk=self.pk) + for sibling in siblings: + clash = own_columns & _occupied_columns(sibling.name, sibling.type) + if clash: + raise ValidationError( + { + "name": _( + "Field name conflicts with column '{column}' already used by field " + "'{other}' on this custom object type." + ).format(column=sorted(clash)[0], other=sibling.name) + } + ) + # related_name can only be set for object-type fields if self.related_name and self.type not in ( CustomFieldTypeChoices.TYPE_OBJECT, @@ -2227,7 +3417,6 @@ def from_db(cls, db, field_names, values): @property def original(self): return self._original - # return self.__class__(**self._loaded_values) @property def through_table_name(self): @@ -2257,211 +3446,136 @@ def through_model_name(self): # the app registry and does not collide with user-visible model names. return f"Through_{self.through_table_name}" + @classmethod + def deserialize_object(cls, data, pk=None): + """Branching merge/revert hook — replays through the real ``save()``. + + Same shape as ``CustomObjectType.deserialize_object``: the default + ``DeserializedObject.save(raw=True)`` skips our ``save()``, so the + column never gets added. This wrapper runs the full lifecycle. + """ + inner = _deserialize_object(cls, data, pk=pk) + + class _SchemaAwareDeserialized: + def __init__(self, deserialized): + self._inner = deserialized + self.object = deserialized.object + + def save(self, using=None, **kwargs): + self.object.save() + if self._inner.m2m_data: + for accessor_name, object_list in self._inner.m2m_data.items(): + getattr(self.object, accessor_name).set(object_list) + self._inner.m2m_data = None + + return _SchemaAwareDeserialized(inner) + def save(self, *args, **kwargs): is_new = self._state.adding - # Auto-assign schema_id for new fields that don't have one yet. - # Increments the monotonic counter on the parent CustomObjectType so that IDs are - # never reused, even after a field is deleted. The UniqueConstraint on - # (schema_id, custom_object_type) is the safety net against races; a concurrent - # writer would get an IntegrityError and must retry. - # Note: bulk_create() bypasses save() entirely, so auto-assignment will NOT fire for - # fields created via CustomObjectTypeField.objects.bulk_create(...). Always set - # schema_id explicitly when using bulk_create. + schema_conn = _get_schema_connection() + + # Auto-assign schema_id from the parent's monotonic counter. IDs are + # never reused, even after a field is deleted; the + # UniqueConstraint(schema_id, custom_object_type) is the race safety net. + # bulk_create() bypasses save() — callers must set schema_id explicitly. if self._state.adding and self.schema_id is None: - with transaction.atomic(): - cot = CustomObjectType.objects.select_for_update().get( + # Atomic and queryset pinned to schema_conn — the branching router + # routes CustomObjectType writes to the branch connection. + with transaction.atomic(using=schema_conn.alias): + cot = CustomObjectType.objects.using(schema_conn.alias).select_for_update().get( pk=self.custom_object_type_id ) new_schema_id = cot.next_schema_id + 1 - # Use update() rather than save() to avoid dispatching post_save on - # CustomObjectType, which would clear the model cache prematurely. - # The model cache must remain valid until this field's own save() calls - # get_model() below (to contribute the new field and alter the DB table). - CustomObjectType.objects.filter(pk=self.custom_object_type_id).update( - next_schema_id=new_schema_id - ) + # update() avoids post_save → clear_model_cache; the cache must + # remain valid until this field's own get_model() below. + CustomObjectType.objects.using(schema_conn.alias).filter( + pk=self.custom_object_type_id + ).update(next_schema_id=new_schema_id) self.schema_id = new_schema_id field_type = FIELD_TYPE_CLASS[self.type]() model = self.custom_object_type.get_model() - with connection.schema_editor() as schema_editor: - if self._state.adding: - if self.is_polymorphic: - # Polymorphic Object: add content_type + object_id columns + index - # Polymorphic MultiObject: create through table with content_type + object_id - if self.type == CustomFieldTypeChoices.TYPE_OBJECT: - field_type.add_polymorphic_object_columns(self, model, schema_editor) - elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: - field_type.create_polymorphic_m2m_table(self, model, schema_editor) - else: - model_field = field_type.get_model_field(self) - model_field.contribute_to_class(model, self.name) - # LazyForeignKey starts with a string remote_field.model. Django's - # lazy_related_operation fires immediately when the target is in - # apps.all_models, but tearDown() cleanup between tests can remove - # the target model from the registry. Resolve it directly here — - # bypassing the app-config's skip guard — so that schema_editor - # .add_field() always sees a model class, not a string. - if isinstance(model_field, LazyForeignKey) and isinstance(model_field.remote_field.model, str): - _app_label, _model_name = model_field._to_model_name.rsplit('.', 1) - _cot_id_str = extract_cot_id_from_model_name(_model_name.lower()) - if _cot_id_str is not None: - try: - _cot = CustomObjectType.objects.get(pk=int(_cot_id_str)) - _actual = _cot.get_model() - model_field.remote_field.model = _actual - model_field.to = _actual - except (CustomObjectType.DoesNotExist, OperationalError, ProgrammingError): - logger.warning( - "Could not resolve LazyForeignKey target %r before add_field; " - "schema_editor.add_field may fail", - model_field._to_model_name, - ) - schema_editor.add_field(model, model_field) - if self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: - field_type.create_m2m_table(self, model, self.name) - else: - # Polymorphic fields: renames and type changes are rejected by clean(). - # Non-schema attributes (label, description, …) may still change here. - # If clean() was bypassed and a rename slipped through, raise rather - # than silently leaving DB columns / through table out of sync. - if self.is_polymorphic or self._original_is_polymorphic: - if self.name != self._original_name: - raise ValidationError( - {"name": _("Cannot rename a polymorphic field after creation.")} - ) + # Schema mutation + audit-key rewrite + cache bump + parent save() share + # one atomic so a failure between DDL and row save can't leave audit + # data rewritten but the field record un-persisted (or vice versa). + with transaction.atomic(using=schema_conn.alias): + with schema_conn.schema_editor() as schema_editor: + if self._state.adding: + if self.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + # Coordinates expand into two concrete columns (latitude/longitude). + for column_name, model_field in field_type.get_model_field(self).items(): + model_field.contribute_to_class(model, column_name) + schema_editor.add_field(model, model_field) + elif self.is_polymorphic: + if self.type == CustomFieldTypeChoices.TYPE_OBJECT: + field_type.add_polymorphic_object_columns(self, model, schema_editor) + elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + field_type.create_polymorphic_m2m_table(self, model, schema_editor) + else: + _schema_add_field(self, model, schema_editor, schema_conn) + _apply_deferred_co_field(self) else: - old_field = field_type.get_model_field(self.original) - old_field.contribute_to_class(model, self._original_name) - - # Special handling for MultiObject fields when the name changes - if ( - self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT - and self.name != self._original_name - ): - # For renamed MultiObject fields, we just need to rename the through table - old_through_table_name = self.original.through_table_name - new_through_table_name = self.through_table_name - - # Check if old through table exists - with connection.cursor() as cursor: - tables = connection.introspection.table_names(cursor) - old_table_exists = old_through_table_name in tables - - if old_table_exists: - # Create temporary models to represent the old and new through table states - old_through_meta = type( - "Meta", - (), - { - "db_table": old_through_table_name, - "app_label": APP_LABEL, - "managed": True, - }, - ) - _old_through_model = generate_model( - f"TempOld{self.original.through_model_name}", - (models.Model,), - { - "__module__": "netbox_custom_objects.models", - "Meta": old_through_meta, - "id": models.AutoField(primary_key=True), - "source": models.ForeignKey( - model, - on_delete=models.CASCADE, - db_column="source_id", - related_name="+", - ), - "target": models.ForeignKey( - model, - on_delete=models.CASCADE, - db_column="target_id", - related_name="+", - ), - }, - ) - - new_through_meta = type( - "Meta", - (), - { - "db_table": new_through_table_name, - "app_label": APP_LABEL, - "managed": True, - }, - ) - new_through_model = generate_model( - f"TempNew{self.through_model_name}", - (models.Model,), - { - "__module__": "netbox_custom_objects.models", - "Meta": new_through_meta, - "id": models.AutoField(primary_key=True), - "source": models.ForeignKey( - model, - on_delete=models.CASCADE, - db_column="source_id", - related_name="+", - ), - "target": models.ForeignKey( - model, - on_delete=models.CASCADE, - db_column="target_id", - related_name="+", - ), - }, - ) - # Rename the table using Django's schema editor. - # new_through_model is passed as the first argument so Django - # can rename associated sequences (e.g. on PostgreSQL). - schema_editor.alter_db_table( - new_through_model, - old_through_table_name, - new_through_table_name, + if self.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + # Only a rename touches the schema; other attribute changes + # (label, description, …) are persisted by super().save() below. + if self.name != self._original_name: + new_fields = list(field_type.get_model_field(self).items()) + old_fields = list(field_type.get_model_field(self.original).items()) + for (old_col, old_field), (new_col, new_field) in zip(old_fields, new_fields): + old_field.contribute_to_class(model, old_col) + new_field.contribute_to_class(model, new_col) + schema_editor.alter_field(model, old_field, new_field) + # Polymorphic renames/type changes are rejected in clean(); + # raise here if one slipped through to avoid silent column drift. + elif self.is_polymorphic or self._original_is_polymorphic: + if self.name != self._original_name: + raise ValidationError( + {"name": _("Cannot rename a polymorphic field after creation.")} ) - else: - # No old table exists, create the new through table - field_type.create_m2m_table(self, model, self.name) - - # Alter the field normally (this updates the field definition) - schema_editor.alter_field(model, old_field, model_field) else: - # Normal field alteration - model_field = field_type.get_model_field(self) - model_field.contribute_to_class(model, self.name) - schema_editor.alter_field(model, old_field, model_field) - - # Ensure FK constraints are properly created for OBJECT fields - should_ensure_fk = False - if self.type == CustomFieldTypeChoices.TYPE_OBJECT and not self.is_polymorphic: - if self._state.adding: - should_ensure_fk = True - else: - type_changed_to_object = ( - self._original_type != CustomFieldTypeChoices.TYPE_OBJECT - and self.type == CustomFieldTypeChoices.TYPE_OBJECT - ) - related_object_changed = ( - self._original_type == CustomFieldTypeChoices.TYPE_OBJECT - and self.related_object_type_id != self._original_related_object_type_id - ) - on_delete_changed = ( - self._original_type == CustomFieldTypeChoices.TYPE_OBJECT - and self.on_delete_behavior != self._original_on_delete_behavior - ) - should_ensure_fk = type_changed_to_object or related_object_changed or on_delete_changed + _schema_alter_field(self.original, self, model, schema_editor, schema_conn) + + # Rewrite historical audit-data keys so any future replay can + # resolve old or new name to the current field name. + if ( + not self._state.adding + and not self.is_polymorphic + and self._original_name != self.name + ): + _rename_objectchange_field_key(self, self._original_name, self.name) - # Clear and refresh the model cache for this CustomObjectType when a field is modified - self.custom_object_type.clear_model_cache(self.custom_object_type.id) + # FK-constraint decision inside the atomic so a rollback discards it too. + should_ensure_fk = False + if self.type == CustomFieldTypeChoices.TYPE_OBJECT and not self.is_polymorphic: + if self._state.adding: + should_ensure_fk = True + else: + type_changed_to_object = ( + self._original_type != CustomFieldTypeChoices.TYPE_OBJECT + and self.type == CustomFieldTypeChoices.TYPE_OBJECT + ) + related_object_changed = ( + self._original_type == CustomFieldTypeChoices.TYPE_OBJECT + and self.related_object_type_id != self._original_related_object_type_id + ) + on_delete_changed = ( + self._original_type == CustomFieldTypeChoices.TYPE_OBJECT + and self.on_delete_behavior != self._original_on_delete_behavior + ) + should_ensure_fk = type_changed_to_object or related_object_changed or on_delete_changed - # Update parent's cache_timestamp to invalidate cache across all workers - self.custom_object_type.save(update_fields=['cache_timestamp']) + self.custom_object_type.clear_model_cache(self.custom_object_type.id) - super().save(*args, **kwargs) + # Bump cache_timestamp to invalidate other workers. snapshot() first + # so change logging records a correct pre-state. + self.custom_object_type.snapshot() + self.custom_object_type.save(update_fields=['cache_timestamp']) - # Ensure FK constraints AFTER the transaction commits to avoid "pending trigger events" errors + super().save(*args, **kwargs) + + # FK constraint runs AFTER commit to avoid "pending trigger events". if should_ensure_fk: _on_delete = self.on_delete_behavior _field_name = self.name @@ -2479,8 +3593,19 @@ def ensure_constraint(): transaction.on_commit(ensure_constraint) - # Reregister SearchIndex with new set of searchable fields - self.custom_object_type.register_custom_object_search_index(model) + # On rename, _schema_alter_field calls contribute_to_class twice on the + # same class — force a no_cache regeneration so _meta is clean. Non- + # rename changes lean on cache_timestamp for lazy invalidation; we skip + # the apps.clear_cache() cascade so signal-driven cache evictions (e.g. + # clear_cache_on_field_save for OBJECT fields) survive. + renamed = ( + not self._state.adding + and not self.is_polymorphic + and self._original_name != self.name + ) + if renamed: + updated_model = self.custom_object_type.get_model(no_cache=True) + self.custom_object_type.register_custom_object_search_index(updated_model) # Clean up stale descriptor when related_name is renamed on an existing polymorphic field if not is_new and self.is_polymorphic and hasattr(self, '_original'): @@ -2488,7 +3613,10 @@ def ensure_constraint(): if old_name and old_name != self.related_name: _unwire_polymorphic_reverse_descriptors(self, related_name=old_name) - # Reindex all objects of this type if search indexing was affected + # Reindex all objects of this type if search indexing was affected. + # self.original (backed by _original) is only set by from_db; the + # `not is_new` branch implies _state.adding is False, which implies + # the row came from the DB, so _original is guaranteed to exist. if is_new: needs_reindex = self.search_weight > 0 else: @@ -2500,14 +3628,20 @@ def ensure_constraint(): def delete(self, *args, **kwargs): field_type = FIELD_TYPE_CLASS[self.type]() model = self.custom_object_type.get_model() + schema_conn = _get_schema_connection() _dropped_through_model = None # Remove reverse descriptors before the M2M rows disappear via super().delete(). if self.is_polymorphic: _unwire_polymorphic_reverse_descriptors(self) - with connection.schema_editor() as schema_editor: - if self.is_polymorphic: + with schema_conn.schema_editor() as schema_editor: + if self.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + # Drop both backing columns (latitude/longitude). + for column_name, model_field in field_type.get_model_field(self).items(): + model_field.contribute_to_class(model, column_name) + schema_editor.remove_field(model, model_field) + elif self.is_polymorphic: if self.type == CustomFieldTypeChoices.TYPE_OBJECT: field_type.remove_polymorphic_object_columns(self, model, schema_editor) elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: @@ -2517,16 +3651,18 @@ def delete(self, *args, **kwargs): pass field_type.drop_polymorphic_m2m_table(self, model, schema_editor) else: - model_field = field_type.get_model_field(self) - model_field.contribute_to_class(model, self.name) - + # _schema_remove_field drops the through table (for MULTIOBJECT) but + # leaves the real through model in apps.all_models; capture it so the + # deregistration below removes it (issue #535). if self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: - _apps = model._meta.apps - _dropped_through_model = _apps.get_model(APP_LABEL, self.through_model_name) - schema_editor.delete_model(_dropped_through_model) - schema_editor.remove_field(model, model_field) + try: + _dropped_through_model = model._meta.apps.get_model(APP_LABEL, self.through_model_name) + except LookupError: + pass + _schema_remove_field(self, model, schema_editor, schema_conn=schema_conn) - # Deregister the dropped through model so the cascade-delete collector no longer queries the missing table. + # Deregister the dropped through model (both polymorphic and plain MULTIOBJECT) + # so the cascade-delete collector no longer queries the now-missing table. if _dropped_through_model is not None: through_name = _dropped_through_model.__name__.lower() with CustomObjectType._global_lock: @@ -2537,15 +3673,19 @@ def delete(self, *args, **kwargs): # Clear the model cache for this CustomObjectType when a field is deleted self.custom_object_type.clear_model_cache(self.custom_object_type.id) - # Update parent's cache_timestamp to invalidate cache across all workers + # snapshot() first so change logging records a correct pre-state. + self.custom_object_type.snapshot() self.custom_object_type.save(update_fields=['cache_timestamp']) super().delete(*args, **kwargs) - # Reregister SearchIndex with new set of searchable fields - self.custom_object_type.register_custom_object_search_index(model) + # Regenerate so the apps registry no longer holds a class with the + # removed column — squash revert may SELECT via the model class between + # field-undo and CO-undo, and a stale class would emit ProgrammingError. + updated_model = self.custom_object_type.get_model() + + self.custom_object_type.register_custom_object_search_index(updated_model) - # Reindex all objects of this type since a searchable field was removed if self.search_weight > 0: _cot_id = self.custom_object_type_id transaction.on_commit(lambda: ReindexCustomObjectTypeJob.enqueue(cot_id=_cot_id)) @@ -2798,6 +3938,7 @@ def clear_cache_on_field_save(sender, instance, **kwargs): try: related_cot = CustomObjectType.objects.get(object_type_id=instance.related_object_type_id) CustomObjectType.clear_model_cache(related_cot.id) + related_cot.snapshot() related_cot.save(update_fields=['cache_timestamp']) except CustomObjectType.DoesNotExist: pass diff --git a/netbox_custom_objects/templates/netbox_custom_objects/custom_object_bulk_delete.html b/netbox_custom_objects/templates/netbox_custom_objects/custom_object_bulk_delete.html index ce1346dd..9a7607f7 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/custom_object_bulk_delete.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/custom_object_bulk_delete.html @@ -14,10 +14,6 @@ {# Confirmation form #}
- {% if branch_warning %} - {% include 'netbox_custom_objects/inc/branch_warning.html' %} - {% endif %} - diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobject.html b/netbox_custom_objects/templates/netbox_custom_objects/customobject.html index 0be4c39c..051d8c0e 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobject.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobject.html @@ -10,9 +10,6 @@ {% load i18n %} {% load custom_object_utils %} -{% block extra_controls %} -{% endblock %} - {% block breadcrumbs %}
  • {{ object.custom_object_type.get_verbose_name_plural }}
  • {{ object.custom_object_type }}
    • @@ -27,6 +24,10 @@ {% block subtitle %}
    + {% if object.owner %} + {{ object.owner|linkify }} + · + {% endif %} {% trans "Created" %} {{ object.created|isodatetime:"minutes" }} {% if object.last_updated %} · @@ -38,6 +39,7 @@ {% block controls %}
    {% block control-buttons %} + {% block extra_controls %}{% endblock %} {# Default buttons #} {% if perms.extras.add_bookmark and object.bookmarks %} {% custom_object_bookmark_button object %} @@ -73,6 +75,14 @@
  • {{ object.custom_object_type.get_verbose_name }}
    • + {% if object.custom_object_type.config_context_enabled %} +
  • + {% trans "Config Context" %} +
    • + {% endif %} +
  • + {% trans "Contacts" %} +
  • {% trans "Journal" %}
    • @@ -123,7 +133,24 @@ {% endif %} - {% customfield_value field object|get_field_value:field %} + {% if field.type == 'coordinates' %} + {% with coords=object|get_field_value:field %} + {% if coords %} + {{ coords }} + {% with map_url=object|get_coordinate_map_url:field %} + {% if map_url %} + + {% trans "Map" %} + + {% endif %} + {% endwith %} + {% else %} + {{ ''|placeholder }} + {% endif %} + {% endwith %} + {% else %} + {% customfield_value field object|get_field_value:field %} + {% endif %} {% endif %} diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobject_edit.html b/netbox_custom_objects/templates/netbox_custom_objects/customobject_edit.html index b10e8fd9..8bf2a0b8 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobject_edit.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobject_edit.html @@ -14,10 +14,6 @@ {% endblock title %} {% block form %} - {% if branch_warning %} - {% include 'netbox_custom_objects/inc/branch_warning.html' %} - {% endif %} - {# Render hidden fields #} {% for field in form.hidden_fields %} {{ field }} @@ -29,15 +25,15 @@ {% block buttons %} {% trans "Cancel" %} {% if object.pk %} - {% else %}
    - -
    diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype.html b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype.html index a6724902..ed0f7b59 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype.html @@ -37,6 +37,10 @@
    {% trans "Custom Object Type" %}
    {% trans "Description" %} {{ object.description|placeholder }} + + {% trans "Config context support" %} + {% checkmark object.config_context_enabled %} + {% trans "Last activity" %} diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_bulk_delete.html b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_bulk_delete.html index a70435c2..4de80c5d 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_bulk_delete.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_bulk_delete.html @@ -2,8 +2,5 @@ {% load i18n %} {% block content %} - {% if branch_bypass_warning %} - {% include 'netbox_custom_objects/inc/branch_bypass_warning.html' %} - {% endif %} {{ block.super }} {% endblock content %} diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_bulk_edit.html b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_bulk_edit.html index e56b8554..80c7c0da 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_bulk_edit.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_bulk_edit.html @@ -2,8 +2,5 @@ {% load i18n %} {% block content %} - {% if branch_bypass_warning %} - {% include 'netbox_custom_objects/inc/branch_bypass_warning.html' %} - {% endif %} {{ block.super }} {% endblock content %} diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_bulk_import.html b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_bulk_import.html index 94a651ec..dd2d2bc6 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_bulk_import.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_bulk_import.html @@ -2,8 +2,5 @@ {% load i18n %} {% block content %} - {% if branch_bypass_warning %} - {% include 'netbox_custom_objects/inc/branch_bypass_warning.html' %} - {% endif %} {{ block.super }} {% endblock content %} diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_delete.html b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_delete.html index b548fe31..67038f43 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_delete.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_delete.html @@ -2,8 +2,5 @@ {% load i18n %} {% block content %} - {% if branch_bypass_warning %} - {% include 'netbox_custom_objects/inc/branch_bypass_warning.html' %} - {% endif %} {{ block.super }} {% endblock content %} diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_edit.html b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_edit.html index d7493013..b2224c1f 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_edit.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype_edit.html @@ -2,8 +2,5 @@ {% load i18n %} {% block form %} - {% if branch_bypass_warning %} - {% include 'netbox_custom_objects/inc/branch_bypass_warning.html' %} - {% endif %} {{ block.super }} {% endblock form %} diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttypefield_edit.html b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttypefield_edit.html index d7493013..b2224c1f 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttypefield_edit.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttypefield_edit.html @@ -2,8 +2,5 @@ {% load i18n %} {% block form %} - {% if branch_bypass_warning %} - {% include 'netbox_custom_objects/inc/branch_bypass_warning.html' %} - {% endif %} {{ block.super }} {% endblock form %} diff --git a/netbox_custom_objects/templates/netbox_custom_objects/htmx/bulk_edit_fields.html b/netbox_custom_objects/templates/netbox_custom_objects/htmx/bulk_edit_fields.html index 44d15a7a..e16b3cec 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/htmx/bulk_edit_fields.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/htmx/bulk_edit_fields.html @@ -24,5 +24,5 @@
    {% trans "Cancel" %} - +
    diff --git a/netbox_custom_objects/templates/netbox_custom_objects/htmx/co_delete_form.html b/netbox_custom_objects/templates/netbox_custom_objects/htmx/co_delete_form.html index 317a2b81..c7e54c4f 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/htmx/co_delete_form.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/htmx/co_delete_form.html @@ -11,9 +11,6 @@
    {% trans "Confirm Deletion" %}
    - {% if branch_warning %} - {% include 'netbox_custom_objects/inc/branch_warning.html' %} - {% endif %}

    {% blocktrans trimmed %} Are you sure you want to delete {{ object_type }} {{ object }}? @@ -65,6 +62,6 @@

    {% else %} {% endif %} - +

    diff --git a/netbox_custom_objects/templates/netbox_custom_objects/htmx/co_quick_add.html b/netbox_custom_objects/templates/netbox_custom_objects/htmx/co_quick_add.html new file mode 100644 index 00000000..19fdd82a --- /dev/null +++ b/netbox_custom_objects/templates/netbox_custom_objects/htmx/co_quick_add.html @@ -0,0 +1,27 @@ +{% load form_helpers %} +{% load helpers %} +{% load i18n %} +
    +

    + {% trans "Quick Add" %} {{ model|meta:"verbose_name"|bettertitle }} +

    + +
    +
    +
    + {% csrf_token %} + {% include 'htmx/form.html' %} +
    + + +
    +
    +
    \ No newline at end of file diff --git a/netbox_custom_objects/templates/netbox_custom_objects/htmx/delete_form.html b/netbox_custom_objects/templates/netbox_custom_objects/htmx/delete_form.html index 9d5ad6c5..0588a3a2 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/htmx/delete_form.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/htmx/delete_form.html @@ -7,9 +7,6 @@
    {% trans "Confirm Deletion" %}
    - {% if branch_bypass_warning %} - {% include 'netbox_custom_objects/inc/branch_bypass_warning.html' %} - {% endif %}

    {% blocktrans trimmed %} Are you sure you want to delete {{ object_type }} {{ object }}? diff --git a/netbox_custom_objects/templates/netbox_custom_objects/inc/branch_bypass_warning.html b/netbox_custom_objects/templates/netbox_custom_objects/inc/branch_bypass_warning.html deleted file mode 100644 index 935bc8bb..00000000 --- a/netbox_custom_objects/templates/netbox_custom_objects/inc/branch_bypass_warning.html +++ /dev/null @@ -1,12 +0,0 @@ -{% load i18n %} - -

    diff --git a/netbox_custom_objects/templates/netbox_custom_objects/inc/branch_warning.html b/netbox_custom_objects/templates/netbox_custom_objects/inc/branch_warning.html deleted file mode 100644 index ed7d0274..00000000 --- a/netbox_custom_objects/templates/netbox_custom_objects/inc/branch_warning.html +++ /dev/null @@ -1,12 +0,0 @@ -{% load i18n %} - - diff --git a/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html b/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html new file mode 100644 index 00000000..aef91dc6 --- /dev/null +++ b/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html @@ -0,0 +1,51 @@ +{% extends base_template %} +{% load helpers %} +{% load i18n %} + +{% block content %} +
    +
    +
    + {% include 'extras/inc/configcontext_data.html' with title="Rendered Context" data=rendered_context format=format copyid="rendered_context" %} +
    +
    +
    +
    + {% include 'extras/inc/configcontext_data.html' with title="Local Context" data=object.local_context_data format=format copyid="local_context" %} +
    + + + {% trans "The local config context overwrites all source contexts" %}. + +
    +
    +
    +

    + {% trans "Source Contexts" %} +
    + {% include 'extras/inc/format_toggle.html' %} +
    +

    + {% for context in source_contexts %} +
    +
    + {{ context.weight }} +
    + {{ context|linkify:"name" }} + {% if context.description %} +
    {{ context.description }} + {% endif %} + {% include 'extras/inc/configcontext_data.html' with data=context.data format=format %} +
    + {% empty %} +
    + {% trans "None found" %} +
    + {% trans "Config contexts are aggregated from objects this type references via fields named site, tenant, role, platform, location, device_type, or cluster." %} +
    +
    + {% endfor %} +
    +
    +
    +{% endblock %} diff --git a/netbox_custom_objects/templates/netbox_custom_objects/object_contacts.html b/netbox_custom_objects/templates/netbox_custom_objects/object_contacts.html new file mode 100644 index 00000000..c9b7f663 --- /dev/null +++ b/netbox_custom_objects/templates/netbox_custom_objects/object_contacts.html @@ -0,0 +1,11 @@ +{% extends 'generic/object_children.html' %} +{% load helpers %} +{% load i18n %} + +{% block extra_controls %} + {% if perms.tenancy.add_contactassignment %} + + {% trans "Add a contact" %} + + {% endif %} +{% endblock %} \ No newline at end of file diff --git a/netbox_custom_objects/templatetags/custom_object_utils.py b/netbox_custom_objects/templatetags/custom_object_utils.py index bfeae6b2..e0c65911 100644 --- a/netbox_custom_objects/templatetags/custom_object_utils.py +++ b/netbox_custom_objects/templatetags/custom_object_utils.py @@ -1,7 +1,9 @@ from django import template -from extras.choices import CustomFieldTypeChoices, CustomFieldUIVisibleChoices +from extras.choices import CustomFieldUIVisibleChoices +from netbox_custom_objects.choices import CustomObjectFieldTypeChoices from netbox_custom_objects.models import CustomObjectTypeField +from netbox_custom_objects.utilities import build_map_url __all__ = ( "get_field_object_type", @@ -9,11 +11,12 @@ "get_field_value", "get_field_is_ui_visible", "get_child_relations", + "get_coordinate_map_url", ) register = template.Library() -custom_field_type_verbose_names = {c[0]: c[1] for c in CustomFieldTypeChoices.CHOICES} +custom_field_type_verbose_names = {c[0]: c[1] for c in CustomObjectFieldTypeChoices.CHOICES} @register.filter(name="get_field_object_type") @@ -28,15 +31,36 @@ def get_field_type_verbose_name(field: CustomObjectTypeField) -> str: @register.filter(name="get_field_value") def get_field_value(obj, field: CustomObjectTypeField): + if field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + latitude = getattr(obj, f"{field.name}_latitude", None) + longitude = getattr(obj, f"{field.name}_longitude", None) + if latitude is None or longitude is None: + return None + return f"{latitude}, {longitude}" return getattr(obj, field.name) +@register.filter(name="get_coordinate_map_url") +def get_coordinate_map_url(obj, field: CustomObjectTypeField): + """Return the external map URL for a coordinates field, or None.""" + if field.type != CustomObjectFieldTypeChoices.TYPE_COORDINATES: + return None + latitude = getattr(obj, f"{field.name}_latitude", None) + longitude = getattr(obj, f"{field.name}_longitude", None) + return build_map_url(latitude, longitude) + + @register.filter(name="get_field_is_ui_visible") def get_field_is_ui_visible(obj, field: CustomObjectTypeField) -> bool: if field.ui_visible == CustomFieldUIVisibleChoices.ALWAYS: return True - if field.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + if field.type == CustomObjectFieldTypeChoices.TYPE_MULTIOBJECT: field_value = getattr(obj, field.name).exists() + elif field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + field_value = ( + getattr(obj, f"{field.name}_latitude", None) is not None + and getattr(obj, f"{field.name}_longitude", None) is not None + ) else: field_value = getattr(obj, field.name) if field.ui_visible == CustomFieldUIVisibleChoices.IF_SET and field_value: diff --git a/netbox_custom_objects/tests/base.py b/netbox_custom_objects/tests/base.py index 058e23cd..a7dc89f8 100644 --- a/netbox_custom_objects/tests/base.py +++ b/netbox_custom_objects/tests/base.py @@ -1,12 +1,94 @@ # Test utilities for netbox_custom_objects plugin +import logging + +from django.apps import apps as django_apps +from django.contrib.contenttypes.management import create_contenttypes from django.db import connection from django.test import Client -from core.models import ObjectType +from core.models import ObjectChange, ObjectType from extras.models import CustomFieldChoiceSet from users.models import Token from utilities.testing import create_test_user -from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField +from netbox_custom_objects.constants import APP_LABEL +from netbox_custom_objects.models import ( + CustomObjectType, + CustomObjectTypeField, + _deferred_co_field_data, +) + +logger = logging.getLogger(__name__) + + +def create_token(user): + """Create an API token for ``user`` and return its plaintext key. + + Handles the NetBox 4.5 token-versioning change (V1 tokens expose ``.token``; + older NetBox exposes ``.key``). + """ + try: + # NetBox >= 4.5 + from users.choices import TokenVersionChoices + token = Token(version=TokenVersionChoices.V1, user=user) + token.save() + return token.token + except ImportError: + # NetBox < 4.5 + token = Token(user=user) + token.save() + return token.key + + +def _recreate_contenttypes(): + """Recreate ContentType rows for all installed apps using get_or_create. + + Called after a TransactionTestCase flush so that subsequent test classes — + whether TransactionTestCase or regular TestCase — can look up ContentTypes. + Using create_contenttypes (get_or_create) avoids the duplicate-key + violations that serialized_rollback causes in the parallel test runner. + """ + for app_config in django_apps.get_app_configs(): + create_contenttypes(app_config, verbosity=0) + + +def _purge_stale_generated_models(): + """Remove dynamically generated CustomObject models from the app registry. + + Regular TestCase subclasses wrap each test in a transaction that is rolled + back at the end. Rolling back the transaction drops any tables created by + DDL inside the test (e.g. CREATE TABLE custom_objects_1), but Django's + in-memory model registry is NOT rolled back. The stale model entry then + causes problems for subsequent TransactionTestCase tests: + + - netbox-branching's get_tables_to_replicate() iterates over registered + models to build the list of tables to clone into the branch schema. + A stale model entry makes it try to COPY a table that no longer exists. + - Django's cascade-delete collector queries non-existent tables. + + Calling this in setUp() of every TransactionTestCase prevents both failure + modes. + """ + stale = [ + name + for name, model in list(django_apps.all_models.get(APP_LABEL, {}).items()) + if getattr(model, '_generated_table_model', False) + ] + for name in stale: + django_apps.all_models[APP_LABEL].pop(name, None) + if stale: + # Expire reverse-relation caches so any other app whose + # ``related_objects`` already pointed at the now-removed dynamic + # models gets rebuilt on next access. ``apps.clear_cache()`` alone + # clears the apps registry's own cache, but per-model + # ``_meta._relation_tree`` snapshots taken by Django outside this + # plugin survive — they need explicit expiry. + from django.contrib.contenttypes.models import ContentType # noqa: PLC0415 + ContentType._meta._expire_cache(forward=False) + for app_models in django_apps.all_models.values(): + for model in app_models.values(): + model._meta._expire_cache(forward=False) + django_apps.clear_cache() + _DYNAMIC_TABLE_PREFIX = "custom_objects_" @@ -67,9 +149,10 @@ def _drop_dynamic_tables(): all_tables = connection.introspection.table_names() dynamic = [t for t in all_tables if t.startswith(_DYNAMIC_TABLE_PREFIX)] if dynamic: + quote = connection.ops.quote_name with connection.cursor() as cursor: for table in dynamic: - cursor.execute(f'DROP TABLE IF EXISTS "{table}" CASCADE') + cursor.execute(f'DROP TABLE IF EXISTS {quote(table)} CASCADE') # Step 5 — rebuild the app registry cache now that both the stale model # entries (step 2) and the stale COT rows (step 3) are gone. get_models() @@ -79,6 +162,32 @@ def _drop_dynamic_tables(): django_apps.clear_cache() +def _reset_netbox_request_context(): + """Clear ``netbox.context.current_request`` and ``events_queue``. + + netbox.context_managers.event_tracking() yields without try/finally, so an + exception inside the ``with`` block leaves current_request set to the + previous request. In test runs that means the next test's pre_save signal + handler attributes ObjectChange to the previous test's user (whose row has + since been truncated by _fixture_teardown), producing + ``IntegrityError: insert or update on table "core_objectchange" violates + foreign key constraint`` on user_id. + """ + try: + from netbox.context import current_request, events_queue, query_cache + except ImportError: + return + current_request.set(None) + events_queue.set({}) + # ``query_cache`` is a ContextVar in current NetBox; in older releases it + # was a thread-local without ``.set()``. Catch AttributeError narrowly so + # an unrelated bug surfaces instead of being swallowed silently. + try: + query_cache.set(None) + except AttributeError: + logger.debug('netbox.context.query_cache has no .set(); skipping reset') + + def create_api_token(user): """Create an API token for *user*, handling the NetBox ≥ 4.5 version field.""" try: @@ -114,21 +223,63 @@ def _pre_setup(cls): _drop_dynamic_tables() super()._pre_setup() - def _fixture_teardown(self): - # Drop dynamic tables before Django's flush; without this, the flush - # command's TRUNCATE of django_content_type fails because our through - # tables have FK references to it. - _drop_dynamic_tables() - super()._fixture_teardown() + def setUp(self): + # Purge stale in-memory model registrations left by earlier TestCase + # classes whose rolled-back transactions dropped the backing tables. + # Must run before any code that iterates the model registry (e.g. + # netbox-branching's get_tables_to_replicate() during provisioning). + _purge_stale_generated_models() + # Clear netbox's request-scoped ContextVars. netbox.context_managers + # event_tracking() sets current_request on entry but only clears it + # *after* yield — if a test raises inside the with block, the cleanup + # never runs and the next test's branch.save() (which reads + # current_request to attribute ObjectChange) still sees the previous + # test's user, whose row was truncated by _fixture_teardown → FK + # violation on core_objectchange.user_id. + _reset_netbox_request_context() + super().setUp() def tearDown(self): + # Reset deferred CO field data so it doesn't bleed into the next test. + _deferred_co_field_data.set(None) + # Defensive reset — see setUp for rationale. Belt-and-braces in case a + # test enters event_tracking but raises before super().tearDown() runs. + _reset_netbox_request_context() + # Delete COTs and their backing tables before the DB flush. Cleanup + # is best-effort — if a previous test left the schema in a weird + # state, log and continue rather than failing tearDown (which would + # mask the real failure that put us here). for cot in CustomObjectType.objects.all(): try: cot.delete() - except Exception as exc: - print(f"WARNING: tearDown could not delete COT {cot.pk}: {exc}") + except Exception: + logger.warning( + 'tearDown could not delete COT %s', cot.pk, exc_info=True, + ) + # Remove any ObjectChange records created during the test (merge/revert creates + # them in main with the test user's ID). If left in place, the serialized_rollback + # snapshot accumulates them and restoring it after the next flush produces FK + # violations (user referenced by ObjectChange no longer exists). + ObjectChange.objects.all().delete() super().tearDown() + def _fixture_teardown(self): + """Flush tables and restore ContentTypes for the next test class. + + TransactionTestCase._fixture_teardown() TRUNCATEs all tables after each + test. Any TestCase class that follows on the same worker then finds no + ContentTypes and fails trying to look up ObjectType rows. Recreating + them here (idempotently, via get_or_create) avoids that without the + duplicate-key violations that serialized_rollback=True causes when the + parallel runner tries to INSERT rows that already exist. + """ + # Drop dynamic tables before Django's flush; without this, the flush + # command's TRUNCATE of django_content_type fails because our through + # tables have FK references to it. + _drop_dynamic_tables() + super()._fixture_teardown() + _recreate_contenttypes() + class CustomObjectsTestCase: """ diff --git a/netbox_custom_objects/tests/query_counts.json b/netbox_custom_objects/tests/query_counts.json index eddfc29c..1c266b6b 100644 --- a/netbox_custom_objects/tests/query_counts.json +++ b/netbox_custom_objects/tests/query_counts.json @@ -1,6 +1,6 @@ { - "customobject-complex:list_objects_with_permission": 42, - "customobject-objectfields:list_objects_with_permission": 51, - "customobject-simple:list_objects_with_permission": 34, - "customobjecttype:list_objects_with_permission": 36 + "customobject-complex:list_objects_with_permission": 41, + "customobject-objectfields:list_objects_with_permission": 50, + "customobject-simple:list_objects_with_permission": 33, + "customobjecttype:list_objects_with_permission": 34 } diff --git a/netbox_custom_objects/tests/test_api.py b/netbox_custom_objects/tests/test_api.py index 3eb5a467..7d4663d6 100644 --- a/netbox_custom_objects/tests/test_api.py +++ b/netbox_custom_objects/tests/test_api.py @@ -2,6 +2,7 @@ Tests for API code paths. """ import uuid +from decimal import Decimal from django.test import TestCase, RequestFactory from django.urls import reverse @@ -12,28 +13,14 @@ from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField from netbox_custom_objects.template_content import CustomObjectLink, LinkedCustomObject -from .base import CustomObjectsTestCase +from .base import CustomObjectsTestCase, create_token from core.models import ObjectType from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Rack, Site from extras.models import Tag -from users.models import ObjectPermission, Token +from users.models import ObjectPermission from virtualization.models import Cluster, ClusterType -def create_token(user): - try: - # NetBox >= 4.5 - from users.choices import TokenVersionChoices - token = Token(version=TokenVersionChoices.V1, user=user) - token.save() - return token.token - except ImportError: - # NetBox < 4.5 - token = Token(user=user) - token.save() - return token.key - - class CustomObjectAPITestCaseMixin: """ Base test class for custom object API endpoints. @@ -388,7 +375,10 @@ def test_create_with_nested_serializers(self): obj_perm.save() obj_perm.users.add(self.user) obj_perm.object_types.add(ObjectType.objects.get_for_model(self.model)) - self.add_permissions('dcim.view_device') + view_device_perm = ObjectPermission(name='view-device-for-nested', actions=['view']) + view_device_perm.save() + view_device_perm.users.add(self.user) + view_device_perm.object_types.add(ObjectType.objects.get_for_model(Device)) devices = Device.objects.all() @@ -766,6 +756,60 @@ def test_group_name_empty_by_default_in_api_response(self): self.assertIn('group_name', response.data) self.assertEqual(response.data['group_name'], '') + def _add_change_permission(self): + obj_perm = ObjectPermission(name='Change permission', actions=['change']) + obj_perm.save() + obj_perm.users.add(self.user) + obj_perm.object_types.add(ObjectType.objects.get_for_model(CustomObjectType)) + + def test_config_context_enabled_set_at_creation(self): + """config_context_enabled can be set to True when creating a type via the API.""" + data = {'name': 'cc_create', 'slug': 'cc-create', 'config_context_enabled': True} + url = reverse('plugins-api:netbox_custom_objects-api:customobjecttype-list') + response = self.client.post(url, data, format='json', **self.header) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.data) + self.assertTrue(response.data['config_context_enabled']) + cot = CustomObjectType.objects.get(pk=response.data['id']) + self.assertTrue(cot.config_context_enabled) + + def test_config_context_enabled_in_detail_response(self): + """The current value is readable via the API detail endpoint.""" + self._add_view_permission() + cot = CustomObjectType.objects.create( + name='cc_detail', slug='cc-detail', config_context_enabled=True, + ) + url = reverse( + 'plugins-api:netbox_custom_objects-api:customobjecttype-detail', + kwargs={'pk': cot.pk}, + ) + response = self.client.get(url, **self.header) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn('config_context_enabled', response.data) + self.assertTrue(response.data['config_context_enabled']) + + def test_config_context_enabled_immutable_after_creation(self): + """Changing config_context_enabled via PATCH is rejected (create-only).""" + self._add_change_permission() + cot = CustomObjectType.objects.create( + name='cc_immut', slug='cc-immut', config_context_enabled=False, + ) + url = reverse( + 'plugins-api:netbox_custom_objects-api:customobjecttype-detail', + kwargs={'pk': cot.pk}, + ) + # APIClient honours format='json' on PATCH (the plain Django client sends + # octet-stream → 415). + response = APIClient().patch( + url, {'config_context_enabled': True}, format='json', **self.header, + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn('config_context_enabled', response.data) + cot.refresh_from_db() + self.assertFalse(cot.config_context_enabled) + class CustomObjectTypeFieldObjectResolutionTest(CustomObjectsTestCase, TestCase): """ @@ -942,7 +986,7 @@ def test_detail_response_contains_netbox_standard_fields(self): response = self.client.get(self._detail_url(instance), **self.header) self.assertEqual(response.status_code, status.HTTP_200_OK) - for field in ('id', 'url', 'display', 'created', 'last_updated', 'tags'): + for field in ('id', 'url', 'display', 'owner', 'created', 'last_updated', 'tags'): self.assertIn(field, response.data, f"Standard field '{field}' missing from response") def test_detail_response_contains_custom_fields(self): @@ -1696,9 +1740,244 @@ def test_get_response_includes_cross_cot_m2m_field(self): self.assertIn(self.obj_target1.pk, ref_ids) -# --------------------------------------------------------------------------- -# Regression: explicit null on optional object/multiobject fields (#550) -# --------------------------------------------------------------------------- +class OwnerAPITest(CustomObjectsTestCase, TestCase): + """Tests for the owner field on custom object instances (#376).""" + + def setUp(self): + super().setUp() + from netbox_custom_objects.models import CustomObjectType + self.cot = CustomObjectType.objects.create(name='OwnedThing', slug='owned-thing') + self.model = self.cot.get_model() + token = create_token(self.user) + self.header = {'HTTP_AUTHORIZATION': f'Token {token}'} + self.client = APIClient() + + def _list_url(self): + return reverse( + 'plugins-api:netbox_custom_objects-api:customobject-list', + kwargs={'custom_object_type': self.cot.slug}, + ) + + def _detail_url(self, pk): + return reverse( + 'plugins-api:netbox_custom_objects-api:customobject-detail', + kwargs={'pk': pk, 'custom_object_type': self.cot.slug}, + ) + + def _add_perm(self, action, model): + perm = ObjectPermission(name=f'{action}-{model._meta.model_name}', actions=[action]) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(model)) + return perm + + def _grant_owner_view(self, suffix=''): + from users.models import Owner + perm = ObjectPermission(name=f'view-owner{suffix}', actions=['view']) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(Owner)) + return perm + + def test_owner_null_by_default(self): + """New CO instances must have owner=None when not explicitly set.""" + obj = self.model.objects.create() + self.assertIsNone(obj.owner) + + def test_owner_present_in_api_response(self): + """GET response must contain an 'owner' key (null when unset).""" + obj = self.model.objects.create() + self._add_perm('view', self.model) + + response = self.client.get(self._detail_url(obj.pk), **self.header) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn('owner', response.data) + self.assertIsNone(response.data['owner']) + + def test_create_with_owner(self): + """POST with owner= must persist the owner FK.""" + from users.models import Owner + owner = Owner.objects.create(name='create-owner') + self._add_perm('add', self.model) + self._grant_owner_view('-create') + + response = self.client.post( + self._list_url(), + {'owner': owner.pk}, + format='json', + **self.header, + ) + self.assertEqual( + response.status_code, + status.HTTP_201_CREATED, + f'Expected 201; got {response.status_code}: {getattr(response, "data", response.content)}', + ) + obj = self.model.objects.get(pk=response.data['id']) + self.assertEqual(obj.owner_id, owner.pk) + + def test_patch_sets_owner(self): + """PATCH with owner= must update the owner FK.""" + from users.models import Owner + obj = self.model.objects.create() + owner = Owner.objects.create(name='patch-owner') + self._add_perm('change', self.model) + self._grant_owner_view('-patch') + + response = self.client.patch( + self._detail_url(obj.pk), + {'owner': owner.pk}, + format='json', + **self.header, + ) + self.assertEqual( + response.status_code, + status.HTTP_200_OK, + f'Expected 200; got {response.status_code}: {getattr(response, "data", response.content)}', + ) + obj.refresh_from_db() + self.assertEqual(obj.owner_id, owner.pk) + + def test_patch_clears_owner(self): + """PATCH with owner=null must clear the owner FK.""" + from users.models import Owner + owner = Owner.objects.create(name='clear-owner') + obj = self.model.objects.create(owner=owner) + self._add_perm('change', self.model) + + response = self.client.patch( + self._detail_url(obj.pk), + {'owner': None}, + format='json', + **self.header, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + obj.refresh_from_db() + self.assertIsNone(obj.owner) + + def test_filter_by_owner_id(self): + """?owner_id= must return only instances with that owner.""" + from users.models import Owner + owner_a = Owner.objects.create(name='filter-owner-a') + owner_b = Owner.objects.create(name='filter-owner-b') + obj_a = self.model.objects.create(owner=owner_a) + obj_b = self.model.objects.create(owner=owner_b) + self._add_perm('view', self.model) + + response = self.client.get( + self._list_url(), + {'owner_id': owner_a.pk}, + **self.header, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + ids = [r['id'] for r in response.data['results']] + self.assertIn(obj_a.pk, ids) + self.assertNotIn(obj_b.pk, ids) + + def test_filter_by_owner_group_id(self): + """?owner_group_id= must return only instances whose owner belongs to that group.""" + from users.models import Owner, OwnerGroup + group_x = OwnerGroup.objects.create(name='group-x') + group_y = OwnerGroup.objects.create(name='group-y') + owner_x = Owner.objects.create(name='owner-in-x', group=group_x) + owner_y = Owner.objects.create(name='owner-in-y', group=group_y) + obj_x = self.model.objects.create(owner=owner_x) + obj_y = self.model.objects.create(owner=owner_y) + self._add_perm('view', self.model) + + response = self.client.get( + self._list_url(), + {'owner_group_id': group_x.pk}, + **self.header, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + ids = [r['id'] for r in response.data['results']] + self.assertIn(obj_x.pk, ids) + self.assertNotIn(obj_y.pk, ids) + + +class ConfigContextAPITest(CustomObjectsTestCase, TestCase): + """REST API exposure of local_context_data for config-context-enabled types (#98).""" + + def setUp(self): + super().setUp() + self.cot = CustomObjectType.objects.create( + name='cc_api', slug='cc-api', config_context_enabled=True, + ) + self.create_custom_object_type_field( + self.cot, name='name', label='Name', type='text', primary=True, required=True, + ) + self.model = self.cot.get_model() + token = create_token(self.user) + self.header = {'HTTP_AUTHORIZATION': f'Token {token}'} + self.client = APIClient() + + def _list_url(self): + return reverse( + 'plugins-api:netbox_custom_objects-api:customobject-list', + kwargs={'custom_object_type': self.cot.slug}, + ) + + def _detail_url(self, pk): + return reverse( + 'plugins-api:netbox_custom_objects-api:customobject-detail', + kwargs={'pk': pk, 'custom_object_type': self.cot.slug}, + ) + + def _add_perm(self, action): + perm = ObjectPermission(name=f'{action}-{self.model._meta.model_name}', actions=[action]) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(self.model)) + return perm + + def test_local_context_data_present_in_response(self): + obj = self.model.objects.create(name='obj-1', local_context_data={'ntp': ['10.0.0.1']}) + self._add_perm('view') + + response = self.client.get(self._detail_url(obj.pk), **self.header) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn('local_context_data', response.data) + self.assertEqual(response.data['local_context_data'], {'ntp': ['10.0.0.1']}) + + def test_set_local_context_data_via_patch(self): + obj = self.model.objects.create(name='obj-2') + self._add_perm('change') + + response = self.client.patch( + self._detail_url(obj.pk), + {'local_context_data': {'role': 'edge'}}, + format='json', + **self.header, + ) + self.assertEqual( + response.status_code, + status.HTTP_200_OK, + f'Expected 200; got {response.status_code}: {getattr(response, "data", response.content)}', + ) + obj.refresh_from_db() + self.assertEqual(obj.local_context_data, {'role': 'edge'}) + + def test_local_context_data_absent_when_disabled(self): + """A type without config context support must not expose local_context_data.""" + cot = CustomObjectType.objects.create(name='cc_off', slug='cc-off') + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, required=True, + ) + model = cot.get_model() + obj = model.objects.create(name='x') + + perm = ObjectPermission(name='view-cc-off', actions=['view']) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(model)) + + url = reverse( + 'plugins-api:netbox_custom_objects-api:customobject-detail', + kwargs={'pk': obj.pk, 'custom_object_type': cot.slug}, + ) + response = self.client.get(url, **self.header) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertNotIn('local_context_data', response.data) class NullOptionalObjectFieldTest(CustomObjectsTestCase, TestCase): @@ -1853,3 +2132,128 @@ def test_post_null_required_object_field_rejected(self): format='json', ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.data) + + +class CoordinatesFieldAPITest(CustomObjectsTestCase, NetBoxTestCase): + """REST API behaviour for the coordinates field type.""" + + @classmethod + def setUpTestData(cls): + cls.cot = CustomObjectType.objects.create( + name="GeoObject", + verbose_name_plural="Geo Objects", + slug="geo-objects", + ) + cls.create_custom_object_type_field( + cls.cot, name="name", type="text", primary=True, required=True + ) + cls.create_custom_object_type_field(cls.cot, name="location", type="coordinates") + cls.model = cls.cot.get_model() + + def setUp(self): + super().setUp() + self.user = create_test_user("geouser") + self.client = APIClient() + token_key = create_token(self.user) + self.header = {"HTTP_AUTHORIZATION": f"Token {token_key}"} + perm = ObjectPermission( + name="geo all", actions=["view", "add", "change", "delete"] + ) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(self.model)) + + def _list_url(self): + return reverse( + "plugins-api:netbox_custom_objects-api:customobject-list", + kwargs={"custom_object_type": self.cot.slug}, + ) + + def _detail_url(self, instance): + return reverse( + "plugins-api:netbox_custom_objects-api:customobject-detail", + kwargs={"pk": instance.pk, "custom_object_type": self.cot.slug}, + ) + + def test_serializer_exposes_flat_latitude_longitude(self): + """The coordinate columns are serialized as two flat fields (NetBox convention).""" + obj = self.model.objects.create( + name="Box", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + response = self.client.get(self._detail_url(obj), **self.header) + self.assertEqual(response.status_code, status.HTTP_200_OK, response.data) + self.assertEqual(Decimal(response.data["location_latitude"]), Decimal("40.712800")) + self.assertEqual(Decimal(response.data["location_longitude"]), Decimal("-74.006000")) + self.assertNotIn("location", response.data) + + def test_create_with_coordinates(self): + """Creating an object via the API persists both coordinate columns.""" + data = { + "name": "Created box", + "location_latitude": "41.000000", + "location_longitude": "-75.000000", + } + response = self.client.post(self._list_url(), data, format="json", **self.header) + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.data) + obj = self.model.objects.get(pk=response.data["id"]) + self.assertEqual(obj.location_latitude, Decimal("41.000000")) + self.assertEqual(obj.location_longitude, Decimal("-75.000000")) + + def test_create_half_populated_pair_rejected(self): + """Setting only one of latitude/longitude is rejected, keyed to the empty field.""" + data = {"name": "Bad box", "location_latitude": "41.000000"} + response = self.client.post(self._list_url(), data, format="json", **self.header) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.data) + # The error is pinned to the missing field, not non_field_errors. + self.assertIn("location_longitude", response.data) + self.assertNotIn("non_field_errors", response.data) + + def test_create_out_of_range_rejected(self): + """Latitude beyond ±90 is rejected by the model field validators.""" + data = { + "name": "Out of range", + "location_latitude": "120.000000", + "location_longitude": "10.000000", + } + response = self.client.post(self._list_url(), data, format="json", **self.header) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.data) + + def test_patch_clearing_one_half_rejected(self): + """A PATCH that clears only one coordinate of a populated pair is rejected.""" + obj = self.model.objects.create( + name="Box", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + response = self.client.patch( + self._detail_url(obj), + {"location_latitude": None}, + format="json", + **self.header, + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.data) + self.assertIn("location_latitude", response.data) + # The DB row is left untouched. + obj.refresh_from_db() + self.assertEqual(obj.location_latitude, Decimal("40.712800")) + self.assertEqual(obj.location_longitude, Decimal("-74.006000")) + + def test_patch_clearing_both_halves_allowed(self): + """A PATCH clearing both coordinates at once is accepted.""" + obj = self.model.objects.create( + name="Box", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + response = self.client.patch( + self._detail_url(obj), + {"location_latitude": None, "location_longitude": None}, + format="json", + **self.header, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK, response.data) + obj.refresh_from_db() + self.assertIsNone(obj.location_latitude) + self.assertIsNone(obj.location_longitude) diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py new file mode 100644 index 00000000..f8c8efc3 --- /dev/null +++ b/netbox_custom_objects/tests/test_branching.py @@ -0,0 +1,3038 @@ +""" +Branching integration tests for netbox-custom-objects. + +Requires netbox-branching to be installed alongside this plugin. All tests +are skipped when netbox-branching is absent so the suite remains clean in +environments that don't use branching. + +These tests use TransactionTestCase (not TestCase) because branch schemas live +in separate PostgreSQL schemas backed by distinct database connections that +cannot be rolled back inside a single SAVEPOINT-based transaction. +""" +import datetime +import decimal +import json +import logging +import os +import time +import unittest +import uuid +from unittest import mock + +from core.models import ObjectType +from dcim.models import Site +from django.contrib.auth import get_user_model +from django.contrib.contenttypes.models import ContentType +from django.db import connection as main_conn, connections +from django.test import RequestFactory, TransactionTestCase, override_settings +from django.urls import reverse +from extras.models import CustomFieldChoiceSet +from rest_framework.test import APIClient + +try: + from netbox.context_managers import event_tracking + from netbox_branching.choices import BranchStatusChoices + from netbox_branching.models import Branch + from netbox_branching.utilities import activate_branch + HAS_BRANCHING = True +except ImportError: + HAS_BRANCHING = False + +from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField +from netbox_custom_objects.tests.base import ( + TransactionCleanupMixin, + _recreate_contenttypes, + create_token, +) + +logger = logging.getLogger(__name__) +User = get_user_model() + + +def _make_request(user): + """Return a fresh request object suitable for event_tracking.""" + request = RequestFactory().get(reverse('home')) + request.id = uuid.uuid4() + request.user = user + return request + + +# Provisioning timeout for branch tests. Override via the +# ``NETBOX_CO_BRANCH_PROVISION_TIMEOUT`` env var (seconds) when CI flakes. +BRANCH_PROVISION_TIMEOUT = float( + os.environ.get('NETBOX_CO_BRANCH_PROVISION_TIMEOUT', '30') +) + + +def _provision_branch(name, merge_strategy=None, user=None, timeout=None): + """Create and wait for a branch to reach READY status. + + ``merge_strategy`` is optional (the Branch field is nullable) — only tests that + actually merge or revert need it; read-only tests (e.g. GraphQL, which can never + write, merge, or sync) leave it ``None``. + """ + if timeout is None: + timeout = BRANCH_PROVISION_TIMEOUT + branch = Branch(name=name, merge_strategy=merge_strategy) + branch.save(provision=False) + branch.provision(user=user) + deadline = time.time() + timeout + while time.time() < deadline: + branch.refresh_from_db() + if branch.status == BranchStatusChoices.READY: + return branch + time.sleep(0.1) + raise TimeoutError( + f'Branch {name!r} did not reach READY within {timeout:.0f} s ' + f'(status={branch.status!r})' + ) + + +def _close_branch_connections(): + """Close any open branch database connections. + + Best-effort cleanup between tests. A ``DatabaseError`` here typically + just means the connection was already closed by a previous teardown + pass; we log at DEBUG so a genuine bug isn't silently hidden but normal + multi-pass teardown stays quiet. + """ + from django.db.utils import DatabaseError + for branch in Branch.objects.all(): + try: + connections[branch.connection_name].close() + except DatabaseError: + logger.debug( + 'failed to close branch connection %r', + branch.connection_name, exc_info=True, + ) + + +class BranchingTestBase(TransactionCleanupMixin): + """ + Common per-test lifecycle for branching-aware test classes. + + Centralises the ``_recreate_contenttypes`` / ``_make_request`` / + ``_close_branch_connections`` boilerplate that was repeated across every + branch test class so a new test class doesn't accidentally skip a step. + + Subclasses still need to inherit from ``TransactionTestCase`` (directly, + not via this mixin) because branch schemas live in separate PostgreSQL + schemas backed by distinct DB connections that can't be rolled back + inside a single SAVEPOINT-based transaction. + """ + + def setUp(self): + # → TransactionCleanupMixin.setUp() → _purge_stale_generated_models() + super().setUp() + _recreate_contenttypes() + self.user = User.objects.create_user(username='testuser') + self.request = _make_request(self.user) + + def tearDown(self): + _close_branch_connections() + # → TransactionCleanupMixin.tearDown() → TransactionTestCase + super().tearDown() + + +# ── Shared merge/revert tests (strategy-agnostic) ──────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class BaseBranchingTests(BranchingTestBase): + """ + Merge and revert tests that run against every merge strategy. + + Subclasses must: + - set ``MERGE_STRATEGY`` to an iterative or squash strategy string + - also inherit from ``TransactionTestCase`` + + Example:: + + class IterativeBranchingTestCase(BaseBranchingTests, TransactionTestCase): + MERGE_STRATEGY = 'iterative' + """ + + MERGE_STRATEGY = None + + # ── simple: one COT, one text field, one CO ─────────────────────────── + + def test_simple_merge_and_revert(self): + """ + Create a COT with a single text field and one custom object instance + inside a branch. Merge to main, then revert. + + Assertions + ---------- + Before merge + - COT is absent from main + - field is absent from main + + After merge + - COT is present in main + - field is present in main + - get() on the CO in main returns the correct field value + + After revert + - COT is absent from main + - field is absent from main + """ + branch = _provision_branch('Simple Branch', self.MERGE_STRATEGY, self.user) + request = _make_request(self.user) + + # ── create inside branch ────────────────────────────────────────── + with activate_branch(branch), event_tracking(request): + cot = CustomObjectType.objects.create(name='simple_cot', slug='simple-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='notes', + label='Notes', + type='text', + ) + Model = cot.get_model() + co = Model.objects.create(notes='hello from branch') + + cot_pk, field_pk, co_pk = cot.pk, field.pk, co.pk + + # ── before merge: nothing in main ───────────────────────────────── + self.assertFalse( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must not be visible in main before merge', + ) + self.assertFalse( + CustomObjectTypeField.objects.filter(pk=field_pk).exists(), + 'Field must not be visible in main before merge', + ) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + # ── after merge: present in main ────────────────────────────────── + self.assertTrue( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must be in main after merge', + ) + self.assertTrue( + CustomObjectTypeField.objects.filter(pk=field_pk).exists(), + 'Field must be in main after merge', + ) + + # get() the CO and verify its value + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_main = cot_main.get_model().objects.get(pk=co_pk) + self.assertEqual(co_main.notes, 'hello from branch') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + branch.refresh_from_db() + + # ── after revert: gone from main ────────────────────────────────── + self.assertFalse( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must not be in main after revert', + ) + self.assertFalse( + CustomObjectTypeField.objects.filter(pk=field_pk).exists(), + 'Field must not be in main after revert', + ) + + # ── comprehensive: one of every field type ──────────────────────────── + + def test_comprehensive_merge_and_revert(self): + """ + Create a COT with one field of each supported type, plus a CO instance, + inside a branch. Merge to main, then revert. + + Field types + ----------- + text — plain VARCHAR column + integer — INTEGER column + decimal — DECIMAL column (exercises numeric precision handling) + boolean — BOOLEAN column + datetime — TIMESTAMPTZ column (exercises timezone-aware handling) + select — VARCHAR column with a ChoiceSet + object — ForeignKey column (to dcim.Site) + multiobject — M2M through-table (to dcim.Site) + + This exercises every distinct schema-editor operation + (add_field, add_FK, create_through_table) across a merge cycle and + verifies that all field values survive the round-trip. + + Assertions mirror test_simple_merge_and_revert but for every field type. + """ + # The Site is created in main before provisioning so it exists in both + # main and the branch schema and is valid as an FK target during merge. + with event_tracking(self.request): + site = Site.objects.create(name='Reference Site', slug='reference-site') + + branch = _provision_branch('Comprehensive Branch', self.MERGE_STRATEGY, self.user) + request = _make_request(self.user) + + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + cot_pk = None + field_pks = {} + co_pk = None + + test_dt = datetime.datetime(2024, 6, 15, 12, 0, 0, tzinfo=datetime.timezone.utc) + test_decimal = decimal.Decimal('3.14') + + # ── create inside branch ────────────────────────────────────────── + with activate_branch(branch), event_tracking(request): + choice_set = CustomFieldChoiceSet.objects.create( + name='Statuses', + extra_choices=[['active', 'Active'], ['inactive', 'Inactive']], + ) + cot = CustomObjectType.objects.create(name='full_cot', slug='full-cot') + cot_pk = cot.pk + + field_specs = [ + ('text_field', {'type': 'text'}), + ('int_field', {'type': 'integer'}), + ('dec_field', {'type': 'decimal'}), + ('bool_field', {'type': 'boolean'}), + ('dt_field', {'type': 'datetime'}), + ('select_field', {'type': 'select', 'choice_set': choice_set}), + ('obj_field', {'type': 'object', 'related_object_type': site_ot}), + ('multi_field', {'type': 'multiobject', 'related_object_type': site_ot}), + ] + for name, kwargs in field_specs: + f = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name=name, + label=name.replace('_', ' ').title(), + **kwargs, + ) + field_pks[name] = f.pk + + Model = cot.get_model() + co = Model.objects.create( + text_field='hello', + int_field=42, + dec_field=test_decimal, + bool_field=True, + dt_field=test_dt, + select_field='active', + obj_field_id=site.pk, + # multi_field left empty — through-table creation is still exercised + ) + co_pk = co.pk + + # ── before merge: nothing in main ───────────────────────────────── + self.assertFalse( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must not be in main before merge', + ) + for name, pk in field_pks.items(): + self.assertFalse( + CustomObjectTypeField.objects.filter(pk=pk).exists(), + f'Field {name!r} must not be in main before merge', + ) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + # ── after merge: present in main, values intact ─────────────────── + self.assertTrue( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must be in main after merge', + ) + for name, pk in field_pks.items(): + self.assertTrue( + CustomObjectTypeField.objects.filter(pk=pk).exists(), + f'Field {name!r} must be in main after merge', + ) + + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_main = cot_main.get_model().objects.get(pk=co_pk) + self.assertEqual(co_main.text_field, 'hello') + self.assertEqual(co_main.int_field, 42) + self.assertEqual(co_main.dec_field, test_decimal) + self.assertTrue(co_main.bool_field) + self.assertEqual(co_main.dt_field, test_dt) + self.assertEqual(co_main.select_field, 'active') + self.assertEqual(co_main.obj_field_id, site.pk) + + # Capture the multi_field's physical through-table name *while* it + # still exists so we can confirm it's gone after revert. + multi_field_main = CustomObjectTypeField.objects.get(pk=field_pks['multi_field']) + through_table = multi_field_main.through_table_name + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + branch.refresh_from_db() + + # ── after revert: gone from main ────────────────────────────────── + self.assertFalse( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must not be in main after revert', + ) + for name, pk in field_pks.items(): + self.assertFalse( + CustomObjectTypeField.objects.filter(pk=pk).exists(), + f'Field {name!r} must not be in main after revert', + ) + # The multi_field's through-table must also be physically dropped from + # main's schema — ORM absence isn't enough; without this assertion an + # orphaned through table could survive the revert and break a later + # COT that picks up the same id. + self.assertNotIn( + through_table, + main_conn.introspection.table_names(), + f'Through-table {through_table!r} must be physically dropped after revert', + ) + + # ── object modified inside branch ───────────────────────────────────── + + def test_object_modified_merge_and_revert(self): + """ + CO that exists in main is modified inside a branch. Merge brings the + new value to main; revert restores the original. + + The COT and field are created in main before provisioning so the branch + has a full schema copy. Only the CO data changes inside the branch. + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='modify_cot', slug='modify-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='notes', + label='Notes', + type='text', + ) + Model = cot.get_model() + co = Model.objects.create(notes='original value') + + co_pk = co.pk + branch = _provision_branch('Modify Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # Modify CO inside the branch. + with activate_branch(branch), event_tracking(branch_request): + branch_co = cot.get_model().objects.get(pk=co_pk) + branch_co.snapshot() # captures pre-change state for ObjectChange.diff()['pre'] during revert + branch_co.notes = 'modified in branch' + branch_co.save() + + # Main must not see the modification yet. + co.refresh_from_db() + self.assertEqual(co.notes, 'original value', 'Main must not see branch modification before merge') + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + co.refresh_from_db() + self.assertEqual(co.notes, 'modified in branch', 'Main must see modified value after merge') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + co.refresh_from_db() + self.assertEqual(co.notes, 'original value', 'Main must have original value after revert') + + # ── object deleted inside branch ────────────────────────────────────── + + def test_object_deleted_merge_and_revert(self): + """ + CO that exists in main is deleted inside a branch. Merge removes it + from main; revert restores it. + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='delete_cot', slug='delete-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='notes', + label='Notes', + type='text', + ) + Model = cot.get_model() + co = Model.objects.create(notes='will be deleted') + + co_pk = co.pk + branch = _provision_branch('Delete Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot.get_model().objects.get(pk=co_pk).delete() + + # CO must still exist in main before merge. + self.assertTrue( + cot.get_model().objects.filter(pk=co_pk).exists(), + 'CO must still exist in main before merge', + ) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + self.assertFalse( + cot.get_model().objects.filter(pk=co_pk).exists(), + 'CO must be deleted from main after merge', + ) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + self.assertTrue( + cot.get_model().objects.filter(pk=co_pk).exists(), + 'CO must be restored in main after revert', + ) + + # ── field renamed inside branch → merge ─────────────────────────────── + + def test_field_rename_merge_and_revert(self): + """ + Field created and then renamed inside a branch. Merge brings the COT + with the renamed column to main; revert removes it. + + Exercises _schema_alter_field via the merge deserialization path using + PK-based rename detection (same PK, different name values). + """ + branch = _provision_branch('Rename Branch', self.MERGE_STRATEGY, self.user) + request = _make_request(self.user) + + with activate_branch(branch), event_tracking(request): + cot = CustomObjectType.objects.create(name='rename_cot', slug='rename-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='old_name', + label='Old Name', + type='text', + ) + # Load from DB so _original is set, then rename. + field = CustomObjectTypeField.objects.get(pk=field.pk) + field.snapshot() # captures pre-change state for ObjectChange.diff()['pre'] during revert + field.name = 'new_name' + field.label = 'New Name' + field.save() + Model = cot.get_model() + co = Model.objects.create(new_name='value after rename') + + field_pk, cot_pk, co_pk = field.pk, cot.pk, co.pk + + self.assertFalse( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must not exist in main before merge', + ) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + self.assertTrue(CustomObjectTypeField.objects.filter(pk=field_pk).exists()) + field_main = CustomObjectTypeField.objects.get(pk=field_pk) + self.assertEqual(field_main.name, 'new_name', 'Field must have new name in main after merge') + + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_main = cot_main.get_model().objects.get(pk=co_pk) + self.assertEqual(co_main.new_name, 'value after rename') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + self.assertFalse(CustomObjectTypeField.objects.filter(pk=field_pk).exists()) + + # ── unique constraint toggled inside branch → merge ─────────────────── + + def test_field_unique_toggle_merge_and_revert(self): + """ + Field created without a unique constraint inside a branch, then + toggled to unique=True. Merge brings the COT with the UNIQUE + constraint to main; revert removes it. + + Exercises alter_field for constraint-only changes via the merge path. + """ + branch = _provision_branch('Unique Branch', self.MERGE_STRATEGY, self.user) + request = _make_request(self.user) + + with activate_branch(branch), event_tracking(request): + cot = CustomObjectType.objects.create(name='unique_cot', slug='unique-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='code', + label='Code', + type='text', + unique=False, + ) + # Load from DB so _original is set, then enable unique. + field = CustomObjectTypeField.objects.get(pk=field.pk) + field.snapshot() # captures pre-change state for ObjectChange.diff()['pre'] during revert + field.unique = True + field.save() + Model = cot.get_model() + co_pk = Model.objects.create(code='ABC').pk + + field_pk, cot_pk = field.pk, cot.pk + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + field_main = CustomObjectTypeField.objects.get(pk=field_pk) + self.assertTrue(field_main.unique, 'Field must have unique=True in main after merge') + + # Verify the UNIQUE constraint exists in main's physical schema. + cot_main = CustomObjectType.objects.get(pk=cot_pk) + table_name = cot_main.get_database_table_name() + with main_conn.cursor() as cursor: + constraints = main_conn.introspection.get_constraints(cursor, table_name) + self.assertTrue( + any(c['unique'] and c.get('columns') == ['code'] for c in constraints.values()), + 'UNIQUE constraint on "code" must exist in main schema after merge', + ) + + # CO with code='ABC' must have survived the merge. + cot_main2 = CustomObjectType.objects.get(pk=cot_pk) + self.assertTrue(cot_main2.get_model().objects.filter(pk=co_pk).exists()) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + + # ── non-schema field attributes inside branch → merge ───────────────── + + def test_field_non_schema_attrs_merge_and_revert(self): + """ + Field attributes that do not affect the physical schema (label, + primary, required, description) survive a branch merge and revert + without causing schema errors or spurious ALTER TABLE calls. + + These attributes are excluded from _field_schema_key and exist only + at the application layer. This test confirms that altering them in + a branch and merging correctly updates the ORM-level field record in + main without touching the DB column definition. + """ + branch = _provision_branch('Attrs Branch', self.MERGE_STRATEGY, self.user) + request = _make_request(self.user) + + with activate_branch(branch), event_tracking(request): + cot = CustomObjectType.objects.create(name='attrs_cot', slug='attrs-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='title', + label='Title', + type='text', + primary=False, + required=False, + description='original description', + ) + # Load from DB so _original is set, then mutate non-schema attrs. + field = CustomObjectTypeField.objects.get(pk=field.pk) + field.snapshot() # captures pre-change state for ObjectChange.diff()['pre'] during revert + field.label = 'Updated Title' + field.primary = True + field.required = True + field.description = 'updated description' + field.save() + + field_pk, cot_pk = field.pk, cot.pk + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + field_main = CustomObjectTypeField.objects.get(pk=field_pk) + self.assertEqual(field_main.label, 'Updated Title') + self.assertTrue(field_main.primary) + self.assertTrue(field_main.required) + self.assertEqual(field_main.description, 'updated description') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + self.assertFalse(CustomObjectTypeField.objects.filter(pk=field_pk).exists()) + + # ── existing main COT extended with new fields inside a branch ──────── + + def test_extend_main_cot_with_new_fields_merge_and_revert(self): + """ + COT exists in main with text + object + multiobject fields and a CO with all + three filled. A branch then *adds* new fields of each type to the same COT + and inserts a new CO that uses both the original and the new fields. Merge + brings the new fields and CO into main; revert removes them and leaves the + original CO intact. + + Exercises ``_schema_add_field`` (including FK and M2M through-table creation) + running against a target schema that already has live data and existing + through-tables — distinct from ``test_comprehensive_merge_and_revert`` which + creates everything greenfield inside the branch. + """ + # Two Sites in main so they exist in both schemas as valid FK targets. + with event_tracking(self.request): + site_a = Site.objects.create(name='Site A', slug='site-a') + site_b = Site.objects.create(name='Site B', slug='site-b') + + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + # ── main: COT with text + object + multiobject; one CO ──────────── + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='extend_cot', slug='extend-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='main_text', label='Main Text', type='text', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='main_obj', label='Main Obj', + type='object', related_object_type=site_ot, + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='main_multi', label='Main Multi', + type='multiobject', related_object_type=site_ot, + ) + MainModel = cot.get_model() + co_main = MainModel.objects.create( + main_text='main co text', + main_obj_id=site_a.pk, + ) + co_main.main_multi.set([site_a, site_b]) + + cot_pk = cot.pk + co_main_pk = co_main.pk + + branch = _provision_branch('Extend Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # ── branch: add new fields of each type, then create a CO with all ─ + with activate_branch(branch), event_tracking(branch_request): + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='branch_text', label='Branch Text', type='text', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='branch_obj', label='Branch Obj', + type='object', related_object_type=site_ot, + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='branch_multi', label='Branch Multi', + type='multiobject', related_object_type=site_ot, + ) + BranchModel = cot.get_model() + co_branch = BranchModel.objects.create( + main_text='branch co original text', + main_obj_id=site_b.pk, + branch_text='branch co new text', + branch_obj_id=site_a.pk, + ) + co_branch.main_multi.set([site_b]) + co_branch.branch_multi.set([site_a, site_b]) + + co_branch_pk = co_branch.pk + + # ── before merge: branch fields and branch CO not visible in main ─ + for fname in ('branch_text', 'branch_obj', 'branch_multi'): + self.assertFalse( + CustomObjectTypeField.objects.filter(custom_object_type=cot, name=fname).exists(), + f'{fname!r} must not be in main before merge', + ) + MainModelPre = cot.get_model() + self.assertFalse( + MainModelPre.objects.filter(pk=co_branch_pk).exists(), + 'Branch CO must not be in main before merge', + ) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + # ── after merge: all six fields present, both COs intact ────────── + cot_main = CustomObjectType.objects.get(pk=cot_pk) + names = set(cot_main.fields(manager='objects').values_list('name', flat=True)) + self.assertEqual( + names, + {'main_text', 'main_obj', 'main_multi', 'branch_text', 'branch_obj', 'branch_multi'}, + ) + + MergedModel = cot_main.get_model() + + # Original main CO untouched. + co_main_after = MergedModel.objects.get(pk=co_main_pk) + self.assertEqual(co_main_after.main_text, 'main co text') + self.assertEqual(co_main_after.main_obj_id, site_a.pk) + self.assertEqual( + set(co_main_after.main_multi.values_list('pk', flat=True)), + {site_a.pk, site_b.pk}, + ) + + # CO created in branch is in main with all field values. + co_branch_after = MergedModel.objects.get(pk=co_branch_pk) + self.assertEqual(co_branch_after.main_text, 'branch co original text') + self.assertEqual(co_branch_after.main_obj_id, site_b.pk) + self.assertEqual(co_branch_after.branch_text, 'branch co new text') + self.assertEqual(co_branch_after.branch_obj_id, site_a.pk) + self.assertEqual( + set(co_branch_after.main_multi.values_list('pk', flat=True)), + {site_b.pk}, + ) + self.assertEqual( + set(co_branch_after.branch_multi.values_list('pk', flat=True)), + {site_a.pk, site_b.pk}, + ) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + # Branch fields gone; main fields still present. + names_after_revert = set( + cot_main.fields(manager='objects').values_list('name', flat=True) + ) + self.assertEqual(names_after_revert, {'main_text', 'main_obj', 'main_multi'}) + + RevertedModel = cot_main.get_model() + # Original main CO survived. + co_main_reverted = RevertedModel.objects.get(pk=co_main_pk) + self.assertEqual(co_main_reverted.main_text, 'main co text') + self.assertEqual(co_main_reverted.main_obj_id, site_a.pk) + self.assertEqual( + set(co_main_reverted.main_multi.values_list('pk', flat=True)), + {site_a.pk, site_b.pk}, + ) + # Branch CO removed. + self.assertFalse( + RevertedModel.objects.filter(pk=co_branch_pk).exists(), + 'CO created in branch must be gone after revert', + ) + + # ── COT deleted inside branch → merge / revert ──────────────────────── + + def test_cot_deleted_in_branch_merge_and_revert(self): + """ + Delete a COT (with fields and CO instances in main) inside a branch, + merge the deletion to main, then revert and verify the schema is + restored. + + Scenario + -------- + 1. Main: create COT with a text field, an object field, and a + multiobject field; insert one CO using all three. + 2. Provision branch. + 3. Branch: delete the COT. + 4. Merge: main loses the COT, its fields, the CO instances, the + main table, and the multi-object through-table. + 5. Revert: COT, fields, the dynamic table, and the multi-object + through-table must all come back at the *original* PKs — the + ContentType pk in particular has to survive the round-trip so + any existing FK references remain valid. + + Both schema directions are exercised: + - Forward (merge): the squash strategy collapses field-level + deletes alongside the COT delete, so ``_schema_remove_field`` + must stay idempotent when the COT's own ``delete()`` already + dropped the through-table. + - Backward (revert): ``CustomObjectType.delete()`` destroys the + related ContentType row to satisfy ChangeDiff's PROTECT FK. + Restoring the COT then requires the original ContentType pk + to come back too — handled by ``restore_object`` (the DELETE- + undo counterpart to ``deserialize_object``). + + CO data preservation is **not** asserted here. The current + delete path drops the dynamic table via raw DDL + (``schema_editor.delete_model``) without firing per-row + ``pre_delete`` signals, so no ObjectChange records exist for the + CO instances and there is nothing for revert to replay. + Recovering CO data across a COT-delete cycle would require + iterating instances and calling ``.delete()`` on each before the + DROP TABLE — a separate, larger change. + """ + # FK target lives in main so both schemas share it. + with event_tracking(self.request): + site = Site.objects.create(name='COT-delete Site', slug='cot-delete-site') + + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + # ── main: COT + fields + CO ─────────────────────────────────────── + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='doomed_cot', slug='doomed-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='note', label='Note', type='text', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='site_ref', label='Site', + type='object', related_object_type=site_ot, + ) + multi_field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='sites', label='Sites', + type='multiobject', related_object_type=site_ot, + ) + Model = cot.get_model() + co = Model.objects.create(note='hello', site_ref_id=site.pk) + co.sites.set([site]) + + cot_pk = cot.pk + co_pk = co.pk + co_table = cot.get_database_table_name() + through_table = multi_field.through_table_name + + # Sanity: physical tables exist in main before we touch anything. + self.assertIn(co_table, main_conn.introspection.table_names()) + self.assertIn(through_table, main_conn.introspection.table_names()) + + branch = _provision_branch('COT Delete Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # ── branch: delete the COT ──────────────────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + branch_cot = CustomObjectType.objects.get(pk=cot_pk) + branch_cot.snapshot() + branch_cot.delete() + + # Main still has the COT before the merge applies. + self.assertTrue(CustomObjectType.objects.filter(pk=cot_pk).exists()) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + # COT, fields, and physical tables must all be gone from main. + self.assertFalse( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must be gone from main after merge of branch deletion', + ) + main_tables = main_conn.introspection.table_names() + self.assertNotIn( + co_table, main_tables, + f'Main CO table {co_table!r} must be dropped after merge', + ) + self.assertNotIn( + through_table, main_tables, + f'Through-table {through_table!r} must be dropped after merge', + ) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + branch.refresh_from_db() + + # COT and fields must come back at their original pks. + self.assertTrue( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must be restored after revert of branch deletion', + ) + restored = CustomObjectType.objects.get(pk=cot_pk) + field_names = set(restored.fields(manager='objects').values_list('name', flat=True)) + self.assertEqual(field_names, {'note', 'site_ref', 'sites'}) + + # The COT comes back with a fresh ContentType/ObjectType pair + # (clean_fields nulls the stale FK; the post_save handler then + # calls get_or_create which creates a new row). The pk is + # intentionally not preserved — cross-branch audit data that + # referenced the original pk was already invalidated when the + # COT was deleted, so a fresh pk is the honest representation. + restored_model = restored.get_model() + self.assertIsNotNone(restored.object_type_id) + self.assertTrue( + ObjectType.objects.filter(pk=restored.object_type_id).exists(), + 'Restored COT must reference a live ObjectType row', + ) + + restored_tables = main_conn.introspection.table_names() + self.assertIn(co_table, restored_tables, 'CO table must be re-created on revert') + self.assertIn(through_table, restored_tables, 'Through-table must be re-created on revert') + + # CO data is NOT restored — see docstring. ``co_pk`` is referenced + # here only to make the unused-variable warning irrelevant; the row + # at that pk legitimately does not exist after revert. + self.assertFalse( + restored_model.objects.filter(pk=co_pk).exists(), + 'CO instances are not recovered by revert (see test docstring)', + ) + + # ── multi-object field rename across merge ──────────────────────────── + + def test_multiobject_field_rename_merge_and_revert(self): + """ + Rename a multi-object field inside a branch and merge. + + Through-table renames are the most fragile schema operation: the + physical table name changes (``alter_db_table``) but the integer + FKs to the parent CO model and to the related object type must + keep pointing at the same rows. This test verifies that: + + * The old through-table is gone from main after merge. + * The new through-table is present and holds the same rows. + * Reading the M2M via the new accessor returns the original + values intact. + * Revert restores the old through-table name and field name. + """ + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + with event_tracking(self.request): + site_a = Site.objects.create(name='M2M Site A', slug='m2m-site-a') + site_b = Site.objects.create(name='M2M Site B', slug='m2m-site-b') + + # ── main: COT with a multi-object field + a CO with M2M values ──── + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='m2m_rename_cot', slug='m2m-rename-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='tags_old', label='Tags', + type='multiobject', related_object_type=site_ot, + ) + MainModel = cot.get_model() + co = MainModel.objects.create() + co.tags_old.set([site_a, site_b]) + + cot_pk = cot.pk + field_pk = field.pk + co_pk = co.pk + old_through = field.through_table_name + + branch = _provision_branch('M2M Rename Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # ── branch: rename the multi-object field ───────────────────────── + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(pk=field_pk) + f.snapshot() + f.name = 'tags_new' + f.label = 'Tags (renamed)' + f.save() + + # Compute the post-rename through-table name from a freshly-loaded + # field record so we don't depend on the in-memory branch state. + renamed_field = CustomObjectTypeField.objects.get(pk=field_pk) + # Field name in main hasn't applied yet (still 'tags_old' there) — we + # need the *branch's* current name, which is what the rename target is. + new_through = ( + f"custom_objects_{renamed_field.custom_object_type_id}_tags_new" + ) + + # Before merge: main still sees the old field/through-table. + self.assertEqual( + CustomObjectTypeField.objects.get(pk=field_pk).name, 'tags_old', + ) + main_tables_before = main_conn.introspection.table_names() + self.assertIn(old_through, main_tables_before) + self.assertNotIn(new_through, main_tables_before) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + # After merge: field name is updated; old through-table is gone, new + # through-table is present, and its rows survived the rename. + self.assertEqual( + CustomObjectTypeField.objects.get(pk=field_pk).name, 'tags_new', + ) + main_tables_after = main_conn.introspection.table_names() + self.assertNotIn( + old_through, main_tables_after, + f'Old through-table {old_through!r} must be gone after rename merge', + ) + self.assertIn( + new_through, main_tables_after, + f'New through-table {new_through!r} must exist after rename merge', + ) + + MergedModel = CustomObjectType.objects.get(pk=cot_pk).get_model() + co_merged = MergedModel.objects.get(pk=co_pk) + self.assertEqual( + set(co_merged.tags_new.values_list('pk', flat=True)), + {site_a.pk, site_b.pk}, + 'M2M values must survive the through-table rename', + ) + # Old accessor must no longer be accessible on the model. + self.assertFalse( + hasattr(co_merged, 'tags_old'), + 'Old field accessor must be gone after rename merge', + ) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + branch.refresh_from_db() + + # Field name restored, through-table restored, rows intact. + self.assertEqual( + CustomObjectTypeField.objects.get(pk=field_pk).name, 'tags_old', + ) + main_tables_reverted = main_conn.introspection.table_names() + self.assertIn( + old_through, main_tables_reverted, + f'Old through-table {old_through!r} must be restored after revert', + ) + self.assertNotIn( + new_through, main_tables_reverted, + f'New through-table {new_through!r} must be gone after revert', + ) + + RevertedModel = CustomObjectType.objects.get(pk=cot_pk).get_model() + co_reverted = RevertedModel.objects.get(pk=co_pk) + self.assertEqual( + set(co_reverted.tags_old.values_list('pk', flat=True)), + {site_a.pk, site_b.pk}, + 'M2M values must survive the round-trip rename → revert', + ) + + # ── single-field DELETE inside branch ───────────────────────────────── + + def test_single_field_delete_merge_and_revert(self): + """ + Delete one field from a COT in a branch (parent COT intact). + + Exercises ``_schema_remove_field`` via the merge replay engine + without the through-table-already-gone shortcut that + ``test_cot_deleted_in_branch_merge_and_revert`` triggers. Covers + the most direct CRUD gap: per-field delete is the common + production case but was previously only exercised transitively. + """ + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + with event_tracking(self.request): + site = Site.objects.create(name='Field-Delete Site', slug='field-delete-site') + cot = CustomObjectType.objects.create(name='field_delete_cot', slug='field-delete-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='keep_me', label='Keep', type='text', + ) + scalar_field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='drop_me', label='Drop', type='integer', + ) + multi_field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='drop_m2m', label='Drop M2M', + type='multiobject', related_object_type=site_ot, + ) + Model = cot.get_model() + co = Model.objects.create(keep_me='hello', drop_me=7) + co.drop_m2m.set([site]) + + cot_pk = cot.pk + co_pk = co.pk + scalar_field_pk = scalar_field.pk + multi_field_pk = multi_field.pk + through_table = multi_field.through_table_name + co_table = cot.get_database_table_name() + + # Sanity: column + through table exist in main before the merge. + with main_conn.cursor() as cursor: + cols_before = { + c.name for c in main_conn.introspection.get_table_description(cursor, co_table) + } + self.assertIn('drop_me', cols_before) + self.assertIn(through_table, main_conn.introspection.table_names()) + + branch = _provision_branch('Field Delete Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + CustomObjectTypeField.objects.get(pk=scalar_field_pk).delete() + CustomObjectTypeField.objects.get(pk=multi_field_pk).delete() + + # Before merge: main still has both fields and their schema. + self.assertTrue(CustomObjectTypeField.objects.filter(pk=scalar_field_pk).exists()) + self.assertTrue(CustomObjectTypeField.objects.filter(pk=multi_field_pk).exists()) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + # Fields gone from ORM; physical column + through table gone too. + self.assertFalse(CustomObjectTypeField.objects.filter(pk=scalar_field_pk).exists()) + self.assertFalse(CustomObjectTypeField.objects.filter(pk=multi_field_pk).exists()) + with main_conn.cursor() as cursor: + cols_after = { + c.name for c in main_conn.introspection.get_table_description(cursor, co_table) + } + self.assertNotIn('drop_me', cols_after, 'drop_me column must be removed from main') + self.assertIn('keep_me', cols_after, 'keep_me column must remain in main') + self.assertNotIn( + through_table, main_conn.introspection.table_names(), + f'Through-table {through_table!r} must be dropped after merge', + ) + + # The retained field's data on the CO must still be readable. + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_main = cot_main.get_model().objects.get(pk=co_pk) + self.assertEqual(co_main.keep_me, 'hello') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + # Both fields are restored, column and through table re-created. + self.assertTrue(CustomObjectTypeField.objects.filter(pk=scalar_field_pk).exists()) + self.assertTrue(CustomObjectTypeField.objects.filter(pk=multi_field_pk).exists()) + with main_conn.cursor() as cursor: + cols_reverted = { + c.name for c in main_conn.introspection.get_table_description(cursor, co_table) + } + self.assertIn('drop_me', cols_reverted, 'drop_me column must be restored after revert') + self.assertIn( + through_table, main_conn.introspection.table_names(), + f'Through-table {through_table!r} must be restored after revert', + ) + + # ── polymorphic OBJECT field merge/revert ───────────────────────────── + + def test_polymorphic_object_field_merge_and_revert(self): + """ + Polymorphic OBJECT field (GenericForeignKey backed by content_type + + object_id columns). None of the existing branching tests cover this + path, and the per-(cot, branch) through-model refactor changed + through-model handling — polymorphic GFK fields have no through but + do create dedicated columns that need branch-aware DDL routing. + """ + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + with event_tracking(self.request): + site = Site.objects.create(name='Poly Site', slug='poly-site') + + branch = _provision_branch('Poly OBJ Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='poly_obj_cot', slug='poly-obj-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='target', + label='Target', + type='object', + is_polymorphic=True, + ) + field.related_object_types.set([site_ot]) + Model = cot.get_model() + co = Model.objects.create( + target_content_type=ContentType.objects.get_for_model(Site), + target_object_id=site.pk, + ) + + cot_pk, field_pk, co_pk = cot.pk, field.pk, co.pk + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_main = cot_main.get_model().objects.get(pk=co_pk) + self.assertEqual(co_main.target_object_id, site.pk) + self.assertEqual( + co_main.target_content_type_id, + ContentType.objects.get_for_model(Site).pk, + ) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + self.assertFalse(CustomObjectTypeField.objects.filter(pk=field_pk).exists()) + + # ── polymorphic MULTIOBJECT field merge/revert ──────────────────────── + + def test_polymorphic_multiobject_field_merge_and_revert(self): + """ + Polymorphic MULTIOBJECT field — through table has + (source_id, content_type_id, object_id) columns. Asserts the + through-table schema lifecycle (create in branch / merge / revert) + AND the M2M data round-trip: rows added via + ``PolymorphicManyToManyManager`` in the branch must appear in main + after merge. + + Data preservation requires three pieces: + - ``PolymorphicManyToManyManager`` fires ``m2m_changed``, triggering + an UPDATE ObjectChange on the parent CO. + - ``CustomObject.serialize_object`` includes polymorphic M2M values + (Django's serializer skips them — the descriptor isn't a real + M2M field on ``_meta``). + - ``CustomObject.deserialize_object`` replays the polymorphic + values via the descriptor after the CO row saves. + """ + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + site_ct = ContentType.objects.get_for_model(Site) + + with event_tracking(self.request): + site_a = Site.objects.create(name='Poly M2M A', slug='poly-m2m-a') + site_b = Site.objects.create(name='Poly M2M B', slug='poly-m2m-b') + + branch = _provision_branch('Poly M2M Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='poly_m2m_cot', slug='poly-m2m-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='targets', + label='Targets', + type='multiobject', + is_polymorphic=True, + ) + field.related_object_types.set([site_ot]) + Model = cot.get_model() + co = Model.objects.create() + co.snapshot() + co.targets.add(site_a) + co.targets.add(site_b) + + cot_pk, field_pk, co_pk = cot.pk, field.pk, co.pk + through_table = field.through_table_name + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + self.assertIn( + through_table, main_conn.introspection.table_names(), + 'Polymorphic through-table must exist in main after merge', + ) + + # Read the through table directly: the polymorphic descriptor's + # query helpers are exercised elsewhere; here we just want to + # confirm the rows landed in main. + with main_conn.cursor() as cursor: + cursor.execute( + f'SELECT object_id FROM "{through_table}" ' + 'WHERE source_id = %s AND content_type_id = %s ORDER BY object_id', + [co_pk, site_ct.pk], + ) + rows = [r[0] for r in cursor.fetchall()] + self.assertEqual( + rows, sorted([site_a.pk, site_b.pk]), + 'Polymorphic M2M rows added in branch must land in main after merge', + ) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + self.assertFalse(CustomObjectTypeField.objects.filter(pk=field_pk).exists()) + self.assertNotIn( + through_table, main_conn.introspection.table_names(), + 'Polymorphic through-table must be dropped after revert', + ) + + # ── self-referential OBJECT field merge/revert ──────────────────────── + + def test_self_referential_object_field_merge_and_revert(self): + """ + Self-referential OBJECT field (FK to the same COT). The generated + model resolves the FK target back to itself in + ``after_model_generation``; we verify that path works across merge. + """ + branch = _provision_branch('Self Ref OBJ Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='self_ref_obj_cot', slug='self-ref-obj-cot') + self_ot = ObjectType.objects.get( + app_label='netbox_custom_objects', + model=cot.get_table_model_name(cot.id).lower(), + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='label', label='Label', type='text', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='parent', label='Parent', + type='object', related_object_type=self_ot, + ) + Model = cot.get_model() + root = Model.objects.create(label='root') + child = Model.objects.create(label='child', parent_id=root.pk) + + cot_pk = cot.pk + root_pk, child_pk = root.pk, child.pk + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + Model = CustomObjectType.objects.get(pk=cot_pk).get_model() + child_main = Model.objects.get(pk=child_pk) + self.assertEqual(child_main.parent_id, root_pk) + self.assertEqual(child_main.label, 'child') + + branch.revert(user=self.user, commit=True) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + + # ── self-referential MULTIOBJECT field merge/revert ─────────────────── + + def test_self_referential_multiobject_field_merge_and_revert(self): + """ + Self-referential MULTIOBJECT — through table with both source and + target FKs pointing at the same dynamic CO model. Exercises the + ``_is_self_referential`` path in + ``MultiObjectFieldType.after_model_generation``. + """ + branch = _provision_branch('Self Ref M2M Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='self_ref_m2m_cot', slug='self-ref-m2m-cot') + self_ot = ObjectType.objects.get( + app_label='netbox_custom_objects', + model=cot.get_table_model_name(cot.id).lower(), + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='label', label='Label', type='text', + ) + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='peers', label='Peers', + type='multiobject', related_object_type=self_ot, + ) + Model = cot.get_model() + a = Model.objects.create(label='a') + b = Model.objects.create(label='b') + c = Model.objects.create(label='c') + a.peers.set([b, c]) + + cot_pk = cot.pk + a_pk, b_pk, c_pk = a.pk, b.pk, c.pk + through_table = field.through_table_name + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + Model = CustomObjectType.objects.get(pk=cot_pk).get_model() + a_main = Model.objects.get(pk=a_pk) + self.assertEqual( + set(a_main.peers.values_list('pk', flat=True)), + {b_pk, c_pk}, + 'Self-referential M2M values must survive merge', + ) + self.assertIn(through_table, main_conn.introspection.table_names()) + + branch.revert(user=self.user, commit=True) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + self.assertNotIn(through_table, main_conn.introspection.table_names()) + + # ── cross-COT FK created entirely inside branch ─────────────────────── + + def test_cross_cot_fk_branch_creates_both_merge_and_revert(self): + """ + Branch creates COT B, then COT A with a FK pointing at B, then a CO + of type A referencing a CO of type B. Merge must apply the COT + creates and field create in an order that respects the FK + dependency, and the data must survive. + """ + branch = _provision_branch('Cross COT Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot_b = CustomObjectType.objects.create(name='cross_b_cot', slug='cross-b-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot_b, name='b_label', label='B Label', type='text', + ) + b_ot = ObjectType.objects.get( + app_label='netbox_custom_objects', + model=cot_b.get_table_model_name(cot_b.id).lower(), + ) + + cot_a = CustomObjectType.objects.create(name='cross_a_cot', slug='cross-a-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot_a, name='a_label', label='A Label', type='text', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot_a, name='b_ref', label='B Ref', + type='object', related_object_type=b_ot, + ) + + BModel = cot_b.get_model() + b_inst = BModel.objects.create(b_label='hello B') + AModel = cot_a.get_model() + a_inst = AModel.objects.create(a_label='hello A', b_ref_id=b_inst.pk) + + cot_a_pk, cot_b_pk = cot_a.pk, cot_b.pk + a_pk, b_pk = a_inst.pk, b_inst.pk + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + AModel = CustomObjectType.objects.get(pk=cot_a_pk).get_model() + BModel = CustomObjectType.objects.get(pk=cot_b_pk).get_model() + self.assertEqual(BModel.objects.get(pk=b_pk).b_label, 'hello B') + a_main = AModel.objects.get(pk=a_pk) + self.assertEqual(a_main.b_ref_id, b_pk) + self.assertEqual(a_main.a_label, 'hello A') + + branch.revert(user=self.user, commit=True) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_a_pk).exists()) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_b_pk).exists()) + + +# ── Concrete test classes (one per merge strategy) ──────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class IterativeBranchingTestCase(BaseBranchingTests, TransactionTestCase): + """Run BaseBranchingTests with the iterative merge strategy.""" + MERGE_STRATEGY = 'iterative' + + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class SquashBranchingTestCase(BaseBranchingTests, TransactionTestCase): + """Run BaseBranchingTests with the squash merge strategy.""" + MERGE_STRATEGY = 'squash' + + +# ── Branch deletion (abandon without merge) ─────────────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class BranchDeletionTestCase(BranchingTestBase, TransactionTestCase): + """ + Deleting a branch without merging must drop the branch's PostgreSQL + schema and must NOT leak any of the branch's COT / field / table state + into main. + + The branch deletion path (``Branch.delete()`` → ``deprovision()`` → + ``DROP SCHEMA ... CASCADE``) bypasses the merge/revert ObjectChange + replay engine. We exercise it here so that the abandon flow stays + correct even though it doesn't go through the same code as merge. + """ + + def test_branch_delete_without_merge_does_not_leak_to_main(self): + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + with event_tracking(self.request): + site = Site.objects.create(name='Abandon Site', slug='abandon-site') + + branch = _provision_branch('Abandon Branch', 'iterative', self.user) + schema_name = branch.schema_name + branch_request = _make_request(self.user) + + # ── branch: create COT + fields + CO that exist ONLY in the branch ─ + branch_cot_pk = None + branch_field_pk = None + branch_co_pk = None + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='abandon_cot', slug='abandon-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='label', + label='Label', + type='text', + ) + multi_field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='multi', label='Multi', + type='multiobject', related_object_type=site_ot, + ) + co = cot.get_model().objects.create(label='only in branch') + co.multi.set([site]) + branch_cot_pk = cot.pk + branch_field_pk = field.pk + branch_co_pk = co.pk + branch_multi_through = multi_field.through_table_name + branch_co_table = cot.get_database_table_name() + + # The branch's schema must exist before we delete it. + with main_conn.cursor() as cursor: + cursor.execute( + 'SELECT 1 FROM information_schema.schemata WHERE schema_name = %s', + [schema_name], + ) + self.assertTrue(cursor.fetchone(), f'Branch schema {schema_name!r} must exist before delete') + + # Main must NOT have any of the branch-only state. + self.assertFalse( + CustomObjectType.objects.filter(pk=branch_cot_pk).exists(), + 'COT created in branch must not be visible in main', + ) + self.assertFalse( + CustomObjectTypeField.objects.filter(pk=branch_field_pk).exists(), + 'Field created in branch must not be visible in main', + ) + main_tables_pre = main_conn.introspection.table_names() + self.assertNotIn(branch_co_table, main_tables_pre) + self.assertNotIn(branch_multi_through, main_tables_pre) + + # ── delete (abandon) the branch ─────────────────────────────────── + branch.delete() + + # Schema must be gone. + with main_conn.cursor() as cursor: + cursor.execute( + 'SELECT 1 FROM information_schema.schemata WHERE schema_name = %s', + [schema_name], + ) + self.assertIsNone( + cursor.fetchone(), + f'Branch schema {schema_name!r} must be dropped after Branch.delete()', + ) + + # Main is still clean — no branch-only state was promoted. + self.assertFalse( + CustomObjectType.objects.filter(pk=branch_cot_pk).exists(), + 'Abandoned-branch COT must not appear in main', + ) + self.assertFalse( + CustomObjectTypeField.objects.filter(pk=branch_field_pk).exists(), + 'Abandoned-branch field must not appear in main', + ) + main_tables_post = main_conn.introspection.table_names() + self.assertNotIn( + branch_co_table, main_tables_post, + 'Branch-only CO table must not appear in main after delete', + ) + self.assertNotIn( + branch_multi_through, main_tables_post, + 'Branch-only through-table must not appear in main after delete', + ) + + # The Branch row itself must be gone. + self.assertFalse( + Branch.objects.filter(pk=branch.pk).exists(), + 'Branch row must be deleted', + ) + + # branch_co_pk is asserted unused but referenced for clarity. + self.assertIsNotNone(branch_co_pk) + + +# ── Sync test ───────────────────────────────────────────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class BranchSyncTestCase(BranchingTestBase, TransactionTestCase): + """ + Test that objects created in main after a branch is provisioned are not + visible in the branch until the branch is synced, and are correctly + available in the branch after sync. + """ + + def test_main_changes_synced_to_branch(self): + """ + A COT, field, and CO created in main *after* a branch is provisioned + must not appear in the branch before sync. After branch.sync() they + must be present and the CO must be retrievable. + + Scenario + -------- + 1. Provision branch (no COT exists yet). + 2. Create COT, field, and CO in main. + 3. Assert they are absent from the branch. + 4. sync() the branch. + 5. Assert they are present in the branch and the CO value is correct. + """ + branch = _provision_branch('Sync Branch', 'iterative', self.user) + request = _make_request(self.user) + + # ── create COT, field, CO in main ───────────────────────────────── + with event_tracking(request): + cot = CustomObjectType.objects.create(name='main_cot', slug='main-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='title', + label='Title', + type='text', + ) + Model = cot.get_model() + co = Model.objects.create(title='main object') + + cot_pk, field_pk, co_pk = cot.pk, field.pk, co.pk + + # ── before sync: absent from branch ─────────────────────────────── + with activate_branch(branch): + self.assertFalse( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must not be in branch before sync', + ) + self.assertFalse( + CustomObjectTypeField.objects.filter(pk=field_pk).exists(), + 'Field must not be in branch before sync', + ) + + # ── sync ────────────────────────────────────────────────────────── + branch.sync(user=self.user, commit=True) + branch.refresh_from_db() + + # ── after sync: present in branch ───────────────────────────────── + with activate_branch(branch): + self.assertTrue( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must be in branch after sync', + ) + self.assertTrue( + CustomObjectTypeField.objects.filter(pk=field_pk).exists(), + 'Field must be in branch after sync', + ) + cot_branch = CustomObjectType.objects.get(pk=cot_pk) + BranchModel = cot_branch.get_model() + co_branch = BranchModel.objects.get(pk=co_pk) + self.assertEqual(co_branch.title, 'main object') + + +# ── Concurrent-edit tests (both main and branch modified before sync/merge) ─── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class ConcurrentEditSyncTestCase(BranchingTestBase, TransactionTestCase): + """ + Sync scenarios where both main and branch accumulate changes before sync(). + + Mirrors netbox-branching's test_sync_m2m_tags_concurrent_changes pattern: + after sync(), main's ObjectChanges are applied on top of whatever the branch + did, so main's post-change state takes precedence for any conflicting record. + """ + + def test_co_values_modified_in_both_sync(self): + """ + CO field values modified in both main and branch before sync. + + Scenario + -------- + 1. Create COT + field 'notes' + two COs in main. + 2. Provision branch (branch sees same COs). + 3. In branch: update shared CO to 'modified in branch'. + 4. In main: update a different CO; create a brand-new CO. + 5. sync() applies main's ObjectChanges to the branch. + + Expected after sync + ------------------- + - Main's new CO is visible in the branch. + - CO modified in main has main's value in the branch. + - Branch's own CO modification is overwritten by main's replay for + the same PK (main wins on sync, same as tag-conflict behaviour). + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='sync_co_cot', slug='sync-co-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='notes', label='Notes', type='text', + ) + Model = cot.get_model() + co_shared = Model.objects.create(notes='original shared') + co_main_only = Model.objects.create(notes='main only original') + + co_shared_pk = co_shared.pk + co_main_only_pk = co_main_only.pk + branch = _provision_branch('Sync CO Both', 'iterative', self.user) + branch_request = _make_request(self.user) + + # ── branch: update the shared CO ───────────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + BM = cot.get_model() + co = BM.objects.get(pk=co_shared_pk) + co.snapshot() + co.notes = 'modified in branch' + co.save() + branch_new = BM.objects.create(notes='new in branch') + branch_new_pk = branch_new.pk + + # ── main: update the other CO; add a new CO ─────────────────────── + with event_tracking(request): + MM = cot.get_model() + co = MM.objects.get(pk=co_main_only_pk) + co.snapshot() + co.notes = 'modified in main' + co.save() + main_new = MM.objects.create(notes='new in main') + main_new_pk = main_new.pk + + # ── sync ────────────────────────────────────────────────────────── + branch.sync(user=self.user, commit=True) + + with activate_branch(branch): + SyncedCOT = CustomObjectType.objects.get(pk=cot.pk) + SM = SyncedCOT.get_model() + + # CO modified in main must reflect main's value. + self.assertEqual(SM.objects.get(pk=co_main_only_pk).notes, 'modified in main') + # CO created in main must be visible in branch after sync. + self.assertTrue(SM.objects.filter(pk=main_new_pk).exists(), + 'CO created in main must appear in branch after sync') + # CO created in branch was not deleted by sync. + self.assertTrue(SM.objects.filter(pk=branch_new_pk).exists(), + 'CO created in branch must still exist after sync') + + def test_field_rename_in_branch_co_add_in_main_sync(self): + """ + Field renamed inside a branch; new CO added in main (no rename in main). + + Scenario + -------- + 1. Create COT + field 'alpha' + CO in main. + 2. Provision branch. + 3. In branch: rename 'alpha' → 'branch_alpha'; create a CO. + 4. In main: create a new CO using the original field name 'alpha'. + 5. sync() replays main's CO-create on top of the branch. + + Expected after sync + ------------------- + - Branch retains the renamed field ('branch_alpha'). + - CO created in main is present in branch. + - CO created in branch still exists. + + This exercises the schema-mismatch path: main's CO ObjectChange carries + the original field key 'alpha' while the branch schema uses 'branch_alpha'. + The CO data from main is applied by the branching engine regardless of the + column name divergence (branching replays at the data layer). + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='sync_rename_cot', slug='sync-rename-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='alpha', label='Alpha', type='text', + ) + cot.get_model().objects.create(alpha='original') + + branch = _provision_branch('Sync Rename Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + # ── branch: rename alpha → branch_alpha; create CO ──────────────── + with activate_branch(branch), event_tracking(branch_request): + field = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + field.snapshot() + field.name = 'branch_alpha' + field.label = 'Branch Alpha' + field.save() + BM = cot.get_model() + branch_new = BM.objects.create(branch_alpha='new in branch') + branch_new_pk = branch_new.pk + + # ── main: add a CO (no field rename) ────────────────────────────── + with event_tracking(request): + cot.get_model().objects.create(alpha='new in main') + + # ── sync ────────────────────────────────────────────────────────── + branch.sync(user=self.user, commit=True) + + with activate_branch(branch): + cot_b = CustomObjectType.objects.get(pk=cot.pk) + field_b = CustomObjectTypeField.objects.get(custom_object_type=cot_b) + # Branch field rename must be preserved (main did not rename). + self.assertEqual(field_b.name, 'branch_alpha', + 'Branch field rename must be preserved after sync') + BM = cot_b.get_model() + self.assertTrue(BM.objects.filter(pk=branch_new_pk).exists(), + 'CO created in branch must survive sync') + + def test_concurrent_field_rename_sync_no_crash(self): + """ + Field renamed to different names in both main and branch before sync. + + The same CustomObjectTypeField PK was modified in both schemas. + _schema_alter_field detects the conflict (neither the original 'alpha' + column nor the target 'main_alpha' column exists in the branch), looks up + the live column name in the branch ('branch_alpha'), and renames it to + 'main_alpha' to converge the branch schema on main's post-sync state. + + Scenario + -------- + 1. Create COT + field 'alpha' in main. + 2. Provision branch. + 3. Branch renames 'alpha' → 'branch_alpha'. + 4. Main renames 'alpha' → 'main_alpha'. + 5. sync() applies main's rename ObjectChange to the branch. + _schema_alter_field resolves the conflict: 'branch_alpha' → 'main_alpha'. + + Expected + -------- + - sync() completes without raising a DB error. + - Branch physical column is 'main_alpha' (converged to main's rename). + - 'branch_alpha' and 'alpha' columns are absent from the branch table. + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='confl_sync_cot', slug='confl-sync-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='alpha', label='Alpha', type='text', + ) + + branch = _provision_branch('Conflict Sync Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + # ── branch: rename alpha → branch_alpha ─────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + f.snapshot() + f.name = 'branch_alpha' + f.label = 'Branch Alpha' + f.save() + + # ── main: rename alpha → main_alpha ─────────────────────────────── + with event_tracking(request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + f.name = 'main_alpha' + f.label = 'Main Alpha' + f.save() + + # ── sync — must not raise. Let any failure propagate with its + # original traceback rather than catching to ``self.fail`` (which + # would flatten the stack). + branch.sync(user=self.user, commit=True) + + branch.refresh_from_db() + + # Branch column should now be 'main_alpha' — the conflict was resolved by + # renaming the branch's live 'branch_alpha' column to main's target name. + branch_conn = connections[branch.connection_name] + with branch_conn.cursor() as cursor: + branch_cols = { + col.name + for col in branch_conn.introspection.get_table_description( + cursor, cot.get_database_table_name(), + ) + } + self.assertIn('main_alpha', branch_cols, 'Branch column must converge to main_alpha after sync') + self.assertNotIn('branch_alpha', branch_cols, 'branch_alpha column must be gone after sync') + self.assertNotIn('alpha', branch_cols, 'alpha column must be gone after sync') + + +# ── Concurrent-edit merge tests ─────────────────────────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class BaseConcurrentEditMergeTests(BranchingTestBase): + """ + Merge scenarios where both main and branch accumulate changes before merge(). + + Subclasses must: + - set ``MERGE_STRATEGY`` to an iterative or squash strategy string + - also inherit from ``TransactionTestCase`` + """ + + MERGE_STRATEGY = None + + def test_field_rename_in_branch_co_changes_merge(self): + """ + Field renamed inside a branch; COs added/updated in branch; main adds a CO. + Merge brings the branch rename and CO changes into main. + + Scenario + -------- + 1. Create COT + field 'alpha' + CO in main. + 2. Provision branch. + 3. In branch: rename 'alpha' → 'beta'; update the existing CO; create a CO. + 4. In main: add a CO (field name 'alpha', no rename). + 5. merge() → revert(). + + Expected after merge + -------------------- + - Field name in main is 'beta'. + - Existing CO has the branch-updated value. + - CO created in branch is present in main. + + Expected after revert + --------------------- + - Field name is 'alpha' again. + - CO created in branch is gone. + - Existing CO has its original value. + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='merge_rename_cot', slug='merge-rename-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='alpha', label='Alpha', type='text', + ) + Model = cot.get_model() + co_existing = Model.objects.create(alpha='original') + + co_existing_pk = co_existing.pk + branch = _provision_branch('Merge Rename Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # ── branch: rename; update CO; create CO ────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + field = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + field.snapshot() + field.name = 'beta' + field.label = 'Beta' + field.save() + BM = cot.get_model() + co = BM.objects.get(pk=co_existing_pk) + co.snapshot() + co.beta = 'updated in branch' + co.save() + branch_new = BM.objects.create(beta='new in branch') + branch_new_pk = branch_new.pk + + # ── main: add a CO (no schema change) ───────────────────────────── + with event_tracking(request): + main_new = cot.get_model().objects.create(alpha='new in main') + main_new_pk = main_new.pk + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + MergedModel = cot.get_model() + field_main = CustomObjectTypeField.objects.get(custom_object_type=cot) + self.assertEqual(field_main.name, 'beta', 'Field must be "beta" in main after merge') + self.assertEqual(MergedModel.objects.get(pk=co_existing_pk).beta, 'updated in branch') + self.assertTrue(MergedModel.objects.filter(pk=branch_new_pk).exists(), + 'CO created in branch must be in main after merge') + self.assertTrue(MergedModel.objects.filter(pk=main_new_pk).exists(), + 'CO added in main must still be present after merge') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + field_reverted = CustomObjectTypeField.objects.get(custom_object_type=cot) + self.assertEqual(field_reverted.name, 'alpha', 'Field must be "alpha" after revert') + RevertedModel = cot.get_model() + self.assertEqual(RevertedModel.objects.get(pk=co_existing_pk).alpha, 'original') + self.assertFalse(RevertedModel.objects.filter(pk=branch_new_pk).exists(), + 'CO created in branch must be gone after revert') + + def test_co_values_modified_in_both_merge(self): + """ + CO values modified in both main and branch before merge. + Branch changes win because merge applies branch ObjectChanges to main. + + Scenario + -------- + 1. Create COT + field 'notes' + shared CO in main. + 2. Provision branch. + 3. Branch updates shared CO to 'modified in branch'; creates a CO. + 4. Main creates a separate CO. + 5. merge() → revert(). + + Expected after merge: branch CO changes are in main, main CO preserved. + Expected after revert: shared CO back to original, branch CO gone. + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='merge_co_cot', slug='merge-co-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='notes', label='Notes', type='text', + ) + Model = cot.get_model() + co_shared = Model.objects.create(notes='original') + + co_shared_pk = co_shared.pk + branch = _provision_branch('Merge CO Both', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # ── branch ──────────────────────────────────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + BM = cot.get_model() + co = BM.objects.get(pk=co_shared_pk) + co.snapshot() + co.notes = 'modified in branch' + co.save() + branch_new = BM.objects.create(notes='new in branch') + branch_new_pk = branch_new.pk + + # ── main ────────────────────────────────────────────────────────── + with event_tracking(request): + main_new = cot.get_model().objects.create(notes='new in main') + main_new_pk = main_new.pk + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + MM = cot.get_model() + self.assertEqual(MM.objects.get(pk=co_shared_pk).notes, 'modified in branch') + self.assertTrue(MM.objects.filter(pk=branch_new_pk).exists()) + self.assertTrue(MM.objects.filter(pk=main_new_pk).exists(), + 'CO added to main must survive the merge') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + RM = cot.get_model() + self.assertEqual(RM.objects.get(pk=co_shared_pk).notes, 'original') + self.assertFalse(RM.objects.filter(pk=branch_new_pk).exists()) + + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class IterativeConcurrentEditMergeTestCase(BaseConcurrentEditMergeTests, TransactionTestCase): + """Run BaseConcurrentEditMergeTests with the iterative merge strategy.""" + MERGE_STRATEGY = 'iterative' + + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class SquashConcurrentEditMergeTestCase(BaseConcurrentEditMergeTests, TransactionTestCase): + """Run BaseConcurrentEditMergeTests with the squash merge strategy.""" + MERGE_STRATEGY = 'squash' + + +# ── Sequential multi-rename tests ───────────────────────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class SequentialRenameTestCase(BranchingTestBase, TransactionTestCase): + """ + Tests for sequential field renames (A→B→C) in a branch with CO changes at + each step, plus independent changes in main. + + Exercises the iterative ObjectChange replay order: the rename chain must be + applied in the right sequence so that each CO update sees the correct column + name at merge time. + + Run with both iterative and squash strategies to verify that squash correctly + collapses the A→B→C chain to a single A→C alter. + """ + + MERGE_STRATEGY = 'iterative' + + def _run_sequential_rename_merge(self, cot_name, cot_slug): + """ + Shared implementation for the sequential rename merge test. + + Branch: rename alpha→beta (update+create CO), rename beta→gamma (update+create CO). + Main: add a new independent field + CO (no rename of alpha). + merge() then revert(). + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name=cot_name, slug=cot_slug) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='alpha', label='Alpha', type='text', + ) + Model = cot.get_model() + co_original = Model.objects.create(alpha='original value') + + co_original_pk = co_original.pk + branch = _provision_branch(f'{cot_name} branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # ── branch: alpha → beta; update CO; create CO ──────────────────── + with activate_branch(branch), event_tracking(branch_request): + field = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + field.snapshot() + field.name = 'beta' + field.label = 'Beta' + field.save() + BM = cot.get_model() + co = BM.objects.get(pk=co_original_pk) + co.snapshot() + co.beta = 'after rename to beta' + co.save() + co_at_beta = BM.objects.create(beta='created at beta') + co_at_beta_pk = co_at_beta.pk + + # ── branch: beta → gamma; update CO; create CO ──────────────────── + with activate_branch(branch), event_tracking(branch_request): + field = CustomObjectTypeField.objects.get(custom_object_type=cot, name='beta') + field.snapshot() + field.name = 'gamma' + field.label = 'Gamma' + field.save() + BM = cot.get_model() + co = BM.objects.get(pk=co_original_pk) + co.snapshot() + co.gamma = 'after rename to gamma' + co.save() + co_at_gamma = BM.objects.create(gamma='created at gamma') + co_at_gamma_pk = co_at_gamma.pk + + # ── main: add a new independent field + CO ──────────────────────── + with event_tracking(request): + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='extra', label='Extra', type='text', + ) + co_main = cot.get_model().objects.create(alpha='main added', extra='extra val') + co_main_pk = co_main.pk + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + field_names = {f.name for f in CustomObjectTypeField.objects.filter(custom_object_type=cot)} + self.assertIn('gamma', field_names, 'Final field name must be "gamma" after merge') + self.assertNotIn('alpha', field_names, '"alpha" must be absent after merge') + self.assertNotIn('beta', field_names, '"beta" must be absent after merge') + self.assertIn('extra', field_names, '"extra" field from main must be present after merge') + + MergedModel = cot.get_model() + self.assertEqual(MergedModel.objects.get(pk=co_original_pk).gamma, 'after rename to gamma') + self.assertTrue(MergedModel.objects.filter(pk=co_at_beta_pk).exists(), + 'CO created at beta step must survive merge') + self.assertTrue(MergedModel.objects.filter(pk=co_at_gamma_pk).exists(), + 'CO created at gamma step must survive merge') + self.assertTrue(MergedModel.objects.filter(pk=co_main_pk).exists(), + 'CO added in main must survive merge') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + field_names_r = {f.name for f in CustomObjectTypeField.objects.filter(custom_object_type=cot)} + self.assertIn('alpha', field_names_r, '"alpha" must be restored after revert') + self.assertNotIn('gamma', field_names_r, '"gamma" must be gone after revert') + + RevertedModel = cot.get_model() + self.assertEqual(RevertedModel.objects.get(pk=co_original_pk).alpha, 'original value', + 'Original CO value must be restored after revert') + self.assertFalse(RevertedModel.objects.filter(pk=co_at_beta_pk).exists()) + self.assertFalse(RevertedModel.objects.filter(pk=co_at_gamma_pk).exists()) + + def test_sequential_renames_alpha_beta_gamma_merge(self): + """Field renamed A→B→C in branch with CO changes at each step; merge + revert.""" + self._run_sequential_rename_merge('seq_iter_cot', 'seq-iter-cot') + + def test_sequential_renames_both_sides_sync(self): + """ + Branch renames A→B→C while main renames A→D. + Both schemas independently rename the same field to different names. + + After sync(), main's rename (A→D) is applied on top of the branch's + state. Because 'alpha' no longer exists in the branch (it was renamed + to 'gamma' via beta), _schema_alter_field detects the conflict, looks up + the live column name in the branch ('gamma'), and renames it to 'delta' + to converge the branch schema on main's post-sync state. + + Scenario + -------- + 1. Create COT + field 'alpha' + CO in main. + 2. Provision branch. + 3. Branch: alpha→beta (update CO), beta→gamma (update CO, add CO). + 4. Main: alpha→delta (update CO, add CO). + 5. sync(): apply main's changes to branch — column converges to 'delta'. + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='seq_sync_cot', slug='seq-sync-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='alpha', label='Alpha', type='text', + ) + co = cot.get_model().objects.create(alpha='original') + + co_pk = co.pk + branch = _provision_branch('Seq Sync Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # ── branch: alpha → beta → gamma ────────────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + f.snapshot() + f.name = 'beta' + f.label = 'Beta' + f.save() + BM = cot.get_model() + co_b = BM.objects.get(pk=co_pk) + co_b.snapshot() + co_b.beta = 'at beta' + co_b.save() + + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='beta') + f.snapshot() + f.name = 'gamma' + f.label = 'Gamma' + f.save() + BM = cot.get_model() + co_g = BM.objects.get(pk=co_pk) + co_g.snapshot() + co_g.gamma = 'at gamma' + co_g.save() + BM.objects.create(gamma='new in branch') + + # ── main: alpha → delta ─────────────────────────────────────────── + with event_tracking(request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + f.name = 'delta' + f.label = 'Delta' + f.save() + MM = cot.get_model() + co_m = MM.objects.get(pk=co_pk) + co_m.snapshot() + co_m.delta = 'updated in main' + co_m.save() + MM.objects.create(delta='main new') + + # ── sync — let any failure propagate with its original traceback ─── + branch.sync(user=self.user, commit=True) + + branch.refresh_from_db() + + # _schema_alter_field resolved the conflict by looking up the live column + # ('gamma') in the branch and renaming it to main's target name ('delta'). + branch_conn = connections[branch.connection_name] + with branch_conn.cursor() as cursor: + branch_cols = { + col.name + for col in branch_conn.introspection.get_table_description( + cursor, cot.get_database_table_name(), + ) + } + self.assertIn('delta', branch_cols, 'Branch column must converge to delta after sync') + self.assertNotIn('gamma', branch_cols, 'gamma column must be gone after sync') + self.assertNotIn('beta', branch_cols, 'beta column must be gone after sync') + self.assertNotIn('alpha', branch_cols, 'alpha column must be gone after sync') + + def test_sequential_renames_both_sides_merge(self): + """ + Branch renames A→B→C; main renames A→D independently. + merge() applies branch's rename chain to main. + + When merging the first branch rename (alpha→beta) into main, 'alpha' no + longer exists in main (it was renamed to 'delta') and 'beta' doesn't exist + either. _schema_alter_field detects the conflict, looks up the live column + in main ('delta'), and renames it to 'beta'. The second branch rename + (beta→gamma) then finds 'beta' in main and renames it normally to 'gamma'. + + Expected after merge: main's physical column is 'gamma'. + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='seq_merge_cot', slug='seq-merge-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='alpha', label='Alpha', type='text', + ) + co = cot.get_model().objects.create(alpha='original') + + co_pk = co.pk + branch = _provision_branch('Seq Merge Conflict Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # ── branch: alpha → beta → gamma ────────────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + f.snapshot() + f.name = 'beta' + f.label = 'Beta' + f.save() + BM = cot.get_model() + co_b = BM.objects.get(pk=co_pk) + co_b.snapshot() + co_b.beta = 'at beta' + co_b.save() + + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='beta') + f.snapshot() + f.name = 'gamma' + f.label = 'Gamma' + f.save() + BM = cot.get_model() + co_g = BM.objects.get(pk=co_pk) + co_g.snapshot() + co_g.gamma = 'at gamma' + co_g.save() + BM.objects.create(gamma='new in branch') + + # ── main: alpha → delta ─────────────────────────────────────────── + with event_tracking(request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + f.name = 'delta' + f.label = 'Delta' + f.save() + + # ── merge — let any failure propagate with its original traceback ── + branch.merge(user=self.user, commit=True) + + branch.refresh_from_db() + + # _schema_alter_field resolved the conflict for the first branch rename + # (alpha→beta): it found 'delta' (main's live column) and renamed it to + # 'beta'. The second rename (beta→gamma) then proceeded normally. + # Main's final physical column should be 'gamma'. + with main_conn.cursor() as cursor: + main_cols = { + col.name + for col in main_conn.introspection.get_table_description( + cursor, cot.get_database_table_name(), + ) + } + self.assertIn('gamma', main_cols, 'Main column must be gamma after merge') + self.assertNotIn('delta', main_cols, 'delta column must be gone after merge') + self.assertNotIn('beta', main_cols, 'beta column must be gone after merge') + self.assertNotIn('alpha', main_cols, 'alpha column must be gone after merge') + + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class SequentialRenameSquashTestCase(SequentialRenameTestCase, TransactionTestCase): + """Run SequentialRenameTestCase with the squash merge strategy.""" + MERGE_STRATEGY = 'squash' + + def test_sequential_renames_alpha_beta_gamma_merge(self): + self._run_sequential_rename_merge('seq_squash_cot', 'seq-squash-cot') + + +# ── Missing field-type coverage (iterative only) ────────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class MissingFieldTypesTestCase(BranchingTestBase, TransactionTestCase): + """ + Field types that ``test_comprehensive_merge_and_revert`` doesn't cover: + longtext, date (separate from datetime), URL, JSON, multiselect. + + Iterative only — strategy-specific bugs in these field types would still + surface in the comprehensive squash test once they're added there. This + standalone class keeps the round-trip times manageable while filling out + the field-type matrix. + """ + + def test_merge_and_revert_for_extra_field_types(self): + with event_tracking(self.request): + choice_set = CustomFieldChoiceSet.objects.create( + name='Multi Statuses', + extra_choices=[['a', 'A'], ['b', 'B'], ['c', 'C']], + ) + + branch = _provision_branch('Extra Types Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='extra_types_cot', slug='extra-types-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='longtext_field', label='Long Text', type='longtext', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='date_field', label='Date', type='date', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='url_field', label='URL', type='url', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='json_field', label='JSON', type='json', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='multiselect_field', label='Multi', + type='multiselect', choice_set=choice_set, + ) + Model = cot.get_model() + co = Model.objects.create( + longtext_field='line1\nline2', + date_field=datetime.date(2026, 5, 21), + url_field='https://example.com/path', + json_field={'k': 'v', 'n': 1, 'list': [1, 2, 3]}, + multiselect_field=['a', 'c'], + ) + + cot_pk, co_pk = cot.pk, co.pk + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + Model = CustomObjectType.objects.get(pk=cot_pk).get_model() + co_main = Model.objects.get(pk=co_pk) + self.assertEqual(co_main.longtext_field, 'line1\nline2') + self.assertEqual(co_main.date_field, datetime.date(2026, 5, 21)) + self.assertEqual(co_main.url_field, 'https://example.com/path') + self.assertEqual(co_main.json_field, {'k': 'v', 'n': 1, 'list': [1, 2, 3]}) + self.assertEqual(sorted(co_main.multiselect_field), ['a', 'c']) + + branch.revert(user=self.user, commit=True) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + + +# ── Field attribute changes & COT update (iterative only) ───────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class FieldAttributeChangesTestCase(BranchingTestBase, TransactionTestCase): + """ + Application-layer field attribute changes that the existing tests don't + cover individually: COT-level updates, field type change, primary swap, + and required toggle. Iterative only — these don't hit strategy-specific + code paths. + """ + + def test_cot_metadata_update_merge(self): + """COT name / verbose_name / description / version edited inside branch.""" + with event_tracking(self.request): + cot = CustomObjectType.objects.create( + name='meta_cot', slug='meta-cot', + verbose_name='Original Name', description='original', + version='1.0.0', + ) + + cot_pk = cot.pk + branch = _provision_branch('Meta Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + branch_cot = CustomObjectType.objects.get(pk=cot_pk) + branch_cot.snapshot() + branch_cot.verbose_name = 'Updated Name' + branch_cot.description = 'updated description' + branch_cot.version = '1.1.0' + branch_cot.save() + + # Main hasn't seen the change yet. + cot.refresh_from_db() + self.assertEqual(cot.verbose_name, 'Original Name') + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + cot.refresh_from_db() + self.assertEqual(cot.verbose_name, 'Updated Name') + self.assertEqual(cot.description, 'updated description') + self.assertEqual(cot.version, '1.1.0') + + def test_field_type_change_text_to_integer_merge(self): + """text → integer field type change across merge. + + The CO has a value that's parseable as both; the column type + change replaces ``VARCHAR`` with ``INTEGER`` via ``alter_field``. + """ + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='retype_cot', slug='retype-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='value', label='Value', type='text', + ) + Model = cot.get_model() + # Use a value that parses as both text and integer so the + # USING cast applied by alter_field can succeed. + co = Model.objects.create(value='42') + + cot_pk, field_pk, co_pk = cot.pk, field.pk, co.pk + branch = _provision_branch('Retype Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(pk=field_pk) + f.snapshot() + f.type = 'integer' + f.save() + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + field_main = CustomObjectTypeField.objects.get(pk=field_pk) + self.assertEqual(field_main.type, 'integer') + + # PostgreSQL column type must be integer. + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_table = cot_main.get_database_table_name() + with main_conn.cursor() as cursor: + cursor.execute( + 'SELECT data_type FROM information_schema.columns ' + 'WHERE table_name = %s AND column_name = %s', + [co_table, 'value'], + ) + data_type = cursor.fetchone()[0] + self.assertEqual(data_type, 'integer') + + # CO value survived the cast. + co_main = cot_main.get_model().objects.get(pk=co_pk) + self.assertEqual(co_main.value, 42) + + def test_primary_field_swap_merge(self): + """Switch which field is ``primary``; __str__ must follow.""" + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='primary_cot', slug='primary-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='code', label='Code', type='text', primary=True, + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='title', label='Title', type='text', primary=False, + ) + Model = cot.get_model() + co = Model.objects.create(code='ABC', title='Widget') + self.assertEqual(str(co), 'ABC') + + cot_pk = cot.pk + co_pk = co.pk + branch = _provision_branch('Primary Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + f_code = CustomObjectTypeField.objects.get(custom_object_type=cot, name='code') + f_code.snapshot() + f_code.primary = False + f_code.save() + f_title = CustomObjectTypeField.objects.get(custom_object_type=cot, name='title') + f_title.snapshot() + f_title.primary = True + f_title.save() + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_main = cot_main.get_model().objects.get(pk=co_pk) + # __str__ now follows the newly-primary 'title' field. + self.assertEqual(str(co_main), 'Widget') + + def test_field_required_toggle_merge(self): + """Toggle a field's required flag from False to True across merge. + + ``required`` is a form-layer attribute in this plugin — every field + constructor in ``field_types.py`` hardcodes ``null=True, blank=True`` + on the model field, so the DB column stays nullable regardless of + ``required``. This test pins both halves of that contract: + ``required=True`` does survive the merge as an ORM attribute (form + validation will reject empty values), but the underlying column does + NOT become NOT NULL. + """ + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='required_cot', slug='required-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='note', label='Note', type='text', + required=False, + ) + + cot_pk, field_pk = cot.pk, field.pk + branch = _provision_branch('Required Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(pk=field_pk) + f.snapshot() + f.required = True + f.save() + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + self.assertTrue( + CustomObjectTypeField.objects.get(pk=field_pk).required, + 'required=True ORM flag must survive the merge', + ) + + # DB column stays nullable — required is enforced at the form layer only. + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_table = cot_main.get_database_table_name() + with main_conn.cursor() as cursor: + cursor.execute( + 'SELECT is_nullable FROM information_schema.columns ' + 'WHERE table_name = %s AND column_name = %s', + [co_table, 'note'], + ) + is_nullable = cursor.fetchone()[0] + self.assertEqual( + is_nullable, 'YES', + 'required=True must NOT produce NOT NULL — this plugin enforces ' + 'required at the form layer only (field_types.py hardcodes null=True).', + ) + + +# ── Tags + journal entries survive merge ────────────────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class TagsAndJournalTestCase(BranchingTestBase, TransactionTestCase): + """ + Tags use a separate code path in ``CustomObject.deserialize_object`` via + the ``is_taggable`` branch. Journal entries are NetBox infrastructure + used by NetBoxModel subclasses. Neither is exercised by the rest of the + branching suite. + """ + + def test_co_with_tags_survives_merge(self): + from extras.models import Tag + + with event_tracking(self.request): + tag_a = Tag.objects.create(name='Branch Tag A', slug='branch-tag-a') + tag_b = Tag.objects.create(name='Branch Tag B', slug='branch-tag-b') + cot = CustomObjectType.objects.create(name='tagged_cot', slug='tagged-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='label', label='Label', type='text', + ) + + cot_pk = cot.pk + branch = _provision_branch('Tagged Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + Model = cot.get_model() + co = Model.objects.create(label='tagged') + co.tags.set([tag_a, tag_b]) + + co_pk = co.pk + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + Model = CustomObjectType.objects.get(pk=cot_pk).get_model() + co_main = Model.objects.get(pk=co_pk) + self.assertEqual( + set(co_main.tags.values_list('name', flat=True)), + {'Branch Tag A', 'Branch Tag B'}, + 'Tags assigned in branch must appear on the merged CO', + ) + + def test_co_with_journal_entry_survives_merge(self): + """Journal entries created against a CO in branch arrive in main on merge.""" + from extras.models import JournalEntry + + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='journal_cot', slug='journal-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='label', label='Label', type='text', + ) + + cot_pk = cot.pk + branch = _provision_branch('Journal Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + Model = cot.get_model() + co = Model.objects.create(label='journaled') + JournalEntry.objects.create( + assigned_object=co, + created_by=self.user, + kind='info', + comments='Note added in branch', + ) + + co_pk = co.pk + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + Model = CustomObjectType.objects.get(pk=cot_pk).get_model() + co_ct = ContentType.objects.get_for_model(Model) + entries = JournalEntry.objects.filter( + assigned_object_type=co_ct, assigned_object_id=co_pk, + ) + self.assertEqual( + list(entries.values_list('comments', flat=True)), + ['Note added in branch'], + ) + + +# ── ChoiceSet lifecycle, search_weight, sync-then-merge ─────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class ChoiceSetSearchLifecycleTestCase(BranchingTestBase, TransactionTestCase): + """Misc lifecycle gaps: ChoiceSet mutation, search_weight changes, + sync→edit→merge chains.""" + + def test_choice_set_choices_mutated_in_branch_merge(self): + """Add a new choice to a ChoiceSet inside a branch; merge to main.""" + with event_tracking(self.request): + cs = CustomFieldChoiceSet.objects.create( + name='Mutable Choices', + extra_choices=[['x', 'X'], ['y', 'Y']], + ) + cot = CustomObjectType.objects.create(name='cs_cot', slug='cs-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='c', label='Choice', + type='select', choice_set=cs, + ) + + cs_pk = cs.pk + branch = _provision_branch('CS Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cs_branch = CustomFieldChoiceSet.objects.get(pk=cs_pk) + cs_branch.snapshot() + cs_branch.extra_choices = [['x', 'X'], ['y', 'Y'], ['z', 'Z']] + cs_branch.save() + Model = cot.get_model() + Model.objects.create(c='z') # uses the new choice + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + cs_main = CustomFieldChoiceSet.objects.get(pk=cs_pk) + keys = {pair[0] for pair in cs_main.extra_choices} + self.assertIn('z', keys, 'New choice must be present in main after merge') + + def test_field_search_weight_change_merge(self): + """Changing ``search_weight`` is a non-DDL field update. It must + survive merge as an ORM-level change; reindexing happens via + ``ReindexCustomObjectTypeJob`` which runs out-of-band.""" + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='sw_cot', slug='sw-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='label', label='Label', + type='text', search_weight=10, + ) + + field_pk = field.pk + branch = _provision_branch('SW Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(pk=field_pk) + f.snapshot() + f.search_weight = 100 + f.save() + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + self.assertEqual( + CustomObjectTypeField.objects.get(pk=field_pk).search_weight, 100, + ) + + def test_sync_then_branch_edit_then_merge_lifecycle(self): + """ + Full lifecycle: main creates state → sync pulls it into branch → + branch edits a CO → merge brings the edit back. No standalone + test covers state persistence across sync and merge in one branch. + """ + branch = _provision_branch('Lifecycle Branch', 'iterative', self.user) + + # Main creates a COT + CO *after* the branch was provisioned. + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='lifecycle_cot', slug='lifecycle-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='label', label='Label', type='text', + ) + Model = cot.get_model() + co = Model.objects.create(label='initial') + + co_pk = co.pk + cot_pk = cot.pk + + # Sync the branch so it sees the COT+CO. + branch.sync(user=self.user, commit=True) + branch.refresh_from_db() + + # Branch edits the CO. + branch_request = _make_request(self.user) + with activate_branch(branch), event_tracking(branch_request): + BranchModel = CustomObjectType.objects.get(pk=cot_pk).get_model() + branch_co = BranchModel.objects.get(pk=co_pk) + branch_co.snapshot() + branch_co.label = 'edited after sync' + branch_co.save() + + # Main hasn't seen the edit yet. + co.refresh_from_db() + self.assertEqual(co.label, 'initial') + + # Merge. + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + co.refresh_from_db() + self.assertEqual( + co.label, 'edited after sync', + 'Edit made after sync must propagate to main on merge', + ) + + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class GraphQLBranchIsolationTestCase(BranchingTestBase, TransactionTestCase): + """ + GraphQL resolves against whichever branch netbox-branching activated for the + request (X-NetBox-Branch header, ``?_branch=``, or the active_branch cookie), + exactly like the REST API and the UI; with no branch active it is main. The + schema is built per-branch, so a branch's custom object types appear only for + requests scoped to that branch and never leak into main. + """ + + def setUp(self): + super().setUp() + from netbox_custom_objects.graphql import live as live_module + self.live = live_module + live_module.reset_cache() + self.addCleanup(live_module.reset_cache) + # The startup guard returns True during the test run; patch it off so the + # live schema machinery runs exactly as it does in production. + patcher = mock.patch( + 'netbox_custom_objects.CustomObjectsPluginConfig.' + 'should_skip_dynamic_model_creation', + return_value=False, + ) + patcher.start() + self.addCleanup(patcher.stop) + + def test_schema_signature_is_branch_aware(self): + # The signature drives rebuilds; computed under a branch it must see that + # branch's custom object types (so the branch gets its own schema), while + # main — unaffected by the unmerged branch — keeps its own. + main_sig = self.live.schema_signature() + + branch = _provision_branch('GraphQL Sig Branch', user=self.user) + branch_request = _make_request(self.user) + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create( + name='branch_only_sig', slug='branch-only-sig' + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='name', label='Name', type='text', + primary=True, required=True, + ) + self.assertNotEqual(main_sig, self.live.schema_signature()) + + # Back on main, the signature is unchanged by the unmerged branch. + self.assertEqual(main_sig, self.live.schema_signature()) + + def test_live_schema_reflects_active_branch(self): + # With a branch active (as the X-NetBox-Branch header path leaves it), the + # schema reflects that branch's custom object types; main, unaffected by the + # unmerged branch, does not. + baseline = self.live.get_live_schema() + self.assertIsNotNone(baseline) + self.assertNotIn('custom_objects_branch_only', str(baseline)) + + branch = _provision_branch('GraphQL Branch Schema', user=self.user) + branch_request = _make_request(self.user) + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create( + name='branch only', slug='branch-only' + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='name', label='Name', type='text', + primary=True, required=True, + ) + # The branch's own schema gains the branch-only type... + self.assertIn( + 'custom_objects_branch_only', str(self.live.get_live_schema()) + ) + + # ...but main never does (the branch isn't merged). + self.assertNotIn( + 'custom_objects_branch_only', str(self.live.get_live_schema()) + ) + + def test_branch_deletion_evicts_cached_schema(self): + # Building a branch's schema caches it under the branch pk; deleting the + # branch must evict that entry so it doesn't leak for the process lifetime. + branch = _provision_branch('GraphQL Evict', user=self.user) + branch_request = _make_request(self.user) + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='evict me', slug='evict-me') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='name', label='Name', type='text', + primary=True, required=True, + ) + self.assertIsNotNone(self.live.get_live_schema()) + + # The branch's schema is now cached under its pk... + self.assertIn(branch.pk, self.live._schema_cache) + + # ...and deleting the branch evicts it. + branch_pk = branch.pk + branch.delete() + self.assertNotIn(branch_pk, self.live._schema_cache) + + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +@override_settings(LOGIN_REQUIRED=True) +class GraphQLBranchEndpointTestCase(BranchingTestBase, TransactionTestCase): + """ + End-to-end against the real ``/graphql/`` endpoint: it serves whichever branch + netbox-branching activated for the request (the ``X-NetBox-Branch`` header or the + ``?_branch=`` query param) and main otherwise — correct schema AND data in each — + and a schema change made inside a branch never leaks into main (the branch is not + merged). + """ + + def setUp(self): + super().setUp() + from netbox_custom_objects.graphql import live as live_module + self.live = live_module + live_module.reset_cache() + self.addCleanup(live_module.reset_cache) + # The startup guard returns True during the test run; patch it off so the + # live schema machinery runs exactly as it does in production. + patcher = mock.patch( + 'netbox_custom_objects.CustomObjectsPluginConfig.' + 'should_skip_dynamic_model_creation', + return_value=False, + ) + patcher.start() + self.addCleanup(patcher.stop) + + # Superuser + token auth, mirroring how an API client reaches the endpoint. + self.user.is_superuser = True + self.user.save() + self.client = APIClient() + token_key = create_token(self.user) + self.header = {'HTTP_AUTHORIZATION': f'Token {token_key}'} + self.url = reverse('graphql') + + def _post(self, query, branch=None, via='header'): + headers = dict(self.header) + url = self.url + if branch is not None: + if via == 'header': + # Django maps HTTP_X_NETBOX_BRANCH → the X-NetBox-Branch request + # header netbox-branching reads to activate the branch. + headers['HTTP_X_NETBOX_BRANCH'] = branch.schema_id + elif via == 'query': + # The ?_branch= query param the UI uses — another branch source + # netbox-branching honours for any request. + url = f'{self.url}?_branch={branch.schema_id}' + response = self.client.post( + url, data={'query': query}, format='json', **headers + ) + return json.loads(response.content) + + def _data(self, query, branch=None, via='header'): + payload = self._post(query, branch=branch, via=via) + self.assertNotIn('errors', payload, msg=str(payload.get('errors'))) + return payload['data'] + + def _assert_query_rejected(self, query, branch=None): + # A field/type absent from the active schema is a GraphQL validation error. + payload = self._post(query, branch=branch) + self.assertIn('errors', payload, msg=f'expected schema to reject query: {payload}') + + def test_main_and_branch_isolated_schema_and_data(self): + # --- main: COT 'server' (field 'name') with one instance --- + server = CustomObjectType.objects.create(name='server', slug='server') + CustomObjectTypeField.objects.create( + custom_object_type=server, name='name', label='Name', type='text', + primary=True, required=True, + ) + server.get_model().objects.create(name='main-server') + + # --- branch: a branch-only COT 'widget', plus a NEW field 'note' added to + # the existing 'server' COT and a branch-only server row that uses it --- + branch = _provision_branch('GraphQL E2E', user=self.user) + branch_request = _make_request(self.user) + with activate_branch(branch), event_tracking(branch_request): + widget = CustomObjectType.objects.create(name='widget', slug='widget') + CustomObjectTypeField.objects.create( + custom_object_type=widget, name='name', label='Name', type='text', + primary=True, required=True, + ) + widget.get_model().objects.create(name='branch-widget') + + server_in_branch = CustomObjectType.objects.get(pk=server.pk) + CustomObjectTypeField.objects.create( + custom_object_type=server_in_branch, name='note', label='Note', + type='text', + ) + server_in_branch.get_model().objects.create(name='branch-server', note='hi') + + self.live.reset_cache() + + # --- MAIN (no header): only main's schema and data --- + main = self._data('{ custom_objects_server_list { name } }') + self.assertEqual( + [r['name'] for r in main['custom_objects_server_list']], ['main-server'], + 'main must not see the branch-only server row', + ) + # The branch-added 'note' field is absent from main's schema. + self._assert_query_rejected('{ custom_objects_server_list { name note } }') + # The branch-only 'widget' COT is absent from main's schema. + self._assert_query_rejected('{ custom_objects_widget_list { name } }') + + # --- BRANCH (header): branch schema (note + widget) and branch data --- + data = self._data( + '{ custom_objects_server_list { name note } ' + 'custom_objects_widget_list { name } }', + branch=branch, + ) + servers = {r['name']: r['note'] for r in data['custom_objects_server_list']} + # The branch is a copy of main plus its own change: both rows are visible, + # and only the branch row carries the branch-only 'note' value. + self.assertIn('main-server', servers) + self.assertEqual(servers.get('branch-server'), 'hi') + self.assertEqual( + [r['name'] for r in data['custom_objects_widget_list']], ['branch-widget'], + ) + + # --- BRANCH via ?_branch= query param (no header): the UI's branch source + # selects the same branch. GraphQL honours every branch source NetBox does, + # not just the header — if it didn't, this would fall back to main and reject + # the branch-only 'widget' field. --- + via_param = self._data( + '{ custom_objects_widget_list { name } }', branch=branch, via='query', + ) + self.assertEqual( + [r['name'] for r in via_param['custom_objects_widget_list']], ['branch-widget'], + ) + + # --- MAIN again: still unchanged by the unmerged branch --- + main_again = self._data('{ custom_objects_server_list { name } }') + self.assertEqual( + [r['name'] for r in main_again['custom_objects_server_list']], ['main-server'], + ) diff --git a/netbox_custom_objects/tests/test_checks.py b/netbox_custom_objects/tests/test_checks.py new file mode 100644 index 00000000..6069588c --- /dev/null +++ b/netbox_custom_objects/tests/test_checks.py @@ -0,0 +1,87 @@ +""" +Tests for the conditional netbox-branching version system check +(``netbox_custom_objects.checks.check_branching_compatibility``). + +The check is pure version-comparison logic gated on whether netbox-branching +is installed, so every branch is exercised by mocking its three inputs: +``apps.is_installed``, ``settings.RELEASE``, and the importlib.metadata +``version`` lookup. No database is required, hence ``SimpleTestCase``. +""" +from importlib.metadata import PackageNotFoundError +from types import SimpleNamespace +from unittest.mock import patch + +from django.test import SimpleTestCase + +from netbox_custom_objects import checks + + +def _run(*, branching_installed=True, netbox_version='4.6.2', branching_version='1.0.4'): + """Invoke the check with each input mocked. + + ``netbox_version`` / ``branching_version`` may be a version string, or an + Exception instance/class to simulate the failure paths (a missing + ``settings.RELEASE``, an unparseable version, or ``PackageNotFoundError``). + """ + if isinstance(netbox_version, str): + release = SimpleNamespace(version=netbox_version) + else: + # No RELEASE attribute -> accessing settings.RELEASE raises AttributeError. + release = None + + def fake_pkg_version(_name): + if isinstance(branching_version, str): + return branching_version + raise branching_version # Exception instance or class + + settings_stub = SimpleNamespace() + if release is not None: + settings_stub.RELEASE = release + + with patch.object(checks.apps, 'is_installed', return_value=branching_installed), \ + patch.object(checks, 'settings', settings_stub), \ + patch.object(checks, '_pkg_version', fake_pkg_version): + return checks.check_branching_compatibility(app_configs=None) + + +def _ids(errors): + return {e.id for e in errors} + + +class CheckBranchingCompatibilityTest(SimpleTestCase): + def test_no_branching_is_noop(self): + """Without netbox-branching installed, the check returns no messages.""" + self.assertEqual(_run(branching_installed=False), []) + + def test_both_versions_satisfied(self): + """Versions at the floor produce no errors.""" + self.assertEqual(_run(netbox_version='4.6.2', branching_version='1.0.4'), []) + + def test_newer_versions_satisfied(self): + self.assertEqual(_run(netbox_version='4.7.0', branching_version='1.1.0'), []) + + def test_netbox_too_old(self): + errors = _run(netbox_version='4.6.1', branching_version='1.0.4') + self.assertEqual(_ids(errors), {'netbox_custom_objects.E001'}) + + def test_branching_too_old(self): + errors = _run(netbox_version='4.6.2', branching_version='1.0.3') + self.assertEqual(_ids(errors), {'netbox_custom_objects.E002'}) + + def test_both_too_old(self): + errors = _run(netbox_version='4.5.0', branching_version='0.9.0') + self.assertEqual(_ids(errors), {'netbox_custom_objects.E001', 'netbox_custom_objects.E002'}) + + def test_missing_release_skips_netbox_check(self): + """A missing settings.RELEASE skips E001 but still evaluates branching.""" + errors = _run(netbox_version=AttributeError, branching_version='1.0.3') + self.assertEqual(_ids(errors), {'netbox_custom_objects.E002'}) + + def test_unparseable_netbox_version_skips_netbox_check(self): + errors = _run(netbox_version='not-a-version', branching_version='1.0.4') + self.assertEqual(errors, []) + + def test_unresolvable_branching_version_warns(self): + """An installed-but-unmeasurable branching dist warns rather than passing silently.""" + errors = _run(netbox_version='4.6.2', branching_version=PackageNotFoundError()) + self.assertEqual(_ids(errors), {'netbox_custom_objects.W001'}) diff --git a/netbox_custom_objects/tests/test_deletion.py b/netbox_custom_objects/tests/test_deletion.py index 5d2f38f3..b6c8d7bd 100644 --- a/netbox_custom_objects/tests/test_deletion.py +++ b/netbox_custom_objects/tests/test_deletion.py @@ -23,21 +23,9 @@ class DeletionTestCase(TransactionCleanupMixin, CustomObjectsTestCase, Transacti """Test deletion scenarios with cascading effects.""" def setUp(self): - """Purge stale dynamic models left in the app registry by earlier TestCase - classes whose setUpTestData transaction was rolled back (dropping the backing - tables). Leaving them registered causes Django's cascade-delete collector to - query non-existent tables when a related core object is deleted. - """ + # TransactionCleanupMixin.setUp() purges stale generated models and + # CustomObjectsTestCase.setUp() creates the test user and client. super().setUp() - stale = [ - name - for name, model in list(django_apps.all_models.get(APP_LABEL, {}).items()) - if getattr(model, '_generated_table_model', False) - ] - for name in stale: - django_apps.all_models[APP_LABEL].pop(name, None) - if stale: - django_apps.clear_cache() # ------------------------------------------------------------------ # Helpers diff --git a/netbox_custom_objects/tests/test_field_types.py b/netbox_custom_objects/tests/test_field_types.py index 1fdb734e..de85d057 100644 --- a/netbox_custom_objects/tests/test_field_types.py +++ b/netbox_custom_objects/tests/test_field_types.py @@ -1,16 +1,20 @@ """ Tests for all the different field types supported by Custom Object Type Fields. """ +from importlib import import_module from unittest import skip from unittest.mock import Mock -from datetime import date, datetime +from datetime import date, datetime, timezone from decimal import Decimal +from django.apps import apps from django.core.exceptions import FieldDoesNotExist, ValidationError +from django.db import connection, models from django.test import TestCase from core.models import ObjectType from dcim.models import Device, DeviceType, ModuleType from netbox_custom_objects.field_types import ( + CoordinatesFieldType, MultiObjectFieldType, MultiSelectFieldType, ObjectFieldType, @@ -189,6 +193,89 @@ def test_integer_field_model_generation(self): self.assertEqual(instance.count, 25) + def test_integer_field_is_64_bit(self): + """Integer fields use a 64-bit (bigint) column, not 32-bit (issue #532).""" + self.create_custom_object_type_field( + self.custom_object_type, + name="count", + label="Count", + type="integer", + ) + + model = self.custom_object_type.get_model() + + # The generated model field must be a BigIntegerField. + self.assertIsInstance( + model._meta.get_field("count"), models.BigIntegerField + ) + + # A value beyond the signed 32-bit range must round-trip through the DB. + big_value = 9_000_000_000 # > 2**31 - 1 (2_147_483_647) + instance = model.objects.create(name="Test", count=big_value) + instance.refresh_from_db() + self.assertEqual(instance.count, big_value) + + def test_integer_field_upgrade_widens_existing_32bit_column(self): + """The 0015 migration widens pre-#532 32-bit integer columns to bigint. + + New tables already get a bigint column (test_integer_field_is_64_bit), + but custom_objects_* tables created before issue #532 have a 32-bit + ``integer`` column that the field-type change alone does not alter -- + they are managed=False. This exercises that upgrade path: realize the + column, force it back to 32-bit to mimic a legacy install, run the + migration's data function, and confirm it is widened in place. + """ + field = self.create_custom_object_type_field( + self.custom_object_type, + name="legacy_count", + label="Legacy Count", + type="integer", + ) + + # Realize the backing model/table/column. + model = self.custom_object_type.get_model() + + table_name = f"custom_objects_{self.custom_object_type.pk}" + column_name = field.name + + def column_data_type(): + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT data_type FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = %s AND column_name = %s + """, + [table_name, column_name], + ) + row = cursor.fetchone() + return row[0] if row else None + + # Mimic a legacy install: force the column back to a 32-bit integer. + with connection.cursor() as cursor: + cursor.execute( + f'ALTER TABLE "{table_name}" ALTER COLUMN "{column_name}" TYPE integer' + ) + self.assertEqual(column_data_type(), "integer") + + # Run the migration's data function against the legacy column. + # widen_integer_columns only reads stable CustomObjectTypeField columns + # (type, custom_object_type_id, name), so the current app registry is + # equivalent to the historical one RunPython would pass. If the function + # ever introspects historical field structure, switch to MigrationLoader. + migration = import_module( + "netbox_custom_objects.migrations.0016_widen_integer_columns" + ) + with connection.schema_editor() as schema_editor: + migration.widen_integer_columns(apps, schema_editor) + + # Column is widened, and a value beyond the 32-bit range round-trips. + self.assertEqual(column_data_type(), "bigint") + big_value = 9_000_000_000 # > 2**31 - 1 (2_147_483_647) + instance = model.objects.create(name="Test", legacy_count=big_value) + instance.refresh_from_db() + self.assertEqual(instance.legacy_count, big_value) + class DecimalFieldTypeTestCase(FieldTypeTestCase): """Test cases for decimal field type.""" @@ -252,6 +339,166 @@ def test_decimal_field_model_generation(self): self.assertEqual(instance.price, Decimal("25.75")) +class CoordinatesFieldTypeTestCase(FieldTypeTestCase): + """Test cases for the coordinates field type.""" + + def test_coordinates_field_creation(self): + """A coordinates field is created with the expected type.""" + field = self.create_custom_object_type_field( + self.custom_object_type, + name="location", + label="Location", + type="coordinates", + ) + self.assertEqual(field.type, "coordinates") + + def test_coordinates_field_model_generation(self): + """One coordinates field expands into two backing decimal columns.""" + self.create_custom_object_type_field( + self.custom_object_type, + name="location", + label="Location", + type="coordinates", + ) + + model = self.custom_object_type.get_model() + column_names = {f.name for f in model._meta.local_fields} + self.assertIn("location_latitude", column_names) + self.assertIn("location_longitude", column_names) + # The logical field name itself is not a real column. + self.assertNotIn("location", column_names) + + # Verify precision/validators mirror NetBox core's Site coordinates. + latitude = model._meta.get_field("location_latitude") + longitude = model._meta.get_field("location_longitude") + self.assertEqual((latitude.max_digits, latitude.decimal_places), (8, 6)) + self.assertEqual((longitude.max_digits, longitude.decimal_places), (9, 6)) + + instance = model.objects.create( + name="Test", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + self.assertEqual(instance.location_latitude, Decimal("40.712800")) + self.assertEqual(instance.location_longitude, Decimal("-74.006000")) + + def test_coordinates_display_value(self): + """get_display_value combines both columns, or returns None when unset.""" + self.create_custom_object_type_field( + self.custom_object_type, + name="location", + label="Location", + type="coordinates", + ) + model = self.custom_object_type.get_model() + field_type = CoordinatesFieldType() + + populated = model.objects.create( + name="A", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + self.assertEqual( + field_type.get_display_value(populated, "location"), + "40.712800, -74.006000", + ) + + empty = model.objects.create(name="B") + self.assertIsNone(field_type.get_display_value(empty, "location")) + + def test_coordinates_validate_pair(self): + """Latitude and longitude must both be set or both be empty.""" + field_type = CoordinatesFieldType() + # Both set / both empty are valid. + field_type.validate_pair(Decimal("1.0"), Decimal("2.0")) + field_type.validate_pair(None, None) + # Half-populated pairs are rejected. + with self.assertRaises(ValidationError): + field_type.validate_pair(Decimal("1.0"), None) + with self.assertRaises(ValidationError): + field_type.validate_pair(None, Decimal("2.0")) + + def test_coordinates_field_rejects_unique(self): + """A coordinates field cannot be marked unique.""" + field = CustomObjectTypeField( + custom_object_type=self.custom_object_type, + name="location", + label="Location", + type="coordinates", + unique=True, + ) + with self.assertRaises(ValidationError): + field.full_clean() + + def test_coordinates_field_search_weight_forced_to_zero(self): + """ + A coordinates field has no column matching its name, so it can never be + indexed for search; clean() forces search_weight to 0 rather than honour a + non-zero value that would silently have no effect. + """ + field = CustomObjectTypeField( + custom_object_type=self.custom_object_type, + name="location", + label="Location", + type="coordinates", + search_weight=500, + ) + field.full_clean() + self.assertEqual(field.search_weight, 0) + + def test_change_existing_field_to_coordinates_rejected(self): + """An existing non-coordinates field cannot be converted to coordinates.""" + field = self.create_custom_object_type_field( + self.custom_object_type, name="location", label="Location", type="text", + ) + field.type = "coordinates" + with self.assertRaises(ValidationError): + field.full_clean() + + def test_change_coordinates_field_to_other_type_rejected(self): + """An existing coordinates field cannot be converted to another type.""" + field = self.create_custom_object_type_field( + self.custom_object_type, name="location", label="Location", type="coordinates", + ) + field.type = "text" + with self.assertRaises(ValidationError): + field.full_clean() + + def test_coordinates_backing_column_collision_rejected(self): + """ + Adding a coordinates field whose backing column collides with an existing + field's column raises a ValidationError rather than a DB error. + """ + self.create_custom_object_type_field( + self.custom_object_type, name="location_latitude", label="Lat", type="text", + ) + field = CustomObjectTypeField( + custom_object_type=self.custom_object_type, + name="location", + label="Location", + type="coordinates", + ) + with self.assertRaises(ValidationError): + field.full_clean() + + def test_field_colliding_with_coordinates_backing_column_rejected(self): + """ + The reverse collision: a plain field named "_longitude" cannot be + added when a coordinates field "" already exists. + """ + self.create_custom_object_type_field( + self.custom_object_type, name="location", label="Location", type="coordinates", + ) + field = CustomObjectTypeField( + custom_object_type=self.custom_object_type, + name="location_longitude", + label="Lon", + type="text", + ) + with self.assertRaises(ValidationError): + field.full_clean() + + class BooleanFieldTypeTestCase(FieldTypeTestCase): """Test cases for boolean field type.""" @@ -364,11 +611,11 @@ def test_datetime_field_creation(self): name="created_datetime", label="Created DateTime", type="datetime", - default="2023-01-01T12:00:00" + default="2023-01-01T12:00:00+00:00" ) self.assertEqual(field.type, "datetime") - self.assertEqual(field.default, "2023-01-01T12:00:00") + self.assertEqual(field.default, "2023-01-01T12:00:00+00:00") def test_datetime_field_validation(self): """Test datetime field validation.""" @@ -399,7 +646,7 @@ def test_datetime_field_model_generation(self): ) model = self.custom_object_type.get_model() - test_datetime = datetime(2023, 1, 1, 12, 0, 0) + test_datetime = datetime(2023, 1, 1, 12, 0, 0, tzinfo=timezone.utc) instance = model.objects.create(name="Test", created_datetime=test_datetime) self.assertEqual(instance.created_datetime, test_datetime) diff --git a/netbox_custom_objects/tests/test_filtersets.py b/netbox_custom_objects/tests/test_filtersets.py index 38058af7..0213e1f5 100644 --- a/netbox_custom_objects/tests/test_filtersets.py +++ b/netbox_custom_objects/tests/test_filtersets.py @@ -770,8 +770,9 @@ def setUpTestData(cls): ) model = cls.cot.get_model() - cls.obj_morning = model.objects.create(ts=datetime.datetime(2025, 3, 10, 9, 0, 0)) - cls.obj_evening = model.objects.create(ts=datetime.datetime(2025, 3, 10, 18, 30, 0)) + utc = datetime.timezone.utc + cls.obj_morning = model.objects.create(ts=datetime.datetime(2025, 3, 10, 9, 0, 0, tzinfo=utc)) + cls.obj_evening = model.objects.create(ts=datetime.datetime(2025, 3, 10, 18, 30, 0, tzinfo=utc)) def _search(self, value): model = self.cot.get_model() @@ -1386,3 +1387,63 @@ def test_form_fields_are_multi_choice(self): self.assertIsInstance(result, dict) for form_field in result.values(): self.assertIsInstance(form_field, DynamicModelMultipleChoiceField) + + +# --------------------------------------------------------------------------- +# CoordinatesFieldType — filter form fields and queryset filtering +# --------------------------------------------------------------------------- + + +class CoordinatesFieldFiltersetTestCase(CustomObjectsTestCase, TestCase): + """A coordinates field yields two numeric filters on its backing columns.""" + + @classmethod + def setUpTestData(cls): + super().setUpTestData() + cls.cot = cls.create_custom_object_type(name="CoordFSTest", slug="coord-fs-test") + cls.create_custom_object_type_field( + cls.cot, name="name", label="Name", type="text", primary=True, required=True + ) + cls.field = cls.create_custom_object_type_field( + cls.cot, name="location", label="Location", type="coordinates" + ) + + model = cls.cot.get_model() + cls.obj1 = model.objects.create( + name="Box1", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + cls.obj2 = model.objects.create( + name="Box2", + location_latitude=Decimal("51.507400"), + location_longitude=Decimal("-0.127800"), + ) + + def _filterset(self, params): + model = self.cot.get_model() + return get_filterset_class(model)(params, model.objects.all()) + + def test_build_filter_for_field_returns_two_filters(self): + filters = build_filter_for_field(self.field) + self.assertEqual( + set(filters), {"location_latitude", "location_longitude"} + ) + self.assertIsInstance(filters["location_latitude"], django_filters.NumberFilter) + + def test_filter_by_latitude(self): + pks = list( + self._filterset({"location_latitude": "40.712800"}).qs.values_list("pk", flat=True) + ) + self.assertIn(self.obj1.pk, pks) + self.assertNotIn(self.obj2.pk, pks) + + def test_filter_by_longitude(self): + pks = list( + self._filterset({"location_longitude": "-0.127800"}).qs.values_list("pk", flat=True) + ) + self.assertIn(self.obj2.pk, pks) + self.assertNotIn(self.obj1.pk, pks) + + def test_no_filter_returns_all(self): + self.assertEqual(self._filterset({}).qs.count(), 2) diff --git a/netbox_custom_objects/tests/test_graphql.py b/netbox_custom_objects/tests/test_graphql.py new file mode 100644 index 00000000..3fb1cbce --- /dev/null +++ b/netbox_custom_objects/tests/test_graphql.py @@ -0,0 +1,767 @@ +""" +Tests for GraphQL support for custom objects. + +Two layers are exercised: + +* Unit tests for the schema-generation helpers (query-field naming, per-structure + type caching, the live-schema signature/rebuild machinery). The plugin's + GraphQL schema contribution is intentionally skipped during the test run + (``should_skip_dynamic_model_creation()`` returns True when ``test`` is on the + command line), so these call the generation functions directly. + +* End-to-end tests that drive the real ``/graphql/`` HTTP endpoint with token + authentication, modelled on NetBox's own GraphQL test pattern. These patch the + startup guard off so the live schema (installed via the view patch in + ``__init__.py``) is built and bound per request, exactly as in production. +""" + +import decimal +import json +from unittest import mock + +import strawberry +from django.test import TestCase, override_settings +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APIClient + +from core.models import ObjectType +from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Region, Site +from users.models import ObjectPermission + +from netbox_custom_objects.graphql import live as live_module +from netbox_custom_objects.graphql import schema as schema_module +from netbox_custom_objects.graphql.schema import build_query_classes, _query_field_name +from netbox_custom_objects.graphql.types import build_object_type + +from .base import CustomObjectsTestCase, create_token + + +class GraphQLSchemaGenerationTestCase(CustomObjectsTestCase, TestCase): + """Tests for the schema/query-class generation helpers.""" + + def test_query_field_name_sanitizes_and_namespaces_slug(self): + used = set() + cot = self.create_custom_object_type(name="Widget", slug="my-widget") + name = _query_field_name(cot, used) + # Hyphens sanitised to underscores, and namespaced with the prefix so the + # field can never collide with a core/plugin root query field. + self.assertEqual(name, "custom_objects_my_widget") + + def test_query_field_name_dedupes_collisions(self): + used = set() + a = self.create_custom_object_type(name="A", slug="a-b") + b = self.create_custom_object_type(name="B", slug="a_b", verbose_name_plural="Bs") + first = _query_field_name(a, used) + second = _query_field_name(b, used) + self.assertEqual(first, "custom_objects_a_b") + self.assertEqual(second, f"custom_objects_a_b_{b.id}") + + def test_query_field_name_avoids_list_suffix_collision(self): + # A type whose slug sanitizes to '_list' must not claim the same + # GraphQL field as another type's auto-generated '_list' list field. + used = set() + a = self.create_custom_object_type(name="Foo", slug="foo") + b = self.create_custom_object_type( + name="Foo List", slug="foo-list", verbose_name_plural="Foo Lists" + ) + name_a = _query_field_name(a, used) + name_b = _query_field_name(b, used) + self.assertEqual(name_a, "custom_objects_foo") + # 'custom_objects_foo_list' is already reserved as A's list field, so B is + # disambiguated. + self.assertNotEqual(name_b, "custom_objects_foo_list") + # All four generated names (singular + list for each type) stay distinct. + all_fields = {name_a, f"{name_a}_list", name_b, f"{name_b}_list"} + self.assertEqual(len(all_fields), 4) + + def test_build_object_type_is_cached_per_structure(self): + # An unchanged COT reuses its built type; a structural change (which bumps + # cache_timestamp) forces a rebuild. + from netbox_custom_objects.graphql import types as types_module + + types_module.clear_type_cache() + self.addCleanup(types_module.clear_type_cache) + + cot = self.create_simple_custom_object_type(name="Cached", slug="cached") + first = build_object_type(cot) + self.assertIs(first, build_object_type(cot)) + + self.create_custom_object_type_field(cot, name="extra", label="Extra", type="text") + cot.refresh_from_db() + self.assertIsNot(first, build_object_type(cot)) + + def test_build_query_classes_skipped_during_tests(self): + # ``test`` is on sys.argv during the suite, so the startup builder + # short-circuits to an empty list rather than touching the DB. + self.create_simple_custom_object_type() + self.assertEqual(build_query_classes(), []) + + def test_module_exposes_schema_list(self): + # The module-level ``schema`` attribute must always be a list so that + # NetBox's ``register_graphql_schema`` (which calls ``.extend``) works. + self.assertIsInstance(schema_module.schema, list) + + def test_real_builder_produces_assemblable_query(self): + # Exercise the actual production builder (normally skipped during tests) + # to confirm it yields a Query class that assembles into a valid schema + # exposing the expected per-type fields. + from strawberry.schema.config import StrawberryConfig + + self.create_simple_custom_object_type(name="Gadget", slug="gadget") + with mock.patch( + "netbox_custom_objects.CustomObjectsPluginConfig." + "should_skip_dynamic_model_creation", + return_value=False, + ): + classes = build_query_classes() + + self.assertEqual(len(classes), 1) + # Assemble exactly as NetBox does: the contributed class as a base of the + # root Query type. + query_cls = strawberry.type(type("Query", (classes[0],), {})) + built = strawberry.Schema( + query=query_cls, config=StrawberryConfig(auto_camel_case=False) + ) + sdl = str(built) + self.assertIn("custom_objects_gadget", sdl) + self.assertIn("custom_objects_gadget_list", sdl) + + def test_self_referential_object_field_does_not_recurse(self): + # A self-referential OBJECT field (FK to the same COT) must not send + # build_object_type into infinite recursion: the cycle guard breaks the + # back-edge with the flat stub. Regression test for the str/int mismatch + # that left the guard inert (extract_cot_id returns a str, the in-progress + # stack holds ints), which caused a RecursionError on this configuration. + from netbox_custom_objects.graphql import types as types_module + + types_module.clear_type_cache() + self.addCleanup(types_module.clear_type_cache) + + cot = self.create_custom_object_type(name="Node", slug="node") + self.create_custom_object_type_field( + cot, name="label", label="Label", type="text", primary=True, required=True + ) + self_ot = ObjectType.objects.get( + app_label="netbox_custom_objects", + model=cot.get_table_model_name(cot.id).lower(), + ) + self.create_custom_object_type_field( + cot, name="parent", label="Parent", type="object", related_object_type=self_ot + ) + cot.refresh_from_db() + + # Completes (no RecursionError) and yields a usable type. + gql_type = build_object_type(cot) + self.assertIsNotNone(gql_type) + # The cyclic build used the flat stub for the self-edge, so the type is + # intentionally not cached — a later top-level query rebuilds it. + self.assertIsNot(gql_type, build_object_type(cot)) + + def test_rebuild_prefetch_covers_field_access(self): + # Regression for the schema-rebuild N+1: build_query_classes preloads types + # with their fields and each field's related type(s). Loading a type via + # that same prefetch must make the rebuild's per-field accesses + # (types.py: the fields iteration, the FK target, and the polymorphic M2M + # targets) issue zero further queries. + from django.db import connection + from django.db.models import Prefetch + from django.test.utils import CaptureQueriesContext + + from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField + + cot = self.create_custom_object_type(name="Pf", slug="pf") + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True + ) + # Non-polymorphic object field → exercises the related_object_type FK. + self.create_custom_object_type_field( + cot, name="single", label="Single", type="object", + related_object_type=self.get_site_object_type(), + ) + # Polymorphic object field → exercises the related_object_types M2M. + self.create_polymorphic_field( + cot, [self.get_site_object_type(), self.get_device_object_type()], + name="poly", type="object", + ) + + # The exact prefetch build_query_classes uses. + fields_qs = ( + CustomObjectTypeField.objects + .select_related("related_object_type") + .prefetch_related("related_object_types") + ) + loaded = CustomObjectType.objects.prefetch_related( + Prefetch("fields", queryset=fields_qs) + ).get(pk=cot.pk) + + with CaptureQueriesContext(connection) as ctx: + for field in loaded.fields.all(): # fields iteration (no query) + if field.related_object_type_id: + _ = field.related_object_type # FK — select_related + if field.is_polymorphic: + list(field.related_object_types.all()) # M2M — prefetch_related + self.assertEqual( + len(ctx.captured_queries), 0, + f"prefetch missed an access: {[q['sql'] for q in ctx.captured_queries]}", + ) + + def test_related_repr_degrades_on_broken_str(self): + # A referenced object whose __str__ (or get_absolute_url) raises must not + # propagate — that would error the whole relationship field instead of just + # this one object. It degrades to a stable placeholder display. + from netbox_custom_objects.graphql.types import _related_repr + + class Broken: + pk = 7 + + class _meta: + label_lower = "app.broken" + + def __str__(self): + raise ValueError("boom") + + def get_absolute_url(self): + raise ValueError("no url") + + rep = _related_repr(Broken()) + self.assertEqual(rep.id, 7) + self.assertEqual(rep.object_type, "app.broken") + self.assertEqual(rep.display, "app.broken:7") # fallback, not an exception + self.assertIsNone(rep.url) + + +class GraphQLLiveSchemaTestCase(CustomObjectsTestCase, TestCase): + """ + The schema must reflect custom object types created/deleted at runtime + without a NetBox restart (issue #30 follow-up). + + ``should_skip_dynamic_model_creation`` returns True during the test run, so + we patch it off for these tests and reset the per-process cache around each. + """ + + def setUp(self): + super().setUp() + live_module.reset_cache() + patcher = mock.patch( + "netbox_custom_objects.CustomObjectsPluginConfig." + "should_skip_dynamic_model_creation", + return_value=False, + ) + patcher.start() + self.addCleanup(patcher.stop) + self.addCleanup(live_module.reset_cache) + + def test_signature_changes_when_type_added(self): + sig_before = live_module.schema_signature() + self.create_simple_custom_object_type(name="Sig", slug="sig") + sig_after = live_module.schema_signature() + self.assertNotEqual(sig_before, sig_after) + + def test_signature_changes_when_field_added(self): + cot = self.create_simple_custom_object_type(name="Sig2", slug="sig2") + sig_before = live_module.schema_signature() + self.create_custom_object_type_field(cot, name="extra", label="Extra", type="text") + sig_after = live_module.schema_signature() + self.assertNotEqual(sig_before, sig_after) + + def test_get_live_schema_rebuilds_on_new_type(self): + # A type that does not exist yet must not be in the schema... + first = live_module.get_live_schema() + self.assertIsNotNone(first) + self.assertNotIn("custom_objects_runtime_thing", str(first)) + + # ...and must appear after creation, with no restart and without manually + # clearing the cache (the signature change drives the rebuild). + self.create_simple_custom_object_type(name="Runtime Thing", slug="runtime_thing") + second = live_module.get_live_schema() + self.assertIsNot(first, second) + sdl = str(second) + self.assertIn("custom_objects_runtime_thing", sdl) + self.assertIn("custom_objects_runtime_thing_list", sdl) + + def test_get_live_schema_cached_when_unchanged(self): + self.create_simple_custom_object_type(name="Stable", slug="stable") + a = live_module.get_live_schema() + b = live_module.get_live_schema() + # No DB change between calls → same cached object, no rebuild. + self.assertIs(a, b) + + def test_live_schema_drops_deleted_type(self): + from netbox_custom_objects.models import CustomObjectType + + cot = self.create_simple_custom_object_type(name="Temp", slug="temp_type") + self.assertIn("custom_objects_temp_type", str(live_module.get_live_schema())) + # Delete via the queryset rather than cot.delete(): the schema-drop + # behaviour only depends on the row being gone (which changes the + # signature and triggers a rebuild), and this avoids the unrelated COT + # teardown machinery. + CustomObjectType.objects.filter(pk=cot.pk).delete() + self.assertNotIn("custom_objects_temp_type", str(live_module.get_live_schema())) + + +class GraphQLSignalRegistrationTestCase(TestCase): + """ + Regression tests for connect_signature_invalidation() guard logic. + + The function must not crash when netbox-branching is installed in the Python + environment but absent from INSTALLED_APPS (which raises RuntimeError, not + ImportError, when Django model classes are imported). + """ + + def _assert_evict_handler_not_registered(self): + """ + Assert that the branch-eviction handler is not registered, using + Signal.disconnect() rather than inspecting the undocumented receivers + list. disconnect() is the public API and returns True only when it + actually removed something — so if it returns False the handler was + never connected (which is the desired outcome). Calling disconnect() + also serves as cleanup: if a prior test or ready() left a stale + registration, this removes it without affecting the assertion. + """ + from django.db.models.signals import post_delete + was_registered = post_delete.disconnect(dispatch_uid="nco_graphql_evict_branch") + self.assertFalse( + was_registered, + "Branch eviction handler must not be registered when branching is not enabled", + ) + + def test_branching_installed_but_not_enabled_does_not_crash(self): + # Simulate netbox-branching installed but not in INSTALLED_APPS by + # patching apps.is_installed to return False for it, then verify that + # connect_signature_invalidation() returns without registering the + # branch-eviction handler. + from django.db.models.signals import post_delete + + from netbox_custom_objects.graphql.live import connect_signature_invalidation + + # Remove any pre-existing registration (e.g. from ready()) so the test + # is self-contained regardless of the environment. + post_delete.disconnect(dispatch_uid="nco_graphql_evict_branch") + + with mock.patch( + "netbox_custom_objects.graphql.live.django_apps.is_installed", + side_effect=lambda app: app != "netbox_branching", + ): + # Must not raise — previously raised RuntimeError here. + connect_signature_invalidation() + + self._assert_evict_handler_not_registered() + + def test_branching_not_installed_does_not_crash(self): + # When netbox-branching is entirely absent, is_installed returns False + # for all apps and the function must return cleanly without registering + # the branch-eviction handler. + from django.db.models.signals import post_delete + + from netbox_custom_objects.graphql.live import connect_signature_invalidation + + post_delete.disconnect(dispatch_uid="nco_graphql_evict_branch") + + with mock.patch( + "netbox_custom_objects.graphql.live.django_apps.is_installed", + side_effect=lambda app: app != "netbox_branching", + ): + connect_signature_invalidation() # must not raise + + self._assert_evict_handler_not_registered() + + +@override_settings(LOGIN_REQUIRED=True) +class GraphQLEndpointTestCase(CustomObjectsTestCase, TestCase): + """ + End-to-end tests against the real ``/graphql/`` HTTP endpoint. + + Patches the startup guard off so the live schema is built and bound to the + (monkey-patched) GraphQL view per request, and authenticates with a token — + the same path a real client takes. The user is a superuser so these tests + focus on schema/resolution correctness rather than permission wiring + (permissions are covered separately below). + """ + + def setUp(self): + super().setUp() + live_module.reset_cache() + self.addCleanup(live_module.reset_cache) + patcher = mock.patch( + "netbox_custom_objects.CustomObjectsPluginConfig." + "should_skip_dynamic_model_creation", + return_value=False, + ) + patcher.start() + self.addCleanup(patcher.stop) + + self.user.is_superuser = True + self.user.save() + # A fresh APIClient using token auth (not the session login set up by the + # base class), mirroring how an API client reaches the endpoint. + self.client = APIClient() + token_key = create_token(self.user) + self.header = {"HTTP_AUTHORIZATION": f"Token {token_key}"} + self.url = reverse("graphql") + + def _gql(self, query): + response = self.client.post( + self.url, data={"query": query}, format="json", **self.header + ) + self.assertEqual( + response.status_code, status.HTTP_200_OK, getattr(response, "content", response) + ) + payload = json.loads(response.content) + self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) + return payload["data"] + + def _make_device(self, name="dev1"): + manufacturer, _ = Manufacturer.objects.get_or_create(name="Mfr", slug="mfr") + device_type, _ = DeviceType.objects.get_or_create( + manufacturer=manufacturer, model="Model", slug="model" + ) + role, _ = DeviceRole.objects.get_or_create(name="Role", slug="role") + site = self._make_site() + return Device.objects.create( + name=name, device_type=device_type, role=role, site=site + ) + + def _make_site(self, name="Site", slug="site", region=None): + return Site.objects.create(name=name, slug=slug, region=region) + + def _site_object_field_type(self, name="Server", slug="server"): + """A COT with a primary text field and a single-object field → Site.""" + cot = self.create_custom_object_type(name=name, slug=slug) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True + ) + self.create_custom_object_type_field( + cot, name="site", label="Site", type="object", + related_object_type=self.get_site_object_type(), + ) + return cot + + def test_scalar_fields_query(self): + cot = self.create_complex_custom_object_type(name="Asset", slug="asset") + model = cot.get_model() + model.objects.create(name="First", count=7, active=True, status="choice1") + + data = self._gql("{ custom_objects_asset_list { id display name count active status } }") + rows = data["custom_objects_asset_list"] + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["display"], "First") + self.assertEqual(rows[0]["name"], "First") + self.assertEqual(rows[0]["count"], 7) + self.assertTrue(rows[0]["active"]) + self.assertEqual(rows[0]["status"], "choice1") + + def test_decimal_field_round_trips(self): + # A decimal field is annotated with the stdlib ``decimal.Decimal``, which + # Strawberry maps to its built-in Decimal scalar. Confirm the value + # survives the round trip through the GraphQL endpoint with full + # precision (Strawberry serializes the scalar to a string). + cot = self.create_custom_object_type(name="Product", slug="product") + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True + ) + self.create_custom_object_type_field( + cot, name="price", label="Price", type="decimal" + ) + model = cot.get_model() + model.objects.create(name="Widget", price=decimal.Decimal("19.99")) + + data = self._gql("{ custom_objects_product_list { name price } }") + rows = data["custom_objects_product_list"] + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["name"], "Widget") + self.assertEqual(decimal.Decimal(str(rows[0]["price"])), decimal.Decimal("19.99")) + + def test_json_field_round_trips(self): + # A JSON field is annotated with Strawberry's JSON scalar; confirm a + # structured value survives the round trip through the endpoint intact. + cot = self.create_custom_object_type(name="Config", slug="config") + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True + ) + self.create_custom_object_type_field( + cot, name="data", label="Data", type="json" + ) + model = cot.get_model() + payload_value = {"enabled": True, "ports": [80, 443]} + model.objects.create(name="web", data=payload_value) + + data = self._gql("{ custom_objects_config_list { name data } }") + rows = data["custom_objects_config_list"] + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["name"], "web") + self.assertEqual(rows[0]["data"], payload_value) + + def test_single_object_query_by_id(self): + cot = self.create_simple_custom_object_type(name="Note", slug="note") + model = cot.get_model() + instance = model.objects.create(name="Hello", description="world") + + data = self._gql( + f"{{ custom_objects_note(id: {instance.pk}) {{ id display name description }} }}" + ) + self.assertEqual(data["custom_objects_note"]["name"], "Hello") + self.assertEqual(data["custom_objects_note"]["description"], "world") + + def test_object_relationship_resolves_to_native_site_type(self): + # A single-object field pointing at a Site must resolve to NetBox's + # SiteType and be fully traversable (including nested relations like + # region) — not a flat stub. + region = Region.objects.create(name="West", slug="west") + site = self._make_site(name="HQ", slug="hq", region=region) + cot = self._site_object_field_type() + model = cot.get_model() + model.objects.create(name="S1", site=site) + + data = self._gql( + "{ custom_objects_server_list { name site { id name slug region { name } } } }" + ) + related = data["custom_objects_server_list"][0]["site"] + self.assertEqual(related["id"], str(site.pk)) + self.assertEqual(related["name"], "HQ") + self.assertEqual(related["slug"], "hq") + # Deep traversal into the related object's own relations proves it is the + # native SiteType, not the flat CustomObjectRelatedObjectType. + self.assertEqual(related["region"]["name"], "West") + + def test_multiobject_relationship_resolves_to_native_device_type(self): + device = self._make_device(name="dev-a") + # Slug 'group' would collide with NetBox's built-in `group_list` root query + # field (user groups) — but the `custom_objects_` prefix keeps it distinct, + # so this exercises both native multiobject resolution and the namespacing. + cot = self.create_multi_object_custom_object_type(name="Group", slug="group") + model = cot.get_model() + instance = model.objects.create(name="G1") + instance.devices.add(device) + + data = self._gql( + "{ custom_objects_group_list { name devices { id name role { name } } } }" + ) + devices = data["custom_objects_group_list"][0]["devices"] + self.assertEqual(len(devices), 1) + self.assertEqual(devices[0]["id"], str(device.pk)) + # 'name'/'role' are Device fields → confirms native DeviceType. + self.assertEqual(devices[0]["name"], "dev-a") + self.assertEqual(devices[0]["role"]["name"], "Role") + + def test_polymorphic_object_field_resolves_to_union(self): + # A polymorphic single-object field exposes a union of its target types; + # an instance pointing at a Site resolves through the SiteType arm. + site = self._make_site(name="PolySite", slug="polysite") + cot = self.create_custom_object_type(name="Binding", slug="binding") + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True + ) + self.create_polymorphic_field( + cot, + [self.get_site_object_type(), self.get_device_object_type()], + name="target", type="object", + ) + model = cot.get_model() + model.objects.create(name="B1", target=site) + + # Alias the per-type 'name' selections: Site.name is String! while + # Device.name is String, and GraphQL's same-response-shape rule forbids + # selecting both under one response key. + data = self._gql( + "{ custom_objects_binding_list { name target { " + "... on SiteType { id siteName: name } " + "... on DeviceType { id deviceName: name } } } }" + ) + target = data["custom_objects_binding_list"][0]["target"] + self.assertEqual(target["id"], str(site.pk)) + self.assertEqual(target["siteName"], "PolySite") + + def test_multiple_types_in_one_schema(self): + # Several custom object types must all be queryable from the same schema. + a = self.create_simple_custom_object_type(name="Alpha", slug="alpha") + b = self.create_simple_custom_object_type(name="Beta", slug="beta") + a.get_model().objects.create(name="a1") + b.get_model().objects.create(name="b1") + + data = self._gql( + "{ custom_objects_alpha_list { name } custom_objects_beta_list { name } }" + ) + self.assertEqual(data["custom_objects_alpha_list"][0]["name"], "a1") + self.assertEqual(data["custom_objects_beta_list"][0]["name"], "b1") + + def test_tags_and_base_fields(self): + cot = self.create_simple_custom_object_type(name="Doc", slug="doc") + model = cot.get_model() + model.objects.create(name="D1") + + data = self._gql("{ custom_objects_doc_list { id display created tags { name } } }") + row = data["custom_objects_doc_list"][0] + self.assertEqual(row["display"], "D1") + self.assertIsNotNone(row["id"]) + self.assertEqual(row["tags"], []) + + +@override_settings(LOGIN_REQUIRED=True) +class GraphQLPermissionTestCase(CustomObjectsTestCase, TestCase): + """ + Object-level view permissions must be enforced for objects reached through a + custom object's relationship fields, not just for the top-level objects. + """ + + def setUp(self): + super().setUp() + live_module.reset_cache() + self.addCleanup(live_module.reset_cache) + patcher = mock.patch( + "netbox_custom_objects.CustomObjectsPluginConfig." + "should_skip_dynamic_model_creation", + return_value=False, + ) + patcher.start() + self.addCleanup(patcher.stop) + + # A non-superuser; permissions are granted explicitly per test. + self.client = APIClient() + token_key = create_token(self.user) + self.header = {"HTTP_AUTHORIZATION": f"Token {token_key}"} + self.url = reverse("graphql") + + # COT 'Server' with a single-object field → Site, holding one Site. + self.site = Site.objects.create(name="Secret", slug="secret") + self.cot = self.create_custom_object_type(name="Server", slug="server") + self.create_custom_object_type_field( + self.cot, name="name", label="Name", type="text", primary=True, required=True + ) + self.create_custom_object_type_field( + self.cot, name="site", label="Site", type="object", + related_object_type=self.get_site_object_type(), + ) + self.model = self.cot.get_model() + self.model.objects.create(name="srv", site=self.site) + + def _grant(self, model_class, name): + perm = ObjectPermission(name=name, actions=["view"]) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(model_class)) + return perm + + def _post(self, query): + response = self.client.post( + self.url, data={"query": query}, format="json", **self.header + ) + self.assertEqual(response.status_code, status.HTTP_200_OK, response.content) + return json.loads(response.content) + + def test_related_object_hidden_without_permission(self): + # View permission on the custom object, but NOT on Site → the related + # site must be withheld (null), while the object itself is returned. + self._grant(self.model, "view-co") + payload = self._post("{ custom_objects_server_list { name site { id name } } }") + self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) + rows = payload["data"]["custom_objects_server_list"] + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["name"], "srv") + self.assertIsNone(rows[0]["site"]) + + def test_related_object_visible_with_permission(self): + # Granting view on Site as well makes the related object appear, proving + # the previous test's null was permission enforcement, not a broken field. + self._grant(self.model, "view-co") + self._grant(Site, "view-site") + payload = self._post("{ custom_objects_server_list { name site { id name } } }") + self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) + related = payload["data"]["custom_objects_server_list"][0]["site"] + self.assertEqual(related["id"], str(self.site.pk)) + self.assertEqual(related["name"], "Secret") + + def test_multiobject_related_filtered_by_permission(self): + # The multi-object resolver filters related objects through the batched + # permission check (_filter_viewable). Without view permission on Device + # the list is empty; granting it makes the device appear. + manufacturer = Manufacturer.objects.create(name="Mfr", slug="mfr") + device_type = DeviceType.objects.create( + manufacturer=manufacturer, model="Model", slug="model" + ) + role = DeviceRole.objects.create(name="Role", slug="role") + site = Site.objects.create(name="DevSite", slug="devsite") + device = Device.objects.create( + name="dev1", device_type=device_type, role=role, site=site + ) + cot = self.create_multi_object_custom_object_type(name="Group", slug="group") + model = cot.get_model() + instance = model.objects.create(name="G1") + instance.devices.add(device) + + # View on the custom object but NOT on Device → devices filtered out. + self._grant(model, "view-grp") + payload = self._post("{ custom_objects_group_list { name devices { id } } }") + self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) + rows = payload["data"]["custom_objects_group_list"] + self.assertEqual(rows[0]["name"], "G1") + self.assertEqual(rows[0]["devices"], []) + + # Granting view on Device makes it appear (batched check lets it through). + self._grant(Device, "view-dev") + payload = self._post("{ custom_objects_group_list { name devices { id } } }") + self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) + devices = payload["data"]["custom_objects_group_list"][0]["devices"] + self.assertEqual(len(devices), 1) + self.assertEqual(devices[0]["id"], str(device.pk)) + + def test_multiobject_related_filtered_to_permitted_subset(self): + # The previous tests grant view on the whole model (all-or-nothing). This + # exercises the core security claim at object level: a *constrained* + # ObjectPermission must filter related objects to just the permitted subset. + manufacturer = Manufacturer.objects.create(name="Mfr", slug="mfr") + device_type = DeviceType.objects.create( + manufacturer=manufacturer, model="Model", slug="model" + ) + role = DeviceRole.objects.create(name="Role", slug="role") + site = Site.objects.create(name="DevSite", slug="devsite") + allowed = Device.objects.create( + name="allowed", device_type=device_type, role=role, site=site + ) + denied = Device.objects.create( + name="denied", device_type=device_type, role=role, site=site + ) + cot = self.create_multi_object_custom_object_type(name="Fleet", slug="fleet") + model = cot.get_model() + instance = model.objects.create(name="F1") + instance.devices.add(allowed, denied) + + self._grant(model, "view-fleet") + # View on Device constrained to only the "allowed" device. + perm = ObjectPermission( + name="view-one-device", actions=["view"], constraints={"name": "allowed"} + ) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(Device)) + + payload = self._post( + "{ custom_objects_fleet_list { name devices { id name } } }" + ) + self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) + devices = payload["data"]["custom_objects_fleet_list"][0]["devices"] + self.assertEqual([d["name"] for d in devices], ["allowed"]) + self.assertEqual(devices[0]["id"], str(allowed.pk)) + + @override_settings(EXEMPT_VIEW_PERMISSIONS=["*"]) + def test_related_anonymous_access_honors_exempt_view_permissions(self): + # Anonymous/None traversal must defer to restrict(), exactly like NetBox's + # BaseObjectType.get_queryset: when the view is exempt, related objects are + # visible, not silently denied. + from django.contrib.auth.models import AnonymousUser + + from netbox_custom_objects.graphql.types import _filter_viewable + + self.assertEqual(_filter_viewable(AnonymousUser(), [self.site]), [self.site]) + self.assertEqual(_filter_viewable(None, [self.site]), [self.site]) + + @override_settings(EXEMPT_VIEW_PERMISSIONS=[]) + def test_related_anonymous_access_denied_without_exemption(self): + # And when the view is not exempt, an unauthenticated user sees nothing — + # again matching restrict()'s anonymous handling. + from django.contrib.auth.models import AnonymousUser + + from netbox_custom_objects.graphql.types import _filter_viewable + + self.assertEqual(_filter_viewable(AnonymousUser(), [self.site]), []) + self.assertEqual(_filter_viewable(None, [self.site]), []) diff --git a/netbox_custom_objects/tests/test_models.py b/netbox_custom_objects/tests/test_models.py index 1dd0690c..e093f23a 100644 --- a/netbox_custom_objects/tests/test_models.py +++ b/netbox_custom_objects/tests/test_models.py @@ -125,8 +125,8 @@ def test_custom_object_type_get_model_without_fields(self): custom_object_type = self.create_custom_object_type(name="TestObject") model = custom_object_type.get_model() - # Base fields: id, created, last_updated - self.assertEqual(len(model._meta.fields), 3) + # Base fields: id, created, last_updated, owner + self.assertEqual(len(model._meta.fields), 4) def test_custom_object_type_get_model_with_primary_field(self): """Test get_model method with a primary field.""" @@ -232,6 +232,10 @@ def test_register_search_index_skips_object_field_absent_from_stub_model(self): related_object_type=self.get_site_object_type(), search_weight=500, ) + # CustomObjectTypeField.save() now caches a full model after each field + # save (to defend against a rename/post_save race), so the cache holds a + # full model here. Clear it to force generation of a fresh stub. + cot.clear_model_cache(cot.id) stub_model = cot.get_model(skip_object_fields=True) model_field_names = ( {f.name for f in stub_model._meta.local_fields} @@ -382,6 +386,212 @@ def test_stale_registry_entry_causes_relation_error_on_related_object_delete(sel django_apps.clear_cache() +class CustomObjectTypeConfigContextTestCase(CustomObjectsTestCase, TestCase): + """Config context support (issue #98) on custom object types.""" + + @staticmethod + def _table_columns(table_name): + with connection.cursor() as cursor: + return { + col.name + for col in connection.introspection.get_table_description(cursor, table_name) + } + + def test_enabled_model_is_config_context_subclass_with_column(self): + """A config-context-enabled type generates a ConfigContextModel subclass + whose backing table has a local_context_data column.""" + from extras.models import ConfigContextModel + + cot = self.create_custom_object_type( + name="cc_enabled", slug="cc-enabled", config_context_enabled=True, + ) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + model = cot.get_model(no_cache=True) + self.assertTrue(issubclass(model, ConfigContextModel)) + self.assertIn("local_context_data", self._table_columns(cot.get_database_table_name())) + + def test_disabled_model_has_no_config_context(self): + """The default (flag False) type is not a ConfigContextModel and has no + local_context_data column.""" + from extras.models import ConfigContextModel + + cot = self.create_custom_object_type(name="cc_disabled", slug="cc-disabled") + self.assertFalse(cot.config_context_enabled) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + model = cot.get_model(no_cache=True) + self.assertFalse(issubclass(model, ConfigContextModel)) + self.assertNotIn("local_context_data", self._table_columns(cot.get_database_table_name())) + + def test_local_context_data_round_trips_and_get_config_context(self): + """An instance stores local_context_data and get_config_context() returns + it without touching the org-dimension aggregation that custom objects lack.""" + cot = self.create_custom_object_type( + name="cc_roundtrip", slug="cc-roundtrip", config_context_enabled=True, + ) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + model = cot.get_model(no_cache=True) + + obj = model(name="obj-1", local_context_data={"ntp_servers": ["10.0.0.1"]}) + obj.save() + obj.refresh_from_db() + self.assertEqual(obj.local_context_data, {"ntp_servers": ["10.0.0.1"]}) + # Safe override: returns the local data, never raises on missing site/tenant/role. + self.assertEqual(obj.get_config_context(), {"ntp_servers": ["10.0.0.1"]}) + + empty = model(name="obj-2") + empty.save() + self.assertEqual(empty.get_config_context(), {}) + + def test_clean_rejects_non_dict_local_context_data(self): + """The inherited ConfigContextModel.clean() validator rejects non-object JSON.""" + cot = self.create_custom_object_type( + name="cc_clean", slug="cc-clean", config_context_enabled=True, + ) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + model = cot.get_model(no_cache=True) + obj = model(name="bad", local_context_data=["not", "a", "dict"]) + with self.assertRaises(ValidationError): + obj.clean() + + def test_config_context_enabled_immutable_in_clean(self): + """full_clean() rejects flipping config_context_enabled on an existing type.""" + cot = self.create_custom_object_type(name="cc_clean_immut", slug="cc-clean-immut") + self.assertFalse(cot.config_context_enabled) + cot.config_context_enabled = True + with self.assertRaises(ValidationError): + cot.full_clean() + + def test_config_context_enabled_clean_allows_unchanged(self): + """Re-validating without changing the flag must not raise.""" + cot = self.create_custom_object_type( + name="cc_clean_ok", slug="cc-clean-ok", config_context_enabled=True, + ) + cot.description = "edited" + cot.full_clean() # should not raise + + # --- Source-context aggregation via field-naming convention (#98 phase 2) --- + + def _site_cot(self, name, slug): + """Config-context-enabled COT with a primary name field and a `site` object field.""" + cot = self.create_custom_object_type(name=name, slug=slug, config_context_enabled=True) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + self.create_custom_object_type_field( + cot, name="site", label="Site", type="object", + related_object_type=self.get_site_object_type(), + ) + return cot + + def test_aggregates_referenced_site_config_context(self): + """A `site` field pulls in ConfigContexts assigned to the referenced Site, + with local_context_data merged on top.""" + from extras.models import ConfigContext + + site = Site.objects.create(name="CC Site", slug="cc-site") + cc = ConfigContext.objects.create(name="cc-site-ctx", weight=1000, is_active=True, + data={"ntp": "10.0.0.1", "dns": "8.8.8.8"}) + cc.sites.add(site) + + cot = self._site_cot("cc_agg_site", "cc-agg-site") + model = cot.get_model(no_cache=True) + obj = model.objects.create(name="o1", site=site, local_context_data={"dns": "1.1.1.1"}) + + rendered = obj.get_config_context() + self.assertEqual(rendered["ntp"], "10.0.0.1") # from source context + self.assertEqual(rendered["dns"], "1.1.1.1") # local overrides source + + def test_no_convention_field_returns_local_only(self): + """Without a convention-named dimension field, no source aggregation happens.""" + from extras.models import ConfigContext + + # A global (unassigned) active context that WOULD match a dimensionful object. + ConfigContext.objects.create(name="global-ctx", weight=1000, is_active=True, data={"g": 1}) + + cot = self.create_custom_object_type(name="cc_nodim", slug="cc-nodim", config_context_enabled=True) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + model = cot.get_model(no_cache=True) + obj = model.objects.create(name="o1", local_context_data={"local": 2}) + + self.assertEqual(obj.get_config_context(), {"local": 2}) + + def test_misnamed_field_is_ignored(self): + """A `site`-named field pointing at the wrong model must not feed the site dimension.""" + from extras.models import ConfigContext + + site = Site.objects.create(name="CC Site 2", slug="cc-site-2") + cc = ConfigContext.objects.create(name="cc-site-ctx-2", weight=1000, is_active=True, data={"x": 1}) + cc.sites.add(site) + + # Field named `site` but pointing at Prefix (wrong model) → ignored. + cot = self.create_custom_object_type(name="cc_wrong", slug="cc-wrong", config_context_enabled=True) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + self.create_custom_object_type_field( + cot, name="site", label="Site", type="object", + related_object_type=self.get_prefix_object_type(), + ) + model = cot.get_model(no_cache=True) + obj = model.objects.create(name="o1", local_context_data={"only": "local"}) + + self.assertEqual(obj.get_config_context(), {"only": "local"}) + + def test_aggregates_referenced_tenant_config_context(self): + """The `tenant` dimension (a direct obj.tenant read in get_for_object) works.""" + from extras.models import ConfigContext + from tenancy.models import Tenant + + tenant = Tenant.objects.create(name="CC Tenant", slug="cc-tenant") + cc = ConfigContext.objects.create(name="cc-tenant-ctx", weight=1000, is_active=True, data={"t": 1}) + cc.tenants.add(tenant) + + cot = self.create_custom_object_type(name="cc_tenant", slug="cc-tenant", config_context_enabled=True) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + self.create_custom_object_type_field( + cot, name="tenant", label="Tenant", type="object", + related_object_type=ObjectType.objects.get(app_label="tenancy", model="tenant"), + ) + model = cot.get_model(no_cache=True) + obj = model.objects.create(name="o1", tenant=tenant) + + self.assertEqual(obj.get_config_context(), {"t": 1}) + + def test_aggregates_referenced_role_config_context(self): + """The `role` dimension (a direct obj.role read + get_ancestors) works.""" + from dcim.models import DeviceRole + from extras.models import ConfigContext + + role = DeviceRole.objects.create(name="CC Role", slug="cc-role", color="ff0000") + cc = ConfigContext.objects.create(name="cc-role-ctx", weight=1000, is_active=True, data={"r": 1}) + cc.roles.add(role) + + cot = self.create_custom_object_type(name="cc_role", slug="cc-role", config_context_enabled=True) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + self.create_custom_object_type_field( + cot, name="role", label="Role", type="object", + related_object_type=ObjectType.objects.get(app_label="dcim", model="devicerole"), + ) + model = cot.get_model(no_cache=True) + obj = model.objects.create(name="o1", role=role) + + self.assertEqual(obj.get_config_context(), {"r": 1}) + + class CustomObjectTypeFieldTestCase(CustomObjectsTestCase, TestCase): """Test cases for CustomObjectTypeField model.""" @@ -427,6 +637,17 @@ def test_custom_object_type_field_name_validation(self): ) field.full_clean() + def test_custom_object_type_field_reserved_name_rejected(self): + """Field names in RESERVED_FIELD_NAMES must be rejected with ValidationError.""" + for reserved in ("owner", "tags", "id", "created", "last_updated", "local_context_data"): + with self.assertRaises(ValidationError, msg=f"Expected ValidationError for reserved name={reserved!r}"): + field = CustomObjectTypeField( + custom_object_type=self.custom_object_type, + name=reserved, + type="text", + ) + field.full_clean() + def test_custom_object_type_field_unique_name_per_type(self): """Test that field names must be unique within a custom object type.""" self.create_custom_object_type_field( @@ -932,6 +1153,35 @@ def test_str_falls_back_when_primary_field_raises_attribute_error(self): self.assertEqual(result, expected) +class M2MSerializationRegressionTestCase(CustomObjectsTestCase, TestCase): + """Guards the ``through._meta.auto_created = model`` opt-in in + ``MultiObjectFieldType.after_model_generation`` — without it, Django's + JSON serializer skips the M2M and merge replay zeroes through-table rows. + """ + + def test_serialize_object_includes_m2m_values(self): + cot = self.create_custom_object_type(name="M2MSerialize", slug="m2m-serialize") + site_ct = self.get_site_object_type() + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, + ) + self.create_custom_object_type_field( + cot, name="sites", label="Sites", type="multiobject", + related_object_type=site_ct, + ) + model = cot.get_model() + site_model = site_ct.model_class() + site_a = site_model.objects.create(name="A", slug="a") + site_b = site_model.objects.create(name="B", slug="b") + + obj = model.objects.create(name="obj") + obj.sites.set([site_a, site_b]) + + data = obj.serialize_object() + self.assertIn("sites", data) + self.assertEqual(set(data["sites"]), {site_a.pk, site_b.pk}) + + class RelatedNameTestCase(CustomObjectsTestCase, TestCase): """Tests for the related_name field on Object and MultiObject fields.""" @@ -1604,12 +1854,16 @@ def test_stub_model_search_index_excludes_absent_fields(self): fields are excluded from the model class. The search index must only contain fields that are actually present. """ - # Refresh so cache_timestamp matches the DB value bumped by the signal during setUp, - # then explicitly generate the stub to re-register its search index (which excludes - # object-type fields). Without refresh_from_db the timestamps mismatch → full model - # is generated instead, which includes 'device' and makes assertNotIn fail. - self.target_cot.refresh_from_db() - self.target_cot.get_model(skip_object_fields=True) + # Force stub semantics: clear any cached full model and re-register the + # search index against a fresh stub. This mirrors the cross-COT scenario + # the regression covers — B's stub gets cached before any caller asks for + # the full model, so register_custom_object_search_index sees only the + # stub fields. Without this, CustomObjectTypeField.save() would have + # already cached a full model with the OBJECT field. + self.target_cot.clear_model_cache(self.target_cot.id) + stub_model = self.target_cot.get_model(skip_object_fields=True) + self.target_cot.register_custom_object_search_index(stub_model) + label = f"netbox_custom_objects.{self.target_cot.get_table_model_name(self.target_cot.id).lower()}" search_index = registry["search"].get(label) @@ -2143,10 +2397,11 @@ class LazySerializerRegistrationTestCase(CustomObjectsTestCase, TestCase): 1. get_serializer_class(model, skip_object_fields=True) used to overwrite the full module-level serializer with a partial one, causing subsequent requests to fail or return incomplete data. - 2. import_string("netbox_custom_objects.api.serializers.Table{N}ModelSerializer") - raised SerializerNotFound in workers whose ready() ran before the COT existed. - The module-level __getattr__ hook (PEP 562) fixes this by generating the - serializer on demand. + 2. get_serializer_for_model() raised SerializerNotFound for a Table{N}Model in + workers whose ready() ran before the COT existed. + The serializer_resolver() hook (registered via PluginConfig.serializer_resolver) + fixes this by generating the serializer on demand, ahead of any import-path + lookup. """ @classmethod @@ -2173,8 +2428,8 @@ def setUpTestData(cls): def _clear_serializer_registrations(self): """Remove any previously registered serializers for parent and child COTs - from the module so that __getattr__ and get_serializer_class() behave as - they would in a fresh worker process.""" + from the module so that serializer_resolver() and get_serializer_class() + behave as they would in a fresh worker process.""" import netbox_custom_objects.api.serializers as ser_module parent_model = self.parent_cot.get_model() child_model = self.child_cot.get_model() @@ -2182,10 +2437,14 @@ def _clear_serializer_registrations(self): attr = f"{model._meta.object_name}Serializer" ser_module.__dict__.pop(attr, None) - def test_module_getattr_generates_serializer_on_demand(self): - """__getattr__ must generate and return a serializer for any Table{N}Model - whose serializer was never pre-registered (simulates a worker that started - before the COT was created — issue #370 Scenario A).""" + def test_resolver_generates_serializer_on_demand(self): + """serializer_resolver() must generate and return a serializer for any + Table{N}Model whose serializer was never pre-registered (simulates a worker + that started before the COT was created — issue #370 Scenario A). + + NetBox calls this resolver from get_serializer_for_model() before falling + back to an import-path lookup, so a missing startup-time registration can + never raise SerializerNotFound.""" import netbox_custom_objects.api.serializers as ser_module self._clear_serializer_registrations() @@ -2196,14 +2455,15 @@ def test_module_getattr_generates_serializer_on_demand(self): self.assertNotIn(serializer_name, ser_module.__dict__, "precondition: serializer must not be pre-registered") - # getattr() must trigger __getattr__ and return a valid class - serializer_cls = getattr(ser_module, serializer_name) + # The resolver must generate and return a valid serializer class on demand + serializer_cls = ser_module.serializer_resolver(parent_model) self.assertIsNotNone(serializer_cls) self.assertIn("title", serializer_cls.Meta.fields) - # After __getattr__ fires, the serializer is cached in __dict__ + # After the resolver fires, the full serializer is cached in __dict__ + # (get_serializer_class registers it via setattr) self.assertIn(serializer_name, ser_module.__dict__, - "serializer must be cached in module __dict__ after first access") + "serializer must be cached in module __dict__ after first resolution") def test_skip_object_fields_does_not_overwrite_full_serializer(self): """get_serializer_class(model, skip_object_fields=True) must not overwrite diff --git a/netbox_custom_objects/tests/test_polymorphic_fields.py b/netbox_custom_objects/tests/test_polymorphic_fields.py index 295edf46..6182dd99 100644 --- a/netbox_custom_objects/tests/test_polymorphic_fields.py +++ b/netbox_custom_objects/tests/test_polymorphic_fields.py @@ -16,24 +16,21 @@ from dcim.models import Site from ipam.models import Prefix, IPAddress from ipam.choices import PrefixStatusChoices -from users.models import ObjectPermission, Token +from users.models import ObjectPermission from netbox_custom_objects.constants import APP_LABEL from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField -from netbox_custom_objects.tests.base import CustomObjectsTestCase, TransactionCleanupMixin +from netbox_custom_objects.tests.base import ( + CustomObjectsTestCase, + TransactionCleanupMixin, + create_token, +) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -def _create_token(user): - from users.choices import TokenVersionChoices - t = Token(version=TokenVersionChoices.V1, user=user) - t.save() - return t.token # plaintext for V1 tokens - - def _grant_perm(user, action, model_class, name=None): perm = ObjectPermission(name=name or f"poly-test-{action}", actions=[action]) perm.save() @@ -57,7 +54,7 @@ def setUp(self): from django.test import Client as DjangoClient from utilities.testing import create_test_user self.user = create_test_user("poly-api-user") - token_key = _create_token(self.user) + token_key = create_token(self.user) self.header = {"HTTP_AUTHORIZATION": f"Token {token_key}"} # Reset client to clear the session cookie set by CustomObjectsTestCase.setUp() # (force_login causes SessionAuthentication to take priority over TokenAuthentication) diff --git a/netbox_custom_objects/tests/test_schema_operations.py b/netbox_custom_objects/tests/test_schema_operations.py index db5f2e42..582cc9c6 100644 --- a/netbox_custom_objects/tests/test_schema_operations.py +++ b/netbox_custom_objects/tests/test_schema_operations.py @@ -8,9 +8,11 @@ from django.apps import apps from django.core.management import call_command +from django.db import connection from django.test import TransactionTestCase from netbox_custom_objects.constants import APP_LABEL +from netbox_custom_objects.models import CustomObjectTypeField from .base import CustomObjectsTestCase, TransactionCleanupMixin @@ -212,3 +214,64 @@ def test_collectstatic_without_database(self): stderr=err, ) # No uncaught exceptions reaching here means success. + + # ------------------------------------------------------------------ + # Coordinates fields expand into two backing columns; verify the schema + # editor manages both on rename and delete. + # ------------------------------------------------------------------ + + def _db_columns(self, model): + """Return the set of actual DB column names for a generated model's table.""" + with connection.cursor() as cursor: + return { + col.name + for col in connection.introspection.get_table_description( + cursor, model._meta.db_table + ) + } + + def test_coordinates_field_rename_renames_both_columns(self): + """Renaming a coordinates field renames both backing DB columns.""" + cot = self.create_custom_object_type(name='coordrename', slug='coord-rename') + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, + ) + field = self.create_custom_object_type_field( + cot, name='location', label='Location', type='coordinates', + ) + + columns = self._db_columns(cot.get_model()) + self.assertIn('location_latitude', columns) + self.assertIn('location_longitude', columns) + + # Reload from DB so the rename path has the original snapshot (set in + # from_db) — this mirrors how the edit view loads the field before saving. + field = CustomObjectTypeField.objects.get(pk=field.pk) + field.name = 'geo' + field.save() + + columns = self._db_columns(cot.get_model()) + self.assertNotIn('location_latitude', columns) + self.assertNotIn('location_longitude', columns) + self.assertIn('geo_latitude', columns) + self.assertIn('geo_longitude', columns) + + def test_coordinates_field_delete_drops_both_columns(self): + """Deleting a coordinates field drops both backing DB columns.""" + cot = self.create_custom_object_type(name='coorddelete', slug='coord-delete') + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, + ) + field = self.create_custom_object_type_field( + cot, name='location', label='Location', type='coordinates', + ) + + columns = self._db_columns(cot.get_model()) + self.assertIn('location_latitude', columns) + self.assertIn('location_longitude', columns) + + field.delete() + + columns = self._db_columns(cot.get_model()) + self.assertNotIn('location_latitude', columns) + self.assertNotIn('location_longitude', columns) diff --git a/netbox_custom_objects/tests/test_views.py b/netbox_custom_objects/tests/test_views.py index 26da1e9f..a7944dfe 100644 --- a/netbox_custom_objects/tests/test_views.py +++ b/netbox_custom_objects/tests/test_views.py @@ -985,3 +985,287 @@ def test_object_selector_search(self): 'q': 'Alpha', }) self.assertEqual(response.status_code, 200) + + +class CoordinatesFieldViewTest(CustomObjectsTestCase, TestCase): + """UI form behaviour for the coordinates field type.""" + + @classmethod + def setUpTestData(cls): + cls.cot = CustomObjectType.objects.create( + name="GeoView", + verbose_name_plural="Geo Views", + slug="geo-views", + ) + CustomObjectTypeField.objects.create( + custom_object_type=cls.cot, name="name", type="text", primary=True, required=True + ) + CustomObjectTypeField.objects.create( + custom_object_type=cls.cot, name="location", type="coordinates" + ) + cls.model = cls.cot.get_model() + + def setUp(self): + super().setUp() + perm = ObjectPermission( + name="geo view all", actions=["view", "add", "change", "delete"] + ) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(self.model)) + + def _add_url(self): + return reverse( + "plugins:netbox_custom_objects:customobject_add", + kwargs={"custom_object_type": self.cot.slug}, + ) + + def test_add_form_renders_two_inputs(self): + response = self.client.get(self._add_url()) + self.assertEqual(response.status_code, 200) + self.assertContains(response, "location_latitude") + self.assertContains(response, "location_longitude") + + def test_create_valid_coordinates(self): + from decimal import Decimal + data = { + "name": "Box", + "location_latitude": "40.712800", + "location_longitude": "-74.006000", + } + response = self.client.post(self._add_url(), data) + self.assertEqual(response.status_code, 302, getattr(response, "content", b"")) + obj = self.model.objects.get(name="Box") + self.assertEqual(obj.location_latitude, Decimal("40.712800")) + self.assertEqual(obj.location_longitude, Decimal("-74.006000")) + + def test_create_half_populated_pair_rejected(self): + data = {"name": "Bad", "location_latitude": "40.712800"} + response = self.client.post(self._add_url(), data) + self.assertEqual(response.status_code, 200) + self.assertFalse(self.model.objects.filter(name="Bad").exists()) + + def _bulk_edit_url(self): + return reverse( + "plugins:netbox_custom_objects:customobject_bulk_edit", + kwargs={"custom_object_type": self.cot.slug}, + ) + + def test_bulk_edit_half_populated_pair_rejected(self): + """Bulk-editing only one of latitude/longitude is rejected.""" + from decimal import Decimal + obj = self.model.objects.create( + name="Existing", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + data = { + "pk": [obj.pk], + "_apply": "Apply", + "location_latitude": "10.000000", + } + response = self.client.post(self._bulk_edit_url(), data) + self.assertEqual(response.status_code, 200) + obj.refresh_from_db() + # Values are unchanged because the form failed validation. + self.assertEqual(obj.location_latitude, Decimal("40.712800")) + self.assertEqual(obj.location_longitude, Decimal("-74.006000")) + + +class QuickAddViewTestCase(CustomObjectsTestCase, TestCase): + """ + Tests for the quick-add flow in CustomObjectEditView. + + Covers GET (modal renders), POST success (object created, quick_add_created.html + returned), and POST validation failure (errors re-rendered in our custom template). + """ + + def setUp(self): + super().setUp() + self.user.is_superuser = True + self.user.save() + + # Target COT: objects of this type will be quick-added. + self.target_cot = self.create_simple_custom_object_type( + name='Target', slug='target', + ) + target_ot = ObjectType.objects.get( + app_label='netbox_custom_objects', + model=self.target_cot.get_table_model_name(self.target_cot.id).lower(), + ) + + # Source COT: has an object field pointing at Target. + self.source_cot = self.create_custom_object_type(name='Source', slug='source') + self.create_custom_object_type_field( + self.source_cot, name='name', label='Name', type='text', + primary=True, required=True, + ) + self.create_custom_object_type_field( + self.source_cot, name='ref', label='Ref', type='object', + related_object_type=target_ot, + ) + + self.add_url = reverse( + 'plugins:netbox_custom_objects:customobject_add', + kwargs={'custom_object_type': self.target_cot.slug}, + ) + + def test_quick_add_get_returns_200(self): + """GET ?_quickadd=True renders the custom quick-add modal without errors.""" + response = self.client.get( + self.add_url, + {'_quickadd': 'True', 'target': 'id_ref'}, + ) + self.assertEqual(response.status_code, 200) + # The custom template (not the core one) is used. + self.assertContains(response, 'hx-post=') + self.assertContains(response, f'/plugins/custom-objects/{self.target_cot.slug}/add/') + + def test_quick_add_post_success_creates_object(self): + """POST with _quickadd in POST data creates the object and returns quick_add_created template.""" + model = self.target_cot.get_model() + count_before = model.objects.count() + + response = self.client.post( + f'{self.add_url}?_quickadd=True&target=id_ref', + data={'quickadd-name': 'quick-created', '_quickadd': ''}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(model.objects.count(), count_before + 1) + # The success template contains the object PK for JS auto-selection. + self.assertContains(response, 'quick-add-object') + self.assertTrue(model.objects.filter(name='quick-created').exists()) + + def test_quick_add_post_validation_failure_rerenders(self): + """POST with missing required field re-renders the quick-add form with errors.""" + response = self.client.post( + f'{self.add_url}?_quickadd=True&target=id_ref', + # name is required but omitted + data={'_quickadd': ''}, + ) + self.assertEqual(response.status_code, 200) + # Error re-render uses our custom template, not a redirect. + self.assertContains(response, 'hx-post=') + # No new object created. + model = self.target_cot.get_model() + self.assertFalse(model.objects.exists()) + + +class CustomObjectConfigContextViewTestCase(CustomObjectsTestCase, TestCase): + """Config context tab on custom object instances (#98).""" + + def _config_context_url(self, cot, instance): + return reverse( + 'plugins:netbox_custom_objects:customobject_configcontext', + kwargs={'custom_object_type': cot.slug, 'pk': instance.pk}, + ) + + def _grant_view(self, model): + perm = ObjectPermission(name=f'view-{model._meta.model_name}', actions=['view']) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(model)) + return perm + + def test_tab_returns_200_and_shows_local_data_when_enabled(self): + cot = self.create_custom_object_type( + name='cc_view', slug='cc-view', config_context_enabled=True, + ) + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, required=True, + ) + model = cot.get_model() + obj = model.objects.create(name='obj-1', local_context_data={'ntp_servers': ['10.0.0.1']}) + self._grant_view(model) + + response = self.client.get(self._config_context_url(cot, obj)) + self.assertEqual(response.status_code, 200) + self.assertContains(response, 'ntp_servers') + self.assertContains(response, 'Config Context') + + def test_tab_lists_aggregated_source_contexts(self): + """A `site` field surfaces the referenced Site's ConfigContext in the tab.""" + from dcim.models import Site + from extras.models import ConfigContext + + site = Site.objects.create(name='Tab Site', slug='tab-site') + cc = ConfigContext.objects.create(name='tab-site-ctx', weight=1000, is_active=True, data={'a': 1}) + cc.sites.add(site) + + cot = self.create_custom_object_type( + name='cc_src', slug='cc-src', config_context_enabled=True, + ) + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, required=True, + ) + self.create_custom_object_type_field( + cot, name='site', label='Site', type='object', + related_object_type=self.get_site_object_type(), + ) + model = cot.get_model() + obj = model.objects.create(name='o1', site=site) + self._grant_view(model) + # The Source Contexts panel restricts to ConfigContexts the user may view + # (matching NetBox's ObjectConfigContextView), so grant that too. + self._grant_view(ConfigContext) + + response = self.client.get(self._config_context_url(cot, obj)) + self.assertEqual(response.status_code, 200) + self.assertContains(response, 'Source Contexts') + self.assertContains(response, 'tab-site-ctx') + + def test_tab_403_without_view_permission(self): + """The tab must honour object-level RBAC, not leak local_context_data.""" + cot = self.create_custom_object_type( + name='cc_rbac', slug='cc-rbac', config_context_enabled=True, + ) + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, required=True, + ) + model = cot.get_model() + obj = model.objects.create(name='secret', local_context_data={'k': 'v'}) + + # No ObjectPermission granted → restricted queryset yields nothing → 404. + response = self.client.get(self._config_context_url(cot, obj)) + self.assertIn(response.status_code, (403, 404)) + self.assertNotContains(response, 'secret', status_code=response.status_code) + + def test_tab_link_present_only_when_enabled(self): + """The detail page shows the Config Context tab only for enabled types.""" + enabled = self.create_custom_object_type( + name='cc_on', slug='cc-on', config_context_enabled=True, + ) + self.create_custom_object_type_field( + enabled, name='name', label='Name', type='text', primary=True, required=True, + ) + disabled = self.create_custom_object_type(name='cc_no', slug='cc-no') + self.create_custom_object_type_field( + disabled, name='name', label='Name', type='text', primary=True, required=True, + ) + on_model = enabled.get_model() + off_model = disabled.get_model() + on_obj = on_model.objects.create(name='on') + off_obj = off_model.objects.create(name='off') + + # The detail view (generic.ObjectView) enforces object-level view perms. + perm = ObjectPermission(name='view-cc-detail', actions=['view']) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(on_model)) + perm.object_types.add(ObjectType.objects.get_for_model(off_model)) + + on_detail = self.client.get(on_obj.get_absolute_url()) + self.assertContains(on_detail, self._config_context_url(enabled, on_obj)) + + off_detail = self.client.get(off_obj.get_absolute_url()) + self.assertNotContains(off_detail, 'config-context') + + def test_tab_404_when_disabled(self): + cot = self.create_custom_object_type(name='cc_off_view', slug='cc-off-view') + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, required=True, + ) + obj = cot.get_model().objects.create(name='x') + + response = self.client.get(self._config_context_url(cot, obj)) + self.assertEqual(response.status_code, 404) diff --git a/netbox_custom_objects/urls.py b/netbox_custom_objects/urls.py index a95a001d..87e755dd 100644 --- a/netbox_custom_objects/urls.py +++ b/netbox_custom_objects/urls.py @@ -79,4 +79,14 @@ views.CustomObjectChangeLogView.as_view(), name="customobject_changelog", ), + path( + "//contacts/", + views.CustomObjectContactsView.as_view(), + name="customobject_contacts", + ), + path( + "//config-context/", + views.CustomObjectConfigContextView.as_view(), + name="customobject_configcontext", + ), ] diff --git a/netbox_custom_objects/utilities.py b/netbox_custom_objects/utilities.py index 432b90bb..b50fb81a 100644 --- a/netbox_custom_objects/utilities.py +++ b/netbox_custom_objects/utilities.py @@ -9,13 +9,48 @@ __all__ = ( "AppsProxy", + "build_map_url", "extract_cot_id_from_model_name", "generate_model", "get_viewname", "install_clear_cache_suppressor", - "is_in_branch", ) + +def build_map_url(latitude, longitude): + """ + Build a map URL for a latitude/longitude pair, mirroring NetBox core's handling. + + Uses NetBox's ``MAPS_URL`` configuration parameter (default Google Maps). If that + value contains ``{lat}``/``{lon}`` placeholders they are substituted; otherwise the + coordinates are appended as a comma-separated pair. Returns ``None`` when either + coordinate is unset or no maps URL is configured. + + NetBox's own ``build_coords_url`` helper is used when available (NetBox >= 4.6.2); + on earlier supported versions we replicate its (simple) behaviour locally. + """ + if latitude is None or longitude is None: + return None + + try: + from netbox.config import get_config + + maps_url = get_config().MAPS_URL + except (ImportError, AttributeError): + maps_url = "https://maps.google.com/?q=" + + if not maps_url: + return None + + try: + from netbox.ui.utils import build_coords_url + + return build_coords_url(maps_url, latitude, longitude) + except ImportError: + if "{lat}" in maps_url or "{lon}" in maps_url: + return maps_url.replace("{lat}", str(latitude)).replace("{lon}", str(longitude)) + return f"{maps_url}{latitude},{longitude}" + # --------------------------------------------------------------------------- # Thread-safe apps.clear_cache suppression # --------------------------------------------------------------------------- @@ -31,6 +66,7 @@ # suppressed, and only for the duration of the critical window. # --------------------------------------------------------------------------- + _suppress_tl = threading.local() # thread-local suppression depth counter _real_clear_cache = None # set by install_clear_cache_suppressor() @@ -194,18 +230,3 @@ def generate_model(*args, **kwargs): model = type(*args, **kwargs) return model - - -def is_in_branch(): - """ - Check if currently operating within a branch. - - Returns: - bool: True if currently in a branch, False otherwise. - """ - try: - from netbox_branching.contextvars import active_branch - return active_branch.get() is not None - except ImportError: - # Branching plugin not installed - return False diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index 66cc6a61..e8ea8773 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -1,22 +1,30 @@ import logging from core.models import ObjectChange +from core.signals import clear_events from core.tables import ObjectChangeTable from django.apps import apps as django_apps +from django.contrib import messages from django.contrib.contenttypes.models import ContentType +from django.db import router, transaction from django.db.models import ProtectedError, Q, RestrictedError +from django.http import Http404 from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse +from django.utils.html import escape +from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ +from utilities.exceptions import AbortRequest, PermissionsViolation from django.views.generic import View from extras.choices import CustomFieldUIVisibleChoices from extras.forms import JournalEntryForm -from extras.models import JournalEntry +from extras.models import ConfigContext, JournalEntry from extras.tables import JournalEntryTable from netbox.forms import ( NetBoxModelBulkEditForm, NetBoxModelImportForm, ) +from netbox.forms.mixins import OwnerMixin as OwnerFormMixin from netbox.views import generic from netbox.views.generic.mixins import TableMixin from utilities.forms import ConfirmationForm, DeleteForm, restrict_form_fields @@ -35,13 +43,23 @@ from . import field_types, filtersets, forms, tables from .models import CustomObject, CustomObjectType, CustomObjectTypeField from extras.choices import CustomFieldTypeChoices +from netbox_custom_objects.choices import CustomObjectFieldTypeChoices from netbox_custom_objects.constants import APP_LABEL from netbox_custom_objects.dynamic_forms import build_filterset_form_class -from netbox_custom_objects.utilities import extract_cot_id_from_model_name, is_in_branch +from netbox_custom_objects.utilities import extract_cot_id_from_model_name logger = logging.getLogger("netbox_custom_objects.views") +def _is_in_branch(): + """True if a netbox-branching branch is active in this context.""" + try: + from netbox_branching.contextvars import active_branch + return active_branch.get() is not None + except ImportError: + return False + + # --------------------------------------------------------------------------- # Sub-field naming helpers for polymorphic form fields # @@ -328,9 +346,6 @@ class CustomObjectTypeEditView(generic.ObjectEditView): form = forms.CustomObjectTypeForm template_name = 'netbox_custom_objects/customobjecttype_edit.html' - def get_extra_context(self, request, instance): - return {'branch_bypass_warning': is_in_branch()} - @register_model_view(CustomObjectType, "delete") class CustomObjectTypeDeleteView(generic.ObjectDeleteView): @@ -338,9 +353,6 @@ class CustomObjectTypeDeleteView(generic.ObjectDeleteView): default_return_url = "plugins:netbox_custom_objects:customobjecttype_list" template_name = 'netbox_custom_objects/customobjecttype_delete.html' - def get_extra_context(self, request, instance): - return {'branch_bypass_warning': is_in_branch()} - def _get_dependent_objects(self, obj): dependent_objects = super()._get_dependent_objects(obj) model = obj.get_model_with_serializer() @@ -401,7 +413,7 @@ def alter_object(self, obj, request, url_args, url_kwargs): return obj def get_extra_context(self, request, instance): - return {'branch_bypass_warning': is_in_branch()} + return {'branch_bypass_warning': _is_in_branch()} @register_model_view(CustomObjectTypeField, "delete") @@ -490,9 +502,6 @@ def _get_dependent_objects(self, obj): return dependent_objects - def get_extra_context(self, request, instance): - return {'branch_bypass_warning': is_in_branch()} - @register_model_view(CustomObjectType, "bulk_import", path="import", detail=False) class CustomObjectTypeBulkImportView(generic.BulkImportView): @@ -500,9 +509,6 @@ class CustomObjectTypeBulkImportView(generic.BulkImportView): model_form = forms.CustomObjectTypeImportForm template_name = 'netbox_custom_objects/customobjecttype_bulk_import.html' - def get_extra_context(self, request): - return {'branch_bypass_warning': is_in_branch()} - @register_model_view(CustomObjectType, "bulk_edit", path="edit", detail=False) class CustomObjectTypeBulkEditView(generic.BulkEditView): @@ -512,9 +518,6 @@ class CustomObjectTypeBulkEditView(generic.BulkEditView): form = forms.CustomObjectTypeBulkEditForm template_name = 'netbox_custom_objects/customobjecttype_bulk_edit.html' - def get_extra_context(self, request): - return {'branch_bypass_warning': is_in_branch()} - @register_model_view(CustomObjectType, "bulk_delete", path="delete", detail=False) class CustomObjectTypeBulkDeleteView(generic.BulkDeleteView): @@ -523,9 +526,6 @@ class CustomObjectTypeBulkDeleteView(generic.BulkDeleteView): table = tables.CustomObjectTypeTable template_name = 'netbox_custom_objects/customobjecttype_bulk_delete.html' - def get_extra_context(self, request): - return {'branch_bypass_warning': is_in_branch()} - # # Custom Objects @@ -616,6 +616,7 @@ def get_extra_context(self, request, instance): class CustomObjectEditView(generic.ObjectEditView): template_name = "netbox_custom_objects/customobject_edit.html" htmx_template_name = "netbox_custom_objects/htmx/edit_fields.html" + _CO_QUICK_ADD_TEMPLATE = 'netbox_custom_objects/htmx/co_quick_add.html' form = None queryset = None object = None @@ -639,6 +640,76 @@ def get_queryset(self, request): model = self.object._meta.model return model.objects.all() + def get(self, request, *args, **kwargs): + if not request.GET.get('_quickadd'): + return super().get(request, *args, **kwargs) + # Quick-add GET: the core htmx/quick_add.html uses {% action_url model 'add' %} + # which cannot resolve custom object URLs (they require a COT slug). + # Use our own template that builds the form action URL from the slug instead. + obj = self.object # already populated by setup() + form = self.form( + instance=obj, + initial=normalize_querydict(request.GET), + prefix='quickadd', + ) + restrict_form_fields(form, request.user) + return render(request, self._CO_QUICK_ADD_TEMPLATE, { + 'model': obj._meta.model, + 'object': obj, + 'form': form, + }) + + def post(self, request, *args, **kwargs): + if '_quickadd' not in request.POST: + return super().post(request, *args, **kwargs) + # Quick-add POST: mirrors the parent's save logic but re-renders validation + # errors with our custom template instead of the core htmx/quick_add.html + # (which uses {% action_url model 'add' %} and fails for COT models). + _qa_logger = logging.getLogger('netbox.views.ObjectEditView') + obj = self.object # already populated by setup() + model = self.queryset.model + + if obj.pk and hasattr(obj, 'snapshot'): + obj.snapshot() + + obj = self.alter_object(obj, request, args, kwargs) + + form = self.form( + data=request.POST, + files=request.FILES, + instance=obj, + prefix='quickadd', + ) + restrict_form_fields(form, request.user) + + if form.is_valid(): + obj._changelog_message = form.cleaned_data.pop('changelog_message', '') + try: + with transaction.atomic(using=router.db_for_write(model)): + object_created = form.instance.pk is None + obj = form.save() + if not self.queryset.filter(pk=obj.pk).exists(): + raise PermissionsViolation() + msg = '{} {}'.format( + 'Created' if object_created else 'Modified', + model._meta.verbose_name, + ) + _qa_logger.info(f"{msg} {obj} (PK: {obj.pk})") + if hasattr(obj, 'get_absolute_url'): + msg = mark_safe(f'{msg} {escape(obj)}') + messages.success(request, msg) + return render(request, 'htmx/quick_add_created.html', {'object': obj}) + except (AbortRequest, PermissionsViolation) as e: + _qa_logger.debug(e.message) + form.add_error(None, e.message) + clear_events.send(sender=self) + + return render(request, self._CO_QUICK_ADD_TEMPLATE, { + 'model': obj._meta.model, + 'object': obj, + 'form': form, + }) + def get_object(self, **kwargs): if self.object: return self.object @@ -692,6 +763,8 @@ def get_form(self, model): "custom_object_type_poly_obj_ct_names": set(), # Maps ct_sub → (obj_sub, field_label) for poly object pair rendering in the template "custom_object_type_poly_obj_pairs": {}, + # Maps coordinates field name → (latitude_field_name, longitude_field_name) + "custom_object_type_coordinates_fields": {}, } # Process custom object type fields (with grouping) @@ -701,6 +774,19 @@ def get_form(self, model): field_type = field_types.FIELD_TYPE_CLASS[field.type]() group_name = field.group_name or None + # Coordinates: one logical field rendered as two grouped latitude/longitude inputs + if field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + sub_fields = field_type.get_form_fields(field) + sub_names = list(sub_fields.keys()) + for sub_name, sub_field in sub_fields.items(): + attrs[sub_name] = sub_field + attrs["custom_object_type_rendered_names"].add(sub_name) + if group_name not in attrs["custom_object_type_field_groups"]: + attrs["custom_object_type_field_groups"][group_name] = [] + attrs["custom_object_type_field_groups"][group_name].extend(sub_names) + attrs["custom_object_type_coordinates_fields"][field.name] = tuple(sub_names) + continue + # Polymorphic single-object: type-selector + object-picker pair if field.is_polymorphic and field.type == CustomFieldTypeChoices.TYPE_OBJECT: ct_sub = f"{field.name}__ct" @@ -756,7 +842,7 @@ def get_form(self, model): form_class = type( f"{model._meta.object_name}Form", - (forms.NetBoxModelForm,), + (OwnerFormMixin, forms.NetBoxModelForm,), attrs, ) @@ -771,6 +857,7 @@ def custom_init(self, *args, **kwargs): self.custom_object_type_poly_obj_fields = attrs["custom_object_type_poly_obj_fields"] self.custom_object_type_poly_obj_ct_names = attrs["custom_object_type_poly_obj_ct_names"] self.custom_object_type_poly_obj_pairs = attrs["custom_object_type_poly_obj_pairs"] + self.custom_object_type_coordinates_fields = attrs["custom_object_type_coordinates_fields"] instance = kwargs.get('instance', None) @@ -938,6 +1025,15 @@ def custom_clean(self): obj_sub, _("Please select an object of the chosen type."), ) + # Coordinates: latitude and longitude must both be set or both be empty. + for field_name, (lat_name, lon_name) in self.custom_object_type_coordinates_fields.items(): + latitude = self.cleaned_data.get(lat_name) + longitude = self.cleaned_data.get(lon_name) + if (latitude is None) != (longitude is None): + self.add_error( + lat_name if latitude is None else lon_name, + _("Latitude and longitude must both be set or both be empty."), + ) return self.cleaned_data form_class.__init__ = custom_init @@ -946,11 +1042,6 @@ def custom_clean(self): return form_class - def get_extra_context(self, request, obj): - return { - 'branch_warning': is_in_branch(), - } - @register_model_view(CustomObject, "delete") class CustomObjectDeleteView(generic.ObjectDeleteView): @@ -1034,7 +1125,7 @@ def _get_dependent_objects(self, obj): } def get_extra_context(self, request, instance): - return {'branch_warning': is_in_branch()} + return {'branch_warning': _is_in_branch()} @register_model_view(CustomObject, "bulk_edit", path="edit", detail=False) @@ -1101,11 +1192,26 @@ def get_form(self, queryset): "custom_object_type_poly_obj_pairs": {}, "custom_object_type_poly_m2m_groups": {}, "custom_object_type_rendered_names": set(), + # field_name → (latitude_field_name, longitude_field_name) + "custom_object_type_coordinates_fields": {}, } for field in self.custom_object_type.fields.prefetch_related('related_object_types').all(): field_type = field_types.FIELD_TYPE_CLASS[field.type]() + # Coordinates: two optional latitude/longitude inputs in bulk edit + if field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + sub_names = [] + for sub_name, sub_field in field_type.get_form_fields(field).items(): + sub_field.required = False + sub_field.widget.is_required = False + sub_field.initial = None + attrs[sub_name] = sub_field + sub_names.append(sub_name) + # (latitude_name, longitude_name) for cross-field validation below. + attrs["custom_object_type_coordinates_fields"][field.name] = tuple(sub_names) + continue + # Polymorphic single-object: scope-style type-selector + object-picker pair if field.is_polymorphic and field.type == CustomFieldTypeChoices.TYPE_OBJECT: ct_sub = f"{field.name}__ct" @@ -1179,6 +1285,23 @@ def bulk_poly_init(self, *args, **kwargs): attrs["__init__"] = bulk_poly_init + coordinates_fields_ref = attrs["custom_object_type_coordinates_fields"] + + def bulk_clean(self): + cleaned_data = NetBoxModelBulkEditForm.clean(self) + # Coordinates: latitude and longitude must both be set or both be empty. + for field_name, (lat_name, lon_name) in coordinates_fields_ref.items(): + latitude = cleaned_data.get(lat_name) + longitude = cleaned_data.get(lon_name) + if (latitude is None) != (longitude is None): + self.add_error( + lat_name if latitude is None else lon_name, + _("Latitude and longitude must both be set or both be empty."), + ) + return cleaned_data + + attrs["clean"] = bulk_clean + form = type( f"{queryset.model._meta.object_name}BulkEditForm", (NetBoxModelBulkEditForm,), @@ -1202,8 +1325,8 @@ def post_save_operations(self, form, obj): # Apply polymorphic M2M sub-fields (union of all selected types). # set() replaces existing values, matching NetBox's standard bulk-edit - # behavior for direct M2M fields (see BulkEditView lines 718-723). - # Fields left blank are skipped so existing data is preserved. + # behavior for direct M2M fields. Fields left blank are skipped so + # existing data is preserved. for field_name, sub_names in form._poly_m2m_field_map.items(): combined = [] has_any = False @@ -1227,15 +1350,9 @@ def get(self, request, **kwargs): return render(request, 'netbox_custom_objects/htmx/bulk_edit_fields.html', { 'form': form, 'return_url': self.get_return_url(request), - 'branch_warning': is_in_branch(), }) return redirect(self.get_return_url(request)) - def get_extra_context(self, request): - return { - 'branch_warning': is_in_branch(), - } - @register_model_view(CustomObject, "bulk_delete", path="delete", detail=False) class CustomObjectBulkDeleteView(CustomObjectTableMixin, generic.BulkDeleteView): @@ -1261,9 +1378,6 @@ def get_queryset(self, request): model = self.custom_object_type.get_model_with_serializer() return model.objects.all() - def get_extra_context(self, request): - return {'branch_warning': is_in_branch()} - @register_model_view(CustomObject, "bulk_import", path="import", detail=False) class CustomObjectBulkImportView(generic.BulkImportView): @@ -1327,11 +1441,6 @@ def get_model_form(self, queryset): return form - def get_extra_context(self, request): - return { - 'branch_warning': is_in_branch(), - } - class CustomObjectJournalView(ConditionalLoginRequiredMixin, View): """ @@ -1457,3 +1566,111 @@ def get(self, request, custom_object_type, **kwargs): "tab": "changelog", }, ) + + +class CustomObjectContactsView(ConditionalLoginRequiredMixin, View): + """ + Custom contacts view for CustomObject instances. + Shows all contacts assigned to a custom object. + """ + + tab = ViewTab( + label=_("Contacts"), + badge=lambda obj: obj.get_contacts().count(), + permission="tenancy.view_contactassignment", + weight=4500, + ) + + def get(self, request, custom_object_type, **kwargs): + from tenancy.tables import ContactAssignmentTable + + object_type = get_object_or_404(CustomObjectType, slug=custom_object_type) + model = object_type.get_model_with_serializer() + + lookup_kwargs = {k: v for k, v in kwargs.items() if k != "custom_object_type"} + obj = get_object_or_404(model.objects.all(), **lookup_kwargs) + + contacts = ( + obj.get_contacts() + .restrict(request.user, "view") + .order_by("priority", "contact", "role") + ) + table = ContactAssignmentTable(data=contacts, orderable=False) + table.configure(request) + + return render( + request, + "netbox_custom_objects/object_contacts.html", + { + "object": obj, + "table": table, + "base_template": "netbox_custom_objects/customobject.html", + "tab": "contacts", + }, + ) + + +class CustomObjectConfigContextView(ConditionalLoginRequiredMixin, View): + """ + Config context view for CustomObject instances. + + Only available when the instance's CustomObjectType has + ``config_context_enabled=True`` (i.e. the generated model mixes in + ConfigContextModel). Source contexts are aggregated from NetBox objects the + custom object references via convention-named fields (site/tenant/role/...); + see CustomObjectConfigContextMixin. local_context_data overrides them. + """ + + base_template = None + tab = ViewTab( + label=_("Config Context"), + visible=lambda obj: obj.custom_object_type.config_context_enabled, + weight=2000, + ) + + def get(self, request, custom_object_type, **kwargs): + object_type = get_object_or_404(CustomObjectType, slug=custom_object_type) + if not object_type.config_context_enabled: + raise Http404(_("Config context support is not enabled for this type.")) + model = object_type.get_model_with_serializer() + + lookup_kwargs = {k: v for k, v in kwargs.items() if k != "custom_object_type"} + # Gate on object-level view permission so the config-context tab can't + # leak local_context_data past NetBox's RBAC (the detail view restricts too). + obj = get_object_or_404(model.objects.restrict(request.user, "view"), **lookup_kwargs) + + # Determine the user's preferred output format (json/yaml), persisting + # an explicit choice the same way NetBox's ObjectConfigContextView does. + if request.GET.get("format") in ("json", "yaml"): + format = request.GET.get("format") + if request.user.is_authenticated: + request.user.config.set("data_format", format, commit=True) + elif request.user.is_authenticated: + format = request.user.config.get("data_format", "json") + else: + format = "json" + + if self.base_template is None: + self.base_template = "netbox_custom_objects/customobject.html" + + # Build the proxy once and reuse it for both the rendered context and the + # (RBAC-restricted) source-context list, instead of letting get_config_context() + # rebuild it internally. + proxy = obj._config_context_source() + if proxy is not None: + source_contexts = ConfigContext.objects.restrict(request.user, "view").get_for_object(proxy) + else: + source_contexts = ConfigContext.objects.none() + + return render( + request, + "netbox_custom_objects/object_configcontext.html", + { + "object": obj, + "rendered_context": obj._render_config_context(proxy), + "source_contexts": source_contexts, + "format": format, + "base_template": self.base_template, + "tab": "configcontext", + }, + ) diff --git a/pyproject.toml b/pyproject.toml index 660736db..33d7b948 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "netboxlabs-netbox-custom-objects" -version = "0.5.3" +version = "0.6.0" description = "A plugin to manage custom objects in NetBox" readme = "README.md" requires-python = ">=3.10" @@ -32,6 +32,11 @@ dependencies = [ [project.optional-dependencies] dev = ["check-manifest", "mkdocs", "mkdocs-material", "ruff"] test = ["coverage", "pytest", "pytest-cov"] +# Install with `pip install "netboxlabs-netbox-custom-objects[branching]"` when +# pairing this plugin with netbox-branching. Note: this extra also implies a +# stricter NetBox version requirement (>= 4.6.2), which is enforced at startup +# by netbox_custom_objects.checks rather than declared here. +branching = ["netbox-branching>=1.0.4"] [project.urls] "Homepage" = "https://netboxlabs.com/"