feat: enable site clusters to run Nautobot Celery workers locally

Nautobot currently runs entirely on the global cluster, including its
Celery workers. Sites that generate heavy background task load have no
way to offload that processing closer to where the work originates,
and a single global worker pool becomes a bottleneck as sites scale.

This adds a site-scoped ArgoCD Application that deploys only the
Celery worker portion of the Nautobot helm chart. The web server,
Redis, and PostgreSQL are all disabled because they remain on the
global cluster — site workers connect back to those shared services.

This lets operators scale worker capacity per-site independently,
run queue-specific workers closer to the hardware they manage, and
reduce cross-cluster task latency for site-driven automation.

Author: syedhaseebahmed

charts/argocd-understack/templates/application-nautobot-worker.yaml

@@ -0,0 +1,49 @@
+{{- if eq (include "understack.isEnabled" (list $.Values.site "nautobot_worker")) "true" }}
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ printf "%s-%s" $.Release.Name "nautobot-worker" }}
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/compare-options: ServerSideDiff=true,IncludeMutationWebhook=true
+spec:
+  destination:
+    namespace: nautobot
+    server: {{ $.Values.cluster_server }}
+  project: understack
+  sources:
+  - chart: nautobot
+    helm:
+      fileParameters:
+      - name: nautobot.config
+        path: $understack/components/nautobot/nautobot_config.py
+      ignoreMissingValueFiles: true
+      releaseName: nautobot-worker
+      valueFiles:
+      - $understack/components/nautobot-worker/values.yaml
+      - $deploy/{{ include "understack.deploy_path" $ }}/nautobot-worker/values.yaml
+    repoURL: https://nautobot.github.io/helm-charts/
+    targetRevision: 2.5.6
+  - path: components/nautobot-worker
+    ref: understack
+    repoURL: {{ include "understack.understack_url" $ }}
+    targetRevision: {{ include "understack.understack_ref" $ }}
+  - path: {{ include "understack.deploy_path" $ }}/nautobot-worker
+    ref: deploy
+    repoURL: {{ include "understack.deploy_url" $ }}
+    targetRevision: {{ include "understack.deploy_ref" $ }}
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    managedNamespaceMetadata:
+      annotations:
+        argocd.argoproj.io/sync-options: Delete=false
+    syncOptions:
+    - CreateNamespace=true
+    - ServerSideApply=true
+    - RespectIgnoreDifferences=true
+    - ApplyOutOfSyncOnly=true
+{{- end }}

charts/argocd-understack/values.yaml

@@ -556,6 +556,12 @@ site:
     # @default -- false
     enabled: false
 
+  # -- Nautobot Celery workers (site-level, connects to global Nautobot)
+  nautobot_worker:
+    # -- Enable/disable deploying Nautobot workers at the site level
+    # @default -- false
+    enabled: false
+
   # -- SNMP exporter for network device monitoring
   snmp_exporter:
     # -- Enable/disable deploying SNMP exporter

components/envoy-configs/templates/gw-external.yaml.tpl

@@ -52,6 +52,39 @@ spec:
           from: {{ .from | default "All" }}
           {{- end }}
     {{- end }}
+    {{- range .Values.routes.tcp }}
+    - name: {{ .listenerName }}
+      port: {{ .gatewayPort }}
+      protocol: TCP
+      allowedRoutes:
+        namespaces:
+          {{- if .selector }}
+          from: Selector
+          selector:
+            {{- .selector | toYaml | nindent 12 }}
+          {{- else }}
+          from: {{ .from | default "All" }}
+          {{- end }}
+    {{- end }}
+    {{- range .Values.routes.tlsTerminatedTcp }}
+    - name: {{ .listenerName }}
+      port: {{ .gatewayPort }}
+      protocol: TLS
+      hostname: {{ .fqdn | quote }}
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - name: {{ .listenerName }}-tls
+      allowedRoutes:
+        namespaces:
+          {{- if .selector }}
+          from: Selector
+          selector:
+            {{- .selector | toYaml | nindent 12 }}
+          {{- else }}
+          from: {{ .from | default "All" }}
+          {{- end }}
+    {{- end }}
   {{- if .Values.gateways.external.serviceAnnotations }}
   infrastructure:
     parametersRef:

components/envoy-configs/templates/tcproute.yaml.tpl

@@ -0,0 +1,26 @@
+{{- range .Values.routes.tcp }}
+---
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: TCPRoute
+metadata:
+  {{- if .name }}
+  name: {{ .name }}
+  {{- else }}
+  name: {{ .service.name }}
+  {{- end }}
+  namespace: {{ .namespace | default "envoy-gateway" }}
+  labels:
+    {{- include "envoy-configs.labels" $ | nindent 4 }}
+spec:
+  parentRefs:
+    - name: {{ $.Values.gateways.external.name }}
+      namespace: {{ $.Values.gateways.external.namespace }}
+      sectionName: {{ .listenerName }}
+  rules:
+    - backendRefs:
+        - name: {{ .service.name }}
+          {{- with .namespace }}
+          namespace: {{ . }}
+          {{- end }}
+          port: {{ .service.port }}
+{{- end }}

components/envoy-configs/templates/tls-terminated-tcproute.yaml.tpl

@@ -0,0 +1,26 @@
+{{- range .Values.routes.tlsTerminatedTcp }}
+---
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: TCPRoute
+metadata:
+  {{- if .name }}
+  name: {{ .name }}
+  {{- else }}
+  name: {{ .service.name }}
+  {{- end }}
+  namespace: {{ .namespace | default "envoy-gateway" }}
+  labels:
+    {{- include "envoy-configs.labels" $ | nindent 4 }}
+spec:
+  parentRefs:
+    - name: {{ $.Values.gateways.external.name }}
+      namespace: {{ $.Values.gateways.external.namespace }}
+      sectionName: {{ .listenerName }}
+  rules:
+    - backendRefs:
+        - name: {{ .service.name }}
+          {{- with .namespace }}
+          namespace: {{ . }}
+          {{- end }}
+          port: {{ .service.port }}
+{{- end }}

components/envoy-configs/values.schema.json

@@ -224,6 +224,151 @@
             ],
             "additionalProperties": false
           }
+        },
+        "tcp": {
+          "type": "array",
+          "description": "TCP routes for non-HTTP services (e.g., PostgreSQL, Redis)",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string",
+                "description": "Name identifier for the TCPRoute resource"
+              },
+              "listenerName": {
+                "type": "string",
+                "description": "Name of the TCP listener on the gateway (must match)"
+              },
+              "gatewayPort": {
+                "type": "integer",
+                "minimum": 1,
+                "maximum": 65535,
+                "description": "Port exposed on the gateway for this TCP route"
+              },
+              "namespace": {
+                "type": "string",
+                "description": "Namespace of the backend service"
+              },
+              "service": {
+                "type": "object",
+                "description": "Kubernetes service backend configuration",
+                "properties": {
+                  "name": {
+                    "type": "string",
+                    "description": "Name of the Kubernetes service"
+                  },
+                  "port": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "maximum": 65535,
+                    "description": "Port of the backend service"
+                  }
+                },
+                "required": [
+                  "name",
+                  "port"
+                ],
+                "additionalProperties": false
+              },
+              "selector": {
+                "type": "object",
+                "description": "Kubernetes-style label selector (key-value pairs)",
+                "additionalProperties": {
+                  "type": "string"
+                }
+              },
+              "from": {
+                "type": "string",
+                "enum": [
+                  "Same",
+                  "All",
+                  "Selector"
+                ],
+                "description": "Specifies where traffic can originate from"
+              }
+            },
+            "required": [
+              "listenerName",
+              "gatewayPort",
+              "service"
+            ],
+            "additionalProperties": false
+          }
+        },
+        "tlsTerminatedTcp": {
+          "type": "array",
+          "description": "TLS-terminated TCP routes. Envoy terminates TLS from the client and forwards plaintext to the backend. Used for encrypting cross-cluster TCP traffic (e.g., Redis) without requiring server-side TLS.",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string",
+                "description": "Name identifier for the TCPRoute resource"
+              },
+              "listenerName": {
+                "type": "string",
+                "description": "Name of the TLS listener on the gateway (must match)"
+              },
+              "fqdn": {
+                "type": "string",
+                "description": "FQDN for the TLS listener (used for SNI and cert generation)"
+              },
+              "gatewayPort": {
+                "type": "integer",
+                "minimum": 1,
+                "maximum": 65535,
+                "description": "Port exposed on the gateway for this route"
+              },
+              "namespace": {
+                "type": "string",
+                "description": "Namespace of the backend service"
+              },
+              "service": {
+                "type": "object",
+                "description": "Kubernetes service backend configuration",
+                "properties": {
+                  "name": {
+                    "type": "string",
+                    "description": "Name of the Kubernetes service"
+                  },
+                  "port": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "maximum": 65535,
+                    "description": "Port of the backend service"
+                  }
+                },
+                "required": [
+                  "name",
+                  "port"
+                ],
+                "additionalProperties": false
+              },
+              "selector": {
+                "type": "object",
+                "description": "Kubernetes-style label selector (key-value pairs)",
+                "additionalProperties": {
+                  "type": "string"
+                }
+              },
+              "from": {
+                "type": "string",
+                "enum": [
+                  "Same",
+                  "All",
+                  "Selector"
+                ],
+                "description": "Specifies where traffic can originate from"
+              }
+            },
+            "required": [
+              "listenerName",
+              "fqdn",
+              "gatewayPort",
+              "service"
+            ],
+            "additionalProperties": false
+          }
         }
       }
     }

components/envoy-configs/values.yaml

@@ -2,3 +2,5 @@ gateways: {}
 routes:
   http: []
   tls: []
+  tcp: []
+  tlsTerminatedTcp: []

components/nautobot-worker/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources: []

components/nautobot-worker/values.yaml

@@ -0,0 +1,78 @@
+# Nautobot Worker (site-level)
+#
+# Deploys only Celery workers that connect back to the global Nautobot
+# database and Redis. The web server is disabled because it lives on
+# the global cluster. Redis and PostgreSQL are disabled because the
+# workers reach the global instances over the network.
+#
+# The deploy repo for each site MUST provide:
+#   - ExternalSecrets for nautobot-django, nautobot-redis, nautobot-db,
+#     nautobot-custom-env, and dockerconfigjson-github-com
+#   - values.yaml overrides for nautobot.db.host and nautobot.redis.host
+#     pointing to the global cluster endpoints
+---
+
+# Disable the Nautobot web server — workers only
+nautobot:
+  enabled: false
+  replicaCount: 0
+
+  db:
+    engine: "django.db.backends.postgresql"
+    # Override in deploy repo values to point at the global CNPG service
+    host: ""
+    port: 5432
+    name: "app"
+    user: "app"
+    existingSecret: "nautobot-db"
+    existingSecretPasswordKey: "password"
+
+  django:
+    existingSecret: nautobot-django
+
+  superUser:
+    enabled: false
+
+  redis:
+    # Override in deploy repo values to point at the global Redis service.
+    # ssl: true because Envoy Gateway terminates TLS on the Redis port —
+    # the worker connects via rediss:// and Envoy forwards plaintext
+    # to Redis locally on the global cluster.
+    host: ""
+    port: 6379
+    ssl: true
+    username: ""
+
+celery:
+  enabled: true
+  concurrency: 2
+  replicaCount: 1
+  extraEnvVarsSecret:
+    - nautobot-django
+  livenessProbe:
+    initialDelaySeconds: 60
+    periodSeconds: 120
+    timeoutSeconds: 60
+  readinessProbe:
+    initialDelaySeconds: 60
+    periodSeconds: 120
+    timeoutSeconds: 60
+
+# Disable celery beat — scheduling runs on the global cluster only
+workers:
+  beat:
+    enabled: false
+
+# Do not deploy local Redis — use the global instance
+redis:
+  enabled: false
+
+# Do not deploy local PostgreSQL — use the global CNPG instance
+postgresql:
+  enabled: false
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: false