test: Add e2e tests for multiple installs in the same cluster

Signed-off-by: Jonathan Stacks <[email protected]>

Author: jonstacks

.github/actions/build-and-test/action.yaml

@@ -17,7 +17,7 @@ inputs:
   kind-version:
     description: "KIND version to use"
     required: false
-    default: "v0.26.0"
+    default: "v0.30.0"
 
 runs:
   using: "composite"

.github/workflows/ci.yaml

@@ -182,3 +182,50 @@ jobs:
         run-e2e: true
         ngrok-api-key: ${{ secrets.NGROK_CI_API_KEY }}
         ngrok-authtoken: ${{ secrets.NGROK_CI_AUTHTOKEN }}
+
+  e2e-multi-namespace:
+    runs-on: ubuntu-latest
+    needs:
+    - changes
+    - kubebuilder-diff
+    - lint
+    - e2e
+    concurrency:
+      group: e2e
+      cancel-in-progress: false
+    timeout-minutes: 30
+    if: |
+      (github.repository == 'ngrok/ngrok-operator') &&
+      (
+        (github.event_name == 'push' && github.ref_name == 'main') ||
+        (github.event_name == 'merge_group')
+      ) &&
+      (
+        (needs.changes.outputs.go == 'true') ||
+        (needs.changes.outputs.charts == 'true') ||
+        (needs.changes.outputs.chartyaml == 'true') ||
+        (needs.changes.outputs.tests == 'true') ||
+        (needs.changes.outputs.make == 'true')
+      )
+    steps:
+    - uses: actions/checkout@v6
+    - uses: nixbuild/nix-quick-install-action@v34
+    - uses: DeterminateSystems/magic-nix-cache-action@b8276522d77f21bf19d3574e5bc99186a6c7aa6c
+    - name: Create Kind Cluster
+      run: nix develop --command make kind-create
+    - name: Deploy
+      run: nix develop --command make deploy_multi_namespace
+      env:
+        NGROK_API_KEY: ${{ secrets.NGROK_CI_API_KEY }}
+        NGROK_AUTHTOKEN: ${{ secrets.NGROK_CI_AUTHTOKEN }}
+    - name: Test
+      run: nix develop --command make e2e-tests-multi-ns
+    - name: Cleanup
+      if: always()
+      run: |
+        echo "Deleting kubernetesoperators..."
+        kubectl -n namespace-a delete kubernetesoperator ngrok-operator-a || true
+        kubectl -n namespace-b delete kubernetesoperator ngrok-operator-b || true
+        echo "Uninstalling the operators..."
+        helm uninstall ngrok-operator-a -n namespace-a || true
+        helm uninstall ngrok-operator-b -n namespace-b || true

tests/chainsaw-multi-ns/README.md

@@ -0,0 +1,40 @@
+# Multi-Namespace E2E Tests
+
+This directory contains Chainsaw tests for validating multi-namespace operator deployments.
+
+## Prerequisites
+
+These tests assume the cluster has been set up using `make deploy_multi_namespace`, which deploys:
+
+1. **CRDs** installed once in `kube-system` via `ngrok-operator-crds` helm chart
+2. **Operator A** (`ngrok-operator-a`) in `namespace-a`:
+   - Watches only `namespace-a`
+   - Uses ingress class `ngrok-a`
+   - Controller name: `k8s.ngrok.com/ingress-controller-a`
+3. **Operator B** (`ngrok-operator-b`) in `namespace-b`:
+   - Watches only `namespace-b`
+   - Uses ingress class `ngrok-b`
+   - Controller name: `k8s.ngrok.com/ingress-controller-b`
+
+## Running Locally
+
+```bash
+# Set up the multi-namespace deployment
+make deploy_multi_namespace
+
+# Run the tests
+make e2e-tests-multi-ns
+```
+
+## Test Structure
+
+- **sanity-checks/**: Validates both operators are running and ingress classes exist
+- **operator-registration/**: Tests that the operators correctly register with ngrok and receive an ID.
+
+## Writing New Tests
+
+When adding tests to this directory:
+
+- Use explicit `metadata.namespace` values (`namespace-a` or `namespace-b`)
+- Use the correct `ingressClassName` (`ngrok-a` or `ngrok-b`)
+- Focus on multi-operator isolation behaviors that can't be tested with a single operator

tests/chainsaw-multi-ns/operator-registration/chainsaw-test.yaml

@@ -0,0 +1,42 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: multi-ns-operator-registration
+spec:
+  steps:
+  - name: assert KubernetesOperator/ngrok-operator-a is registered in namespace-a
+    try:
+    - assert:
+        resource:
+          apiVersion: ngrok.k8s.ngrok.com/v1alpha1
+          kind: KubernetesOperator
+          metadata:
+            name: ngrok-operator-a
+            namespace: namespace-a
+          spec:
+            deployment:
+              name: ngrok-operator-a
+              namespace: namespace-a
+          status:
+            registrationStatus: registered
+            (id != null && starts_with(id, 'k8sop_')): true
+            (uri != null): true
+
+  - name: assert KubernetesOperator/ngrok-operator-b is registered in namespace-b
+    try:
+    - assert:
+        resource:
+          apiVersion: ngrok.k8s.ngrok.com/v1alpha1
+          kind: KubernetesOperator
+          metadata:
+            name: ngrok-operator-b
+            namespace: namespace-b
+          spec:
+            deployment:
+              name: ngrok-operator-b
+              namespace: namespace-b
+          status:
+            registrationStatus: registered
+            (id != null && starts_with(id, 'k8sop_')): true
+            (uri != null): true

tests/chainsaw-multi-ns/sanity-checks/agent-pod-a.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app.kubernetes.io/component: agent
+  namespace: namespace-a
+status:
+  (conditions[?type == 'Ready']):
+  - status: "True"
+  ~.containerStatuses:
+    ready: true
+    restartCount: 0
+  phase: Running

tests/chainsaw-multi-ns/sanity-checks/agent-pod-b.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app.kubernetes.io/component: agent
+  namespace: namespace-b
+status:
+  (conditions[?type == 'Ready']):
+  - status: "True"
+  ~.containerStatuses:
+    ready: true
+    restartCount: 0
+  phase: Running

tests/chainsaw-multi-ns/sanity-checks/chainsaw-test.yaml

@@ -0,0 +1,31 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: multi-ns-sanity-checks
+spec:
+  steps:
+  - name: check ingress class ngrok-a exists
+    try:
+    - assert:
+        file: ./ingress-class-a.yaml
+  - name: check ingress class ngrok-b exists
+    try:
+    - assert:
+        file: ./ingress-class-b.yaml
+  - name: check operator-a pods are running
+    try:
+    - assert:
+        file: ./operator-pod-a.yaml
+  - name: check operator-b pods are running
+    try:
+    - assert:
+        file: ./operator-pod-b.yaml
+  - name: check agent-a pods are running
+    try:
+    - assert:
+        file: ./agent-pod-a.yaml
+  - name: check agent-b pods are running
+    try:
+    - assert:
+        file: ./agent-pod-b.yaml

tests/chainsaw-multi-ns/sanity-checks/ingress-class-a.yaml

@@ -0,0 +1,6 @@
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  name: ngrok-a
+spec:
+  controller: k8s.ngrok.com/ingress-controller-a

tests/chainsaw-multi-ns/sanity-checks/ingress-class-b.yaml

@@ -0,0 +1,6 @@
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  name: ngrok-b
+spec:
+  controller: k8s.ngrok.com/ingress-controller-b

tests/chainsaw-multi-ns/sanity-checks/operator-pod-a.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+  namespace: namespace-a
+status:
+  (conditions[?type == 'Ready']):
+  - status: "True"
+  ~.containerStatuses:
+    ready: true
+    restartCount: 0
+  phase: Running