test(e2e): run conformance in gateway lanes

Signed-off-by: Evan Lezar <elezar@nvidia.com>

Author: elezar

.github/actions/setup-e2e-cli/action.yml

@@ -1,5 +1,11 @@
 name: Setup E2E CLI
-description: Download an architecture-matched prebuilt OpenShell CLI for E2E tests
+description: Download architecture-matched prebuilt OpenShell host CLIs for E2E tests
+
+inputs:
+  conformance-artifact-prefix:
+    description: Optional conformance CLI artifact name prefix; linux-<arch> is appended automatically
+    required: false
+    default: ""
 
 runs:
   using: composite
@@ -17,3 +23,24 @@ runs:
         chmod +x "$cli"
         "$cli" --version
         echo "OPENSHELL_BIN=$cli" >> "$GITHUB_ENV"
+
+    - name: Download prebuilt conformance CLI
+      if: inputs.conformance-artifact-prefix != ''
+      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      with:
+        name: ${{ format('{0}-{1}', inputs.conformance-artifact-prefix, runner.arch == 'X64' && 'x86_64-unknown-linux-musl' || 'aarch64-unknown-linux-musl') }}
+        path: .e2e/prebuilt-conformance
+
+    - name: Configure prebuilt conformance CLI
+      if: inputs.conformance-artifact-prefix != ''
+      shell: bash
+      run: |
+        set -euo pipefail
+        conformance="$GITHUB_WORKSPACE/.e2e/prebuilt-conformance/openshell-conformance"
+        if [[ ! -f "$conformance" ]]; then
+          echo "downloaded artifact is missing $conformance" >&2
+          exit 1
+        fi
+        chmod +x "$conformance"
+        "$conformance" list --output json >/dev/null
+        echo "OPENSHELL_CONFORMANCE_BIN=$conformance" >> "$GITHUB_ENV"

.github/workflows/branch-e2e.yml

@@ -105,6 +105,31 @@ jobs:
       cargo-version: ${{ needs.version.outputs.cargo }}
     secrets: inherit
 
+  build-conformance:
+    needs: [pr_metadata, version]
+    if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_any_e2e == 'true'
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        include:
+          - triple: x86_64-unknown-linux-musl
+            runner: linux-amd64-cpu8
+            dev_shell: .#devShells.x86_64-linux.musl
+          - triple: aarch64-unknown-linux-musl
+            runner: linux-arm64-cpu8
+            dev_shell: .#devShells.aarch64-linux.musl
+    uses: ./.github/workflows/build-binaries.yml
+    with:
+      package: openshell-conformance-cli
+      binary: openshell-conformance
+      artifact-name: openshell-conformance-${{ matrix.triple }}
+      triple: ${{ matrix.triple }}
+      runner: ${{ matrix.runner }}
+      dev-shell: ${{ matrix.dev_shell }}
+      cargo-version: ${{ needs.version.outputs.cargo }}
+    secrets: inherit
+
   build-sandbox:
     needs: version
     permissions:
@@ -241,7 +266,7 @@ jobs:
     secrets: inherit
 
   docker-e2e:
-    needs: [pr_metadata, build-cli, build-gateway, build-supervisor-image]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway, build-supervisor-image]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -251,9 +276,10 @@ jobs:
     with:
       image-tag: ${{ github.sha }}
       runner: linux-arm64-cpu8
+      conformance-artifact-prefix: openshell-conformance
 
   podman-e2e:
-    needs: [pr_metadata, build-cli, build-gateway, build-supervisor-image]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway, build-supervisor-image]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -262,18 +288,21 @@ jobs:
     uses: ./.github/workflows/e2e-podman-test.yml
     with:
       image-tag: ${{ github.sha }}
+      conformance-artifact-prefix: openshell-conformance
 
   vm-e2e:
-    needs: [pr_metadata, build-cli, build-gateway, build-vm-driver]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway, build-vm-driver]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-vm-test.yml
+    with:
+      conformance-artifact-prefix: openshell-conformance
 
   docker-external-driver-e2e:
-    needs: [pr_metadata, build-cli, build-gateway-plain, build-driver-docker, build-supervisor-image]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway-plain, build-driver-docker, build-supervisor-image]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -285,11 +314,12 @@ jobs:
       runner: linux-arm64-cpu8
       gateway-artifact: openshell-gateway-plain-aarch64-unknown-linux-gnu
       external-driver-binary: openshell-driver-docker
+      conformance-artifact-prefix: openshell-conformance
       suite-matrix: >-
         [{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}]
 
   podman-external-driver-e2e:
-    needs: [pr_metadata, build-cli, build-gateway-plain, build-driver-podman, build-supervisor-image]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway-plain, build-driver-podman, build-supervisor-image]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -300,11 +330,12 @@ jobs:
       image-tag: ${{ github.sha }}
       gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu
       external-driver-binary: openshell-driver-podman
+      conformance-artifact-prefix: openshell-conformance
       suite-matrix: >-
         [{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}]
 
   vm-external-driver-e2e:
-    needs: [pr_metadata, build-cli, build-gateway-plain, build-vm-driver]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway-plain, build-vm-driver]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -315,9 +346,10 @@ jobs:
       gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu
       suite-name: external-driver
       e2e-task: e2e:vm:external-driver
+      conformance-artifact-prefix: openshell-conformance
 
   gpu-e2e:
-    needs: [pr_metadata, build-cli, build-gateway, build-supervisor-image]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway, build-supervisor-image]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_gpu_e2e == 'true'
     permissions:
       actions: read
@@ -326,9 +358,10 @@ jobs:
     uses: ./.github/workflows/e2e-gpu-test.yaml
     with:
       image-tag: ${{ github.sha }}
+      conformance-artifact-prefix: openshell-conformance
 
   kubernetes-e2e:
-    needs: [pr_metadata, build-cli, build-gateway-image, build-supervisor-image]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway-image, build-supervisor-image]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     strategy:
       fail-fast: false
@@ -356,9 +389,10 @@ jobs:
       job-name: Kubernetes E2E (Rust smoke, ${{ matrix.topology }}, Agent Sandbox ${{ matrix.agent_sandbox_api }})
       agent-sandbox-version: ${{ matrix.agent_sandbox_version }}
       extra-helm-values: ${{ matrix.extra_helm_values }}
+      conformance-artifact-prefix: openshell-conformance
 
   kubernetes-workspace-managed-e2e:
-    needs: [pr_metadata, build-cli, build-gateway-image, build-supervisor-image]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway-image, build-supervisor-image]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -369,9 +403,10 @@ jobs:
       image-tag: ${{ github.sha }}
       job-name: Kubernetes E2E (workspace managed mode)
       e2e-task: e2e:kubernetes:workspace-managed
+      conformance-artifact-prefix: openshell-conformance
 
   kubernetes-external-driver-e2e:
-    needs: [pr_metadata, build-cli, build-gateway-plain, build-driver-kubernetes, build-supervisor-image]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway-plain, build-driver-kubernetes, build-supervisor-image]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -385,9 +420,10 @@ jobs:
       gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu
       external-driver-binary: openshell-driver-kubernetes
       cluster-images: supervisor
+      conformance-artifact-prefix: openshell-conformance
 
   kubernetes-workspace-operator-e2e:
-    needs: [pr_metadata, build-cli, build-gateway-image, build-supervisor-image]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway-image, build-supervisor-image]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -398,9 +434,10 @@ jobs:
       image-tag: ${{ github.sha }}
       job-name: Kubernetes E2E (workspace operator mode)
       e2e-task: e2e:kubernetes:workspace-operator
+      conformance-artifact-prefix: openshell-conformance
 
   kubernetes-ha-e2e:
-    needs: [pr_metadata, build-cli, build-gateway-image, build-supervisor-image]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway-image, build-supervisor-image]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true'
     permissions:
       actions: read
@@ -412,9 +449,10 @@ jobs:
       job-name: Kubernetes HA E2E (Rust smoke)
       extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml
       external-postgres-secret: openshell-ha-pg
+      conformance-artifact-prefix: openshell-conformance
 
   kubernetes-credential-drivers-e2e:
-    needs: [pr_metadata, build-cli, build-gateway-image, build-supervisor-image]
+    needs: [pr_metadata, build-cli, build-conformance, build-gateway-image, build-supervisor-image]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true'
     permissions:
       actions: read
@@ -425,6 +463,7 @@ jobs:
       image-tag: ${{ github.sha }}
       job-name: Kubernetes Credential Drivers E2E
       e2e-task: e2e:kubernetes:credential-drivers
+      conformance-artifact-prefix: openshell-conformance
 
   core-e2e-result:
     name: Core E2E result

.github/workflows/e2e-docker-test.yml

@@ -25,6 +25,10 @@ on:
         required: false
         type: string
         default: ""
+      conformance-artifact-prefix:
+        required: false
+        type: string
+        default: ""
       suite-matrix:
         required: false
         type: string
@@ -66,6 +70,8 @@ jobs:
           persist-credentials: false
 
       - uses: ./.github/actions/setup-e2e-cli
+        with:
+          conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
 
       - uses: ./.github/actions/setup-e2e-gateway
         with:

.github/workflows/e2e-gpu-test.yaml

@@ -7,6 +7,11 @@ on:
         description: "Image tag to test (typically the commit SHA)"
         required: true
         type: string
+      conformance-artifact-prefix:
+        description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is <triple>)"
+        required: false
+        type: string
+        default: ""
 
 permissions:
   actions: read
@@ -61,6 +66,8 @@ jobs:
 
       - name: Use prebuilt OpenShell CLI
         uses: ./.github/actions/setup-e2e-cli
+        with:
+          conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
 
       - name: Use prebuilt OpenShell gateway
         uses: ./.github/actions/setup-e2e-gateway

.github/workflows/e2e-kubernetes-test.yml

@@ -52,6 +52,11 @@ on:
         required: false
         type: string
         default: ""
+      conformance-artifact-prefix:
+        description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is <triple>)"
+        required: false
+        type: string
+        default: ""
       external-driver-binary:
         description: "Optional standalone compute driver binary used to compose a local test image"
         required: false
@@ -92,6 +97,8 @@ jobs:
 
       - name: Use prebuilt OpenShell CLI
         uses: ./.github/actions/setup-e2e-cli
+        with:
+          conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
 
       - name: Use prebuilt OpenShell gateway
         if: inputs.gateway-artifact != ''

.github/workflows/e2e-podman-test.yml

@@ -21,6 +21,10 @@ on:
         required: false
         type: string
         default: ""
+      conformance-artifact-prefix:
+        required: false
+        type: string
+        default: ""
       suite-matrix:
         required: false
         type: string
@@ -53,6 +57,8 @@ jobs:
           persist-credentials: false
 
       - uses: ./.github/actions/setup-e2e-cli
+        with:
+          conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
 
       - uses: ./.github/actions/setup-e2e-gateway
         with:

.github/workflows/e2e-vm-test.yml

@@ -22,6 +22,10 @@ on:
         required: false
         type: string
         default: e2e:vm
+      conformance-artifact-prefix:
+        required: false
+        type: string
+        default: ""
 
 permissions:
   actions: read
@@ -43,6 +47,8 @@ jobs:
           persist-credentials: false
 
       - uses: ./.github/actions/setup-e2e-cli
+        with:
+          conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
 
       - uses: ./.github/actions/setup-e2e-gateway
         with:

TESTING.md

@@ -171,7 +171,7 @@ key.
 Run the Docker-backed Rust CLI e2e suite:
 
 ```shell
-mise run e2e:rust
+mise run e2e:docker
 ```
 
 Run the minimal portable CLI conformance profile against the gateway selected
@@ -210,7 +210,9 @@ openshell sandbox list --output json
 openshell sandbox delete <sandbox-name>
 ```
 
-Run the Podman-backed Rust CLI e2e suite:
+Gateway-backed Rust E2E tasks build the standalone conformance CLI, run its
+registered scenarios against the configured gateway, then run any lane-specific
+Rust tests that still apply. Run the Podman-backed Rust CLI e2e suite:
 
 ```shell
 mise run e2e:podman

docs/index.yml

@@ -27,9 +27,6 @@ navigation:
   title: "Kubernetes"
 - folder: reference
   title: "Reference"
-  contents:
-  - page: "OpenShell conformance CLI"
-    path: reference/conformance-cli.mdx
 - folder: security
   title: "Security"
 - folder: resources