Skip to content

Commit 3535a1a

Browse files
elezarelezar
committed
test(e2e): run conformance in gateway lanes
Signed-off-by: Evan Lezar <elezar@nvidia.com>
1 parent a764f85 commit 3535a1a

14 files changed

Lines changed: 343 additions & 93 deletions

File tree

‎.github/actions/setup-e2e-cli/action.yml‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
name: Setup E2E CLI
2-
description: Download an architecture-matched prebuilt OpenShell CLI for E2E tests
2+
description: Download architecture-matched prebuilt OpenShell host CLIs for E2E tests
33

44
inputs:
55
artifact-prefix:
66
description: Artifact name prefix; linux-<arch> is appended automatically
77
required: true
8+
conformance-artifact-prefix:
9+
description: Optional conformance CLI artifact name prefix; linux-<arch> is appended automatically
10+
required: false
11+
default: ""
812

913
runs:
1014
using: composite
@@ -27,3 +31,24 @@ runs:
2731
chmod +x "$cli"
2832
"$cli" --version
2933
echo "OPENSHELL_BIN=$cli" >> "$GITHUB_ENV"
34+
35+
- name: Download prebuilt conformance CLI
36+
if: inputs.conformance-artifact-prefix != ''
37+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
38+
with:
39+
name: ${{ format('{0}-linux-{1}', inputs.conformance-artifact-prefix, runner.arch == 'X64' && 'amd64' || 'arm64') }}
40+
path: .e2e/prebuilt-conformance
41+
42+
- name: Configure prebuilt conformance CLI
43+
if: inputs.conformance-artifact-prefix != ''
44+
shell: bash
45+
run: |
46+
set -euo pipefail
47+
conformance="$GITHUB_WORKSPACE/.e2e/prebuilt-conformance/openshell-conformance"
48+
if [[ ! -f "$conformance" ]]; then
49+
echo "downloaded artifact is missing $conformance" >&2
50+
exit 1
51+
fi
52+
chmod +x "$conformance"
53+
"$conformance" list --output json >/dev/null
54+
echo "OPENSHELL_CONFORMANCE_BIN=$conformance" >> "$GITHUB_ENV"

‎.github/workflows/branch-e2e.yml‎

Lines changed: 42 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,17 @@ jobs:
109109
component: cli
110110
platform: linux/amd64,linux/arm64
111111

112+
build-conformance:
113+
needs: [pr_metadata]
114+
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_any_e2e == 'true'
115+
permissions:
116+
contents: read
117+
packages: write
118+
uses: ./.github/workflows/docker-build.yml
119+
with:
120+
component: conformance
121+
platform: linux/amd64,linux/arm64
122+
112123
build-driver-vm-linux:
113124
needs: [pr_metadata]
114125
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
@@ -124,7 +135,7 @@ jobs:
124135
[{"arch":"amd64","runner":"linux-amd64-cpu8","target":"x86_64-unknown-linux-gnu","zig_target":"x86_64-unknown-linux-gnu.2.28","platform":"linux-x86_64","guest_arch":"x86_64"}]
125136
126137
e2e:
127-
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux]
138+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, build-driver-vm-linux]
128139
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
129140
permissions:
130141
actions: read
@@ -135,11 +146,12 @@ jobs:
135146
image-tag: ${{ github.sha }}
136147
runner: linux-arm64-cpu8
137148
cli-artifact-prefix: rust-binary-cli
149+
conformance-artifact-prefix: rust-binary-conformance
138150
gateway-artifact-prefix: rust-binary-gateway
139151
vm-driver-artifact-name: driver-vm-linux-amd64
140152

141153
gpu-e2e:
142-
needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
154+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
143155
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_gpu_e2e == 'true'
144156
permissions:
145157
actions: read
@@ -149,10 +161,11 @@ jobs:
149161
with:
150162
image-tag: ${{ github.sha }}
151163
cli-artifact-prefix: rust-binary-cli
164+
conformance-artifact-prefix: rust-binary-conformance
152165
gateway-artifact-prefix: rust-binary-gateway
153166

154167
kubernetes-e2e:
155-
needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
168+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
156169
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
157170
strategy:
158171
fail-fast: false
@@ -181,9 +194,10 @@ jobs:
181194
agent-sandbox-version: ${{ matrix.agent_sandbox_version }}
182195
extra-helm-values: ${{ matrix.extra_helm_values }}
183196
cli-artifact-prefix: rust-binary-cli
197+
conformance-artifact-prefix: rust-binary-conformance
184198

185199
kubernetes-workspace-managed-e2e:
186-
needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
200+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
187201
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
188202
permissions:
189203
actions: read
@@ -195,9 +209,10 @@ jobs:
195209
job-name: Kubernetes E2E (workspace managed mode)
196210
e2e-task: e2e:kubernetes:workspace-managed
197211
cli-artifact-prefix: rust-binary-cli
212+
conformance-artifact-prefix: rust-binary-conformance
198213

199214
kubernetes-external-driver-e2e:
200-
needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
215+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
201216
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
202217
permissions:
203218
actions: read
@@ -209,9 +224,10 @@ jobs:
209224
job-name: Kubernetes E2E (external compute driver)
210225
e2e-task: e2e:kubernetes:external-driver
211226
cli-artifact-prefix: rust-binary-cli
227+
conformance-artifact-prefix: rust-binary-conformance
212228

213229
kubernetes-workspace-operator-e2e:
214-
needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
230+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
215231
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
216232
permissions:
217233
actions: read
@@ -223,9 +239,10 @@ jobs:
223239
job-name: Kubernetes E2E (workspace operator mode)
224240
e2e-task: e2e:kubernetes:workspace-operator
225241
cli-artifact-prefix: rust-binary-cli
242+
conformance-artifact-prefix: rust-binary-conformance
226243

227244
kubernetes-ha-e2e:
228-
needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
245+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
229246
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true'
230247
permissions:
231248
actions: read
@@ -238,9 +255,10 @@ jobs:
238255
extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml
239256
external-postgres-secret: openshell-ha-pg
240257
cli-artifact-prefix: rust-binary-cli
258+
conformance-artifact-prefix: rust-binary-conformance
241259

242260
kubernetes-credential-drivers-e2e:
243-
needs: [pr_metadata, build-gateway, build-supervisor]
261+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
244262
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true'
245263
permissions:
246264
actions: read
@@ -251,10 +269,12 @@ jobs:
251269
image-tag: ${{ github.sha }}
252270
job-name: Kubernetes Credential Drivers E2E
253271
e2e-task: e2e:kubernetes:credential-drivers
272+
cli-artifact-prefix: rust-binary-cli
273+
conformance-artifact-prefix: rust-binary-conformance
254274

255275
core-e2e-result:
256276
name: Core E2E result
257-
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e]
277+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e]
258278
if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
259279
runs-on: ubuntu-latest
260280
steps:
@@ -263,6 +283,7 @@ jobs:
263283
BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }}
264284
BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }}
265285
BUILD_CLI_RESULT: ${{ needs.build-cli.result }}
286+
BUILD_CONFORMANCE_RESULT: ${{ needs.build-conformance.result }}
266287
BUILD_DRIVER_VM_RESULT: ${{ needs.build-driver-vm-linux.result }}
267288
E2E_RESULT: ${{ needs.e2e.result }}
268289
KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }}
@@ -276,6 +297,7 @@ jobs:
276297
"build-gateway:$BUILD_GATEWAY_RESULT" \
277298
"build-supervisor:$BUILD_SUPERVISOR_RESULT" \
278299
"build-cli:$BUILD_CLI_RESULT" \
300+
"build-conformance:$BUILD_CONFORMANCE_RESULT" \
279301
"build-driver-vm-linux:$BUILD_DRIVER_VM_RESULT" \
280302
"e2e:$E2E_RESULT" \
281303
"kubernetes-e2e:$KUBERNETES_E2E_RESULT" \
@@ -293,7 +315,7 @@ jobs:
293315
294316
gpu-e2e-result:
295317
name: GPU E2E result
296-
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, gpu-e2e]
318+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, gpu-e2e]
297319
if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_gpu_e2e == 'true'
298320
runs-on: ubuntu-latest
299321
steps:
@@ -302,6 +324,7 @@ jobs:
302324
BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }}
303325
BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }}
304326
BUILD_CLI_RESULT: ${{ needs.build-cli.result }}
327+
BUILD_CONFORMANCE_RESULT: ${{ needs.build-conformance.result }}
305328
GPU_E2E_RESULT: ${{ needs.gpu-e2e.result }}
306329
run: |
307330
set -euo pipefail
@@ -310,6 +333,7 @@ jobs:
310333
"build-gateway:$BUILD_GATEWAY_RESULT" \
311334
"build-supervisor:$BUILD_SUPERVISOR_RESULT" \
312335
"build-cli:$BUILD_CLI_RESULT" \
336+
"build-conformance:$BUILD_CONFORMANCE_RESULT" \
313337
"gpu-e2e:$GPU_E2E_RESULT"; do
314338
name="${item%%:*}"
315339
result="${item#*:}"
@@ -322,7 +346,7 @@ jobs:
322346
323347
kubernetes-ha-e2e-result:
324348
name: Kubernetes HA E2E result
325-
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, kubernetes-ha-e2e]
349+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, kubernetes-ha-e2e]
326350
if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true'
327351
runs-on: ubuntu-latest
328352
steps:
@@ -331,6 +355,7 @@ jobs:
331355
BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }}
332356
BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }}
333357
BUILD_CLI_RESULT: ${{ needs.build-cli.result }}
358+
BUILD_CONFORMANCE_RESULT: ${{ needs.build-conformance.result }}
334359
KUBERNETES_HA_E2E_RESULT: ${{ needs.kubernetes-ha-e2e.result }}
335360
run: |
336361
set -euo pipefail
@@ -339,6 +364,7 @@ jobs:
339364
"build-gateway:$BUILD_GATEWAY_RESULT" \
340365
"build-supervisor:$BUILD_SUPERVISOR_RESULT" \
341366
"build-cli:$BUILD_CLI_RESULT" \
367+
"build-conformance:$BUILD_CONFORMANCE_RESULT" \
342368
"kubernetes-ha-e2e:$KUBERNETES_HA_E2E_RESULT"; do
343369
name="${item%%:*}"
344370
result="${item#*:}"
@@ -351,21 +377,25 @@ jobs:
351377
352378
kubernetes-credential-drivers-e2e-result:
353379
name: Kubernetes Credential Drivers E2E result
354-
needs: [pr_metadata, build-gateway, build-supervisor, kubernetes-credential-drivers-e2e]
380+
needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, kubernetes-credential-drivers-e2e]
355381
if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true'
356382
runs-on: ubuntu-latest
357383
steps:
358384
- name: Verify Kubernetes credential drivers E2E jobs
359385
env:
360386
BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }}
361387
BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }}
388+
BUILD_CLI_RESULT: ${{ needs.build-cli.result }}
389+
BUILD_CONFORMANCE_RESULT: ${{ needs.build-conformance.result }}
362390
KUBERNETES_CREDENTIAL_DRIVERS_E2E_RESULT: ${{ needs.kubernetes-credential-drivers-e2e.result }}
363391
run: |
364392
set -euo pipefail
365393
failed=0
366394
for item in \
367395
"build-gateway:$BUILD_GATEWAY_RESULT" \
368396
"build-supervisor:$BUILD_SUPERVISOR_RESULT" \
397+
"build-cli:$BUILD_CLI_RESULT" \
398+
"build-conformance:$BUILD_CONFORMANCE_RESULT" \
369399
"kubernetes-credential-drivers-e2e:$KUBERNETES_CREDENTIAL_DRIVERS_E2E_RESULT"; do
370400
name="${item%%:*}"
371401
result="${item#*:}"

‎.github/workflows/docker-build.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ on:
44
workflow_call:
55
inputs:
66
component:
7-
description: "Component to build (gateway, supervisor, cli)"
7+
description: "Component to build (gateway, supervisor, cli, conformance)"
88
required: true
99
type: string
1010
timeout-minutes:
@@ -104,6 +104,12 @@ jobs:
104104
features=""
105105
has_image=false
106106
;;
107+
conformance)
108+
binary_component=conformance
109+
binary_name=openshell-conformance
110+
features=""
111+
has_image=false
112+
;;
107113
*)
108114
echo "unsupported component: $component" >&2
109115
exit 1

‎.github/workflows/e2e-gpu-test.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ on:
1212
required: false
1313
type: string
1414
default: ""
15+
conformance-artifact-prefix:
16+
description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is linux-<arch>)"
17+
required: false
18+
type: string
19+
default: ""
1520
gateway-artifact-prefix:
1621
description: "Optional prebuilt gateway artifact prefix (artifact suffix is linux-<arch>)"
1722
required: false
@@ -74,6 +79,7 @@ jobs:
7479
uses: ./.github/actions/setup-e2e-cli
7580
with:
7681
artifact-prefix: ${{ inputs.cli-artifact-prefix }}
82+
conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
7783

7884
- name: Use prebuilt OpenShell gateway
7985
if: inputs.gateway-artifact-prefix != ''

‎.github/workflows/e2e-kubernetes-test.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ on:
5252
required: false
5353
type: string
5454
default: ""
55+
conformance-artifact-prefix:
56+
description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is linux-<arch>)"
57+
required: false
58+
type: string
59+
default: ""
5560

5661
permissions:
5762
actions: read
@@ -85,6 +90,7 @@ jobs:
8590
uses: ./.github/actions/setup-e2e-cli
8691
with:
8792
artifact-prefix: ${{ inputs.cli-artifact-prefix }}
93+
conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
8894

8995
- name: Install mise
9096
run: |

‎.github/workflows/e2e-test.yml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ on:
2222
required: false
2323
type: string
2424
default: ""
25+
conformance-artifact-prefix:
26+
description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is linux-<arch>)"
27+
required: false
28+
type: string
29+
default: ""
2530
gateway-artifact-prefix:
2631
description: "Optional prebuilt gateway artifact prefix (artifact suffix is linux-<arch>)"
2732
required: false
@@ -57,7 +62,7 @@ jobs:
5762
cmd: "mise run --no-deps --skip-deps e2e:oidc-pkce:docker"
5863
apt_packages: "openssh-client"
5964
- suite: rust-docker
60-
cmd: "mise run --no-deps --skip-deps e2e:rust"
65+
cmd: "mise run --no-deps --skip-deps e2e:docker"
6166
apt_packages: "openssh-client"
6267
- suite: rust-docker-external-driver
6368
cmd: "env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:docker:external-driver"
@@ -93,6 +98,7 @@ jobs:
9398
uses: ./.github/actions/setup-e2e-cli
9499
with:
95100
artifact-prefix: ${{ inputs.cli-artifact-prefix }}
101+
conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
96102

97103
- name: Use prebuilt OpenShell gateway
98104
if: inputs.gateway-artifact-prefix != ''
@@ -183,6 +189,7 @@ jobs:
183189
uses: ./.github/actions/setup-e2e-cli
184190
with:
185191
artifact-prefix: ${{ inputs.cli-artifact-prefix }}
192+
conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
186193

187194
- name: Use prebuilt OpenShell gateway
188195
if: inputs.gateway-artifact-prefix != ''
@@ -340,6 +347,7 @@ jobs:
340347
uses: ./.github/actions/setup-e2e-cli
341348
with:
342349
artifact-prefix: ${{ inputs.cli-artifact-prefix }}
350+
conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
343351

344352
- name: Use prebuilt OpenShell gateway
345353
if: inputs.gateway-artifact-prefix != ''

0 commit comments

Comments
 (0)