Skip to content

Commit e8f6b75

Browse files
eurijoneurijon
committed
Add OSAC plugin skeleton and E2E CI workflow
Introduce the OSAC operator as an enclave plugin with a minimal deploy task that pulls the operator manifests from a container image and applies them to the cluster. Includes a post-validation check that waits for the controller-manager deployment to become available. The E2E workflow deploys a connected cluster and runs the plugin on top, triggered by changes to plugins/osac/ or manual dispatch.
1 parent be135e5 commit e8f6b75

6 files changed

Lines changed: 704 additions & 0 deletions

File tree

‎.github/workflows/e2e-osac-plugin.yml‎

Lines changed: 369 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,369 @@
1+
name: E2E OSAC Plugin
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
skip-cleanup:
7+
description: 'Skip cleanup after deployment (leave infrastructure running)'
8+
required: false
9+
type: boolean
10+
default: false
11+
send-slack-notification:
12+
description: 'Send Slack notification on completion'
13+
required: false
14+
type: boolean
15+
default: false
16+
pull_request:
17+
types: [opened, synchronize, reopened]
18+
paths:
19+
- 'plugins/osac/**'
20+
- '.github/workflows/e2e-osac-plugin.yml'
21+
22+
permissions:
23+
contents: read
24+
checks: write
25+
26+
concurrency:
27+
group: e2e-osac-${{ github.event.pull_request.number || github.run_id }}
28+
cancel-in-progress: true
29+
30+
jobs:
31+
e2e-osac:
32+
name: E2E OSAC Plugin (Connected)
33+
runs-on: [self-hosted, enclave-large]
34+
timeout-minutes: 240
35+
36+
env:
37+
DEV_SCRIPTS_PATH: ${{ vars.DEV_SCRIPTS_PATH }}
38+
BASE_WORKING_DIR: ${{ vars.BASE_WORKING_DIR }}
39+
PULL_SECRET: ${{ secrets.PULL_SECRET }}
40+
ENCLAVE_DEPLOYMENT_MODE: connected
41+
CLEANUP_AFTER: 'true'
42+
STORAGE_PLUGIN: lvms
43+
ENABLED_PLUGINS: lvms
44+
OPENSHIFT_CI: "true"
45+
ENCLAVE_ENABLE_GPU_PASSTHROUGH: "false"
46+
LZ_OS_VARIANT: ${{ vars.LZ_OS_VARIANT }}
47+
48+
steps:
49+
- name: Checkout code
50+
uses: actions/checkout@v4
51+
52+
- name: Generate unique cluster name
53+
uses: ./.github/actions/setup-cluster-name
54+
with:
55+
naming-strategy: hash
56+
prefix: osac
57+
run-id: ${{ github.run_id }}
58+
59+
- name: Setup cluster-specific working directory
60+
run: |
61+
make -f Makefile.ci setup-working-dir
62+
echo "WORKING_DIR=$(cat /tmp/working_dir)" >> $GITHUB_ENV
63+
64+
- name: Workflow information
65+
run: |
66+
echo "## E2E OSAC Plugin - Connected Mode" >> $GITHUB_STEP_SUMMARY
67+
echo "" >> $GITHUB_STEP_SUMMARY
68+
echo "**Started at**: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> $GITHUB_STEP_SUMMARY
69+
echo "**Cluster name**: $ENCLAVE_CLUSTER_NAME" >> $GITHUB_STEP_SUMMARY
70+
echo "**Trigger**: ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY
71+
echo "" >> $GITHUB_STEP_SUMMARY
72+
73+
- name: Pre-flight checks
74+
uses: ./.github/actions/preflight-checks
75+
with:
76+
title: E2E OSAC Plugin
77+
check-pull-secret: 'true'
78+
check-system-resources: 'true'
79+
check-libvirt: 'true'
80+
81+
- name: Allocate unique subnet for cluster
82+
uses: ./.github/actions/allocate-subnet
83+
84+
- name: Create infrastructure
85+
env:
86+
WORKING_DIR: ${{ env.WORKING_DIR }}
87+
run: |
88+
echo "## Creating Infrastructure" >> $GITHUB_STEP_SUMMARY
89+
make -f Makefile.ci environment
90+
echo "Infrastructure created" >> $GITHUB_STEP_SUMMARY
91+
92+
- name: Provision Landing Zone
93+
env:
94+
LZ_CLOUD_IMAGE_URL: ${{ vars.LZ_CLOUD_IMAGE_URL }}
95+
LZ_CLOUD_IMAGE_NAME: ${{ vars.LZ_CLOUD_IMAGE_NAME }}
96+
LZ_RHSM_ORG: ${{ secrets.LZ_RHSM_ORG }}
97+
LZ_RHSM_ACTIVATION_KEY: ${{ secrets.LZ_RHSM_ACTIVATION_KEY }}
98+
run: |
99+
echo "## Provisioning Landing Zone" >> $GITHUB_STEP_SUMMARY
100+
make -f Makefile.ci provision-landing-zone
101+
echo "Landing Zone provisioned" >> $GITHUB_STEP_SUMMARY
102+
103+
- name: Install Enclave Lab
104+
run: |
105+
echo "## Installing Enclave Lab" >> $GITHUB_STEP_SUMMARY
106+
make -f Makefile.ci install-enclave
107+
echo "Enclave Lab installed" >> $GITHUB_STEP_SUMMARY
108+
109+
- name: Bootstrap - Setup environment
110+
id: bootstrap_setup
111+
run: |
112+
echo "## Bootstrap: Setup" >> $GITHUB_STEP_SUMMARY
113+
make -f Makefile.ci deploy-cluster-setup
114+
echo "Setup complete" >> $GITHUB_STEP_SUMMARY
115+
116+
- name: Bootstrap - Validate configuration
117+
id: bootstrap_validate
118+
run: |
119+
echo "## Bootstrap: Validate" >> $GITHUB_STEP_SUMMARY
120+
make -f Makefile.ci deploy-cluster-validate
121+
echo "Validation complete" >> $GITHUB_STEP_SUMMARY
122+
123+
- name: Bootstrap - Download content
124+
id: bootstrap_download_content
125+
run: |
126+
echo "## Bootstrap: Download Content" >> $GITHUB_STEP_SUMMARY
127+
make -f Makefile.ci deploy-cluster-prepare
128+
echo "Download complete" >> $GITHUB_STEP_SUMMARY
129+
130+
- name: Bootstrap - Build local cache
131+
id: bootstrap_build_cache
132+
run: |
133+
echo "## Bootstrap: Build Cache" >> $GITHUB_STEP_SUMMARY
134+
make -f Makefile.ci deploy-cluster-mirror
135+
echo "Cache build complete" >> $GITHUB_STEP_SUMMARY
136+
137+
- name: Bootstrap - Acquire hardware
138+
id: bootstrap_acquire_hardware
139+
run: |
140+
echo "## Bootstrap: Acquire Hardware" >> $GITHUB_STEP_SUMMARY
141+
make -f Makefile.ci deploy-cluster-acquire-hardware
142+
echo "Hardware acquired" >> $GITHUB_STEP_SUMMARY
143+
144+
- name: Bootstrap - Deploy cluster
145+
id: bootstrap_deploy
146+
run: |
147+
echo "## Bootstrap: Deploy Cluster" >> $GITHUB_STEP_SUMMARY
148+
make -f Makefile.ci deploy-cluster-install
149+
echo "Cluster deployed" >> $GITHUB_STEP_SUMMARY
150+
151+
- name: Bootstrap - Post-install
152+
id: bootstrap_post_install
153+
run: |
154+
echo "## Bootstrap: Post-Install" >> $GITHUB_STEP_SUMMARY
155+
make -f Makefile.ci deploy-cluster-post-install
156+
echo "Post-install complete" >> $GITHUB_STEP_SUMMARY
157+
158+
- name: Bootstrap - Operators
159+
id: bootstrap_operators
160+
run: |
161+
echo "## Bootstrap: Operators" >> $GITHUB_STEP_SUMMARY
162+
make -f Makefile.ci deploy-cluster-operators
163+
echo "Operators installed" >> $GITHUB_STEP_SUMMARY
164+
165+
- name: Deploy OSAC plugin
166+
id: deploy_osac
167+
run: |
168+
echo "## Deploy OSAC Plugin" >> $GITHUB_STEP_SUMMARY
169+
echo "" >> $GITHUB_STEP_SUMMARY
170+
make -f Makefile.ci deploy-plugin PLUGIN=osac
171+
echo "OSAC plugin deployed" >> $GITHUB_STEP_SUMMARY
172+
173+
- name: Verify cluster deployment
174+
if: success()
175+
id: verify_cluster
176+
run: make -f Makefile.ci verify-cluster
177+
178+
- name: Collect artifacts
179+
if: always()
180+
uses: ./.github/actions/collect-artifacts
181+
with:
182+
artifact-type: deployment
183+
output-directory: artifacts
184+
185+
- name: Collect full diagnostics on failure
186+
if: failure()
187+
run: |
188+
echo "## Collecting Full Diagnostics (failure)" >> $GITHUB_STEP_SUMMARY
189+
echo "" >> $GITHUB_STEP_SUMMARY
190+
191+
if ! ./scripts/verification/collect_ci_artifacts.sh full artifacts 2>&1 | tee artifact-collection-full.log; then
192+
echo "Full artifact collection failed; continuing with must-gather" >> $GITHUB_STEP_SUMMARY
193+
fi
194+
195+
LZ_IP=$(./scripts/utils/get_landing_zone_ip.sh)
196+
if [ -n "$LZ_IP" ]; then
197+
SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10"
198+
199+
if ssh $SSH_OPTS cloud-user@$LZ_IP "test -f /home/cloud-user/enclave/scripts/diagnostics/gather.sh" 2>/dev/null; then
200+
echo "" >> $GITHUB_STEP_SUMMARY
201+
echo "### Running must-gather..." >> $GITHUB_STEP_SUMMARY
202+
203+
mkdir -p artifacts/must-gather
204+
GATHER_OUT=$(ssh $SSH_OPTS cloud-user@$LZ_IP \
205+
"cd /home/cloud-user/enclave/scripts/diagnostics && \
206+
export KUBECONFIG=/home/cloud-user/ocp-cluster/auth/kubeconfig && \
207+
GITHUB_RUN_ID='${{ github.run_id }}' ./gather.sh --must-gather=full ../../config/global.yaml 2>&1" || true)
208+
echo "$GATHER_OUT" >> artifacts/must-gather/gather-output.txt
209+
210+
scp $SSH_OPTS "cloud-user@${LZ_IP}:/home/cloud-user/enclave/scripts/diagnostics/lz-logs-*.tar.gz" artifacts/must-gather/ 2>/dev/null || true
211+
scp $SSH_OPTS "cloud-user@${LZ_IP}:/home/cloud-user/enclave/scripts/diagnostics/cluster-logs-*.tar.gz" artifacts/must-gather/ 2>/dev/null || true
212+
213+
if ls artifacts/must-gather/*-logs-*.tar.gz 1>/dev/null 2>&1; then
214+
echo "Collected must-gather archives" >> $GITHUB_STEP_SUMMARY
215+
else
216+
echo "Must-gather failed (see must-gather/gather-output.txt)" >> $GITHUB_STEP_SUMMARY
217+
fi
218+
fi
219+
fi
220+
221+
- name: Upload artifacts
222+
if: always()
223+
id: upload_artifacts
224+
uses: actions/upload-artifact@v4
225+
with:
226+
name: e2e-osac-${{ env.ENCLAVE_CLUSTER_NAME }}-${{ github.run_id }}
227+
path: artifacts/
228+
retention-days: 7
229+
if-no-files-found: warn
230+
231+
- name: Get artifact download URL
232+
if: always()
233+
id: artifact_url
234+
run: |
235+
ARTIFACT_ID=""
236+
for attempt in 1 2 3 4 5; do
237+
sleep $((attempt * 2))
238+
ARTIFACT_ID=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts \
239+
--jq '[.artifacts[] | select(.name == "e2e-osac-${{ env.ENCLAVE_CLUSTER_NAME }}-${{ github.run_id }}")] | sort_by(.created_at) | last // empty | .id')
240+
if [ -n "$ARTIFACT_ID" ]; then
241+
break
242+
fi
243+
done
244+
245+
if [ -n "$ARTIFACT_ID" ]; then
246+
ARTIFACT_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/${ARTIFACT_ID}"
247+
echo "ARTIFACT_URL=${ARTIFACT_URL}" >> $GITHUB_OUTPUT
248+
else
249+
echo "ARTIFACT_URL=" >> $GITHUB_OUTPUT
250+
fi
251+
env:
252+
GH_TOKEN: ${{ github.token }}
253+
254+
- name: Cleanup infrastructure
255+
if: always() && (github.event_name != 'workflow_dispatch' || inputs.skip-cleanup != true)
256+
run: |
257+
echo "## Cleanup Infrastructure" >> $GITHUB_STEP_SUMMARY
258+
echo "" >> $GITHUB_STEP_SUMMARY
259+
echo "Removing infrastructure for cluster: $ENCLAVE_CLUSTER_NAME" >> $GITHUB_STEP_SUMMARY
260+
261+
mkdir -p cleanup-logs
262+
263+
set +e
264+
make -f Makefile.ci clean 2>&1 | tee cleanup-logs/cleanup.log
265+
CLEANUP_EXIT_CODE=$?
266+
set -e
267+
268+
if grep -q "WARNING:" cleanup-logs/cleanup.log; then
269+
echo "" >> $GITHUB_STEP_SUMMARY
270+
echo "Cleanup completed with warnings:" >> $GITHUB_STEP_SUMMARY
271+
echo '```' >> $GITHUB_STEP_SUMMARY
272+
grep "WARNING:" cleanup-logs/cleanup.log >> $GITHUB_STEP_SUMMARY
273+
echo '```' >> $GITHUB_STEP_SUMMARY
274+
elif [ $CLEANUP_EXIT_CODE -eq 0 ]; then
275+
echo "Infrastructure cleaned up successfully" >> $GITHUB_STEP_SUMMARY
276+
else
277+
echo "Cleanup completed with errors (exit code: $CLEANUP_EXIT_CODE)" >> $GITHUB_STEP_SUMMARY
278+
fi
279+
280+
- name: Cleanup skipped notice
281+
if: always() && github.event_name == 'workflow_dispatch' && inputs.skip-cleanup == true
282+
run: |
283+
echo "## Cleanup Skipped" >> $GITHUB_STEP_SUMMARY
284+
echo "" >> $GITHUB_STEP_SUMMARY
285+
echo "Infrastructure cleanup was skipped as requested." >> $GITHUB_STEP_SUMMARY
286+
echo "**Cluster name**: $ENCLAVE_CLUSTER_NAME" >> $GITHUB_STEP_SUMMARY
287+
288+
- name: Upload post-cleanup artifacts
289+
if: always()
290+
uses: actions/upload-artifact@v4
291+
with:
292+
name: post-cleanup-osac-${{ github.run_id }}
293+
path: cleanup-logs/
294+
retention-days: 7
295+
296+
- name: Summary
297+
if: always()
298+
run: |
299+
echo "" >> $GITHUB_STEP_SUMMARY
300+
echo "---" >> $GITHUB_STEP_SUMMARY
301+
echo "" >> $GITHUB_STEP_SUMMARY
302+
echo "## Deployment Summary" >> $GITHUB_STEP_SUMMARY
303+
echo "- **Mode**: Connected" >> $GITHUB_STEP_SUMMARY
304+
echo "- **Plugin**: osac" >> $GITHUB_STEP_SUMMARY
305+
echo "- **Cluster**: $ENCLAVE_CLUSTER_NAME" >> $GITHUB_STEP_SUMMARY
306+
echo "- **Branch**: ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY
307+
echo "- **Commit**: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY
308+
echo "- **Triggered by**: @${{ github.actor }}" >> $GITHUB_STEP_SUMMARY
309+
echo "- **Completed at**: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> $GITHUB_STEP_SUMMARY
310+
echo "- **Status**: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY
311+
312+
- name: Determine failed step
313+
if: failure()
314+
id: failed_step
315+
env:
316+
STEPS_JSON: ${{ toJSON(steps) }}
317+
run: |
318+
declare -A step_names=(
319+
["bootstrap_setup"]="Bootstrap: Setup environment"
320+
["bootstrap_validate"]="Bootstrap: Validate configuration"
321+
["bootstrap_download_content"]="Bootstrap: Download content"
322+
["bootstrap_build_cache"]="Bootstrap: Build local cache"
323+
["bootstrap_acquire_hardware"]="Bootstrap: Acquire hardware"
324+
["bootstrap_deploy"]="Bootstrap: Deploy cluster"
325+
["bootstrap_post_install"]="Bootstrap: Post-install"
326+
["bootstrap_operators"]="Bootstrap: Operators"
327+
["deploy_osac"]="Deploy OSAC plugin"
328+
["verify_cluster"]="Verify cluster deployment"
329+
)
330+
331+
step_order=(
332+
bootstrap_setup
333+
bootstrap_validate
334+
bootstrap_download_content
335+
bootstrap_build_cache
336+
bootstrap_acquire_hardware
337+
bootstrap_deploy
338+
bootstrap_post_install
339+
bootstrap_operators
340+
deploy_osac
341+
verify_cluster
342+
)
343+
344+
FAILED_STEP=""
345+
for step_id in "${step_order[@]}"; do
346+
outcome=$(printf '%s' "$STEPS_JSON" | jq -r --arg id "$step_id" '.[$id].outcome // "skipped"')
347+
if [ "$outcome" == "failure" ]; then
348+
FAILED_STEP="${step_names[$step_id]}"
349+
break
350+
fi
351+
done
352+
353+
if [ -n "$FAILED_STEP" ]; then
354+
echo "FAILED_STEP=$FAILED_STEP" >> $GITHUB_OUTPUT
355+
fi
356+
357+
- name: Notify Slack
358+
if: always() && inputs.send-slack-notification == true
359+
uses: ./.github/actions/notify-slack
360+
with:
361+
status: ${{ job.status }}
362+
workflow-name: E2E OSAC Plugin
363+
cluster-name: ${{ env.ENCLAVE_CLUSTER_NAME }}
364+
slack-webhook-urls: ${{ secrets.SLACK_WEBHOOK_URLS }}
365+
workflow-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
366+
branch-name: ${{ github.ref_name }}
367+
commit-sha: ${{ github.sha }}
368+
failed-step: ${{ steps.failed_step.outputs.FAILED_STEP }}
369+
artifact-url: ${{ steps.artifact_url.outputs.ARTIFACT_URL }}

0 commit comments

Comments
 (0)