feat(e2e): comprehensive E2E test framework with auto-validation #74
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Definitions | |
| on: | |
| workflow_dispatch: | |
| push: | |
| paths: | |
| - 'go.mod' | |
| - 'go.sum' | |
| pull_request: | |
| paths: | |
| - 'go.mod' | |
| - 'go.sum' | |
| jobs: | |
| test-components: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout current repository | |
| uses: actions/checkout@v5 | |
| - name: Setup KubeVela Environment | |
| uses: ./.github/actions/setup-vela-environment | |
| - name: Run Component E2E Tests | |
| run: | | |
| echo "Running E2E tests for component definitions..." | |
| make test-e2e-components \ | |
| TESTDATA_PATH=${{ github.workspace }}/test/builtin-definition-example | |
| - name: Summary | |
| if: always() | |
| run: | | |
| echo "## Component Definition Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- **KubeVela Source:** Extracted from go.mod replace directive" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Installed Definitions" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| kubectl get componentdefinitions -n vela-system --no-headers | wc -l | xargs -I {} echo "Component Definitions: {}" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Test Applications" >> $GITHUB_STEP_SUMMARY | |
| echo "Tested applications from \`test/builtin-definition-example/applications/components/\`:" >> $GITHUB_STEP_SUMMARY | |
| ls -1 test/builtin-definition-example/applications/components/*.yaml | xargs -I {} basename {} | while read f; do echo "- $f"; done >> $GITHUB_STEP_SUMMARY | |
| test-traits: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout current repository | |
| uses: actions/checkout@v5 | |
| - name: Setup KubeVela Environment | |
| uses: ./.github/actions/setup-vela-environment | |
| - name: Run Trait E2E Tests | |
| run: | | |
| echo "Running E2E tests for trait definitions..." | |
| make test-e2e-traits \ | |
| TESTDATA_PATH=${{ github.workspace }}/test/builtin-definition-example | |
| - name: Summary | |
| if: always() | |
| run: | | |
| echo "## Trait Definition Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- **KubeVela Source:** Extracted from go.mod replace directive" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Installed Definitions" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| kubectl get traitdefinitions -n vela-system --no-headers | wc -l | xargs -I {} echo "Trait Definitions: {}" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Test Applications" >> $GITHUB_STEP_SUMMARY | |
| echo "Tested applications from \`test/builtin-definition-example/applications/trait/\`:" >> $GITHUB_STEP_SUMMARY | |
| ls -1 test/builtin-definition-example/applications/trait/*.yaml | xargs -I {} basename {} | while read f; do echo "- $f"; done >> $GITHUB_STEP_SUMMARY | |
| test-policies: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout current repository | |
| uses: actions/checkout@v5 | |
| - name: Setup KubeVela Environment | |
| uses: ./.github/actions/setup-vela-environment | |
| - name: Run Policy E2E Tests | |
| run: | | |
| echo "Running E2E tests for policy definitions..." | |
| make test-e2e-policies \ | |
| TESTDATA_PATH=${{ github.workspace }}/test/builtin-definition-example | |
| - name: Summary | |
| if: always() | |
| run: | | |
| echo "## Policy Definition Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- **KubeVela Source:** Extracted from go.mod replace directive" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Installed Definitions" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| kubectl get policydefinitions -n vela-system --no-headers | wc -l | xargs -I {} echo "Policy Definitions: {}" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Test Applications" >> $GITHUB_STEP_SUMMARY | |
| echo "Tested applications from \`test/builtin-definition-example/applications/policies/\`:" >> $GITHUB_STEP_SUMMARY | |
| ls -1 test/builtin-definition-example/applications/policies/*.yaml | xargs -I {} basename {} | while read f; do echo "- $f"; done >> $GITHUB_STEP_SUMMARY | |
| test-workflowsteps: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout current repository | |
| uses: actions/checkout@v5 | |
| - name: Setup KubeVela Environment | |
| uses: ./.github/actions/setup-vela-environment | |
| - name: Run WorkflowStep E2E Tests | |
| run: | | |
| echo "Running E2E tests for workflowstep definitions..." | |
| make test-e2e-workflowsteps \ | |
| TESTDATA_PATH=${{ github.workspace }}/test/builtin-definition-example | |
| - name: Summary | |
| if: always() | |
| run: | | |
| echo "## WorkflowStep Definition Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- **KubeVela Source:** Extracted from go.mod replace directive" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Installed Definitions" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| kubectl get workflowstepdefinitions -n vela-system --no-headers | wc -l | xargs -I {} echo "WorkflowStep Definitions: {}" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Test Applications" >> $GITHUB_STEP_SUMMARY | |
| echo "Tested applications from \`test/builtin-definition-example/applications/workflowsteps/\`:" >> $GITHUB_STEP_SUMMARY | |
| ls -1 test/builtin-definition-example/applications/workflowsteps/*.yaml | xargs -I {} basename {} | while read f; do echo "- $f"; done >> $GITHUB_STEP_SUMMARY | |
| test-parity: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout current repository | |
| uses: actions/checkout@v5 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.23' | |
| - name: Install k3d | |
| shell: bash | |
| run: | | |
| curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash | |
| - name: Set up Kubernetes (k3d) | |
| shell: bash | |
| run: | | |
| k3d cluster create vela-test --wait --timeout 120s | |
| kubectl config use-context k3d-vela-test | |
| - name: Download and install Vela CLI (latest release) | |
| shell: bash | |
| run: | | |
| curl -fsSl https://kubevela.io/script/install.sh | bash | |
| - name: Install KubeVela | |
| shell: bash | |
| run: | | |
| vela install | |
| - name: Wait for KubeVela to be ready | |
| shell: bash | |
| run: | | |
| kubectl wait --for=condition=available --timeout=300s deployment/kubevela-vela-core -n vela-system || true | |
| kubectl get pods -n vela-system | |
| - name: Generate CUE baseline (with built-in definitions) | |
| run: | | |
| echo "Generating CUE dry-run baseline from built-in definitions..." | |
| make generate-baseline \ | |
| BASELINE_DIR=/tmp/cue-baseline \ | |
| TESTDATA_PATH=${{ github.workspace }}/test/builtin-definition-example | |
| echo "Baseline files generated:" | |
| ls -la /tmp/cue-baseline/ | |
| - name: Extract KubeVela repository info from go.mod | |
| id: extract-vela-info | |
| shell: bash | |
| run: | | |
| REPLACE_LINE=$(grep "replace github.com/oam-dev/kubevela =>" go.mod || true) | |
| if [ -z "$REPLACE_LINE" ]; then | |
| echo "repository=oam-dev/kubevela" >> $GITHUB_OUTPUT | |
| echo "ref=master" >> $GITHUB_OUTPUT | |
| else | |
| REPLACEMENT=$(echo "$REPLACE_LINE" | sed 's/.*=> //') | |
| REPO=$(echo "$REPLACEMENT" | awk '{print $1}' | sed 's|github.com/||') | |
| VERSION=$(echo "$REPLACEMENT" | awk '{print $2}') | |
| COMMIT_SHA=$(echo "$VERSION" | sed 's/.*-//') | |
| echo "repository=$REPO" >> $GITHUB_OUTPUT | |
| echo "ref=$COMMIT_SHA" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Clone KubeVela and build CLI | |
| shell: bash | |
| run: | | |
| git clone "https://github.com/${{ steps.extract-vela-info.outputs.repository }}.git" kubevela-source | |
| cd kubevela-source | |
| git checkout "${{ steps.extract-vela-info.outputs.ref }}" | |
| make vela-cli | |
| - name: Uninstall built-in definitions and install defkit | |
| run: | | |
| kubectl delete componentdefinitions --all -n vela-system || true | |
| kubectl delete traitdefinitions --all -n vela-system || true | |
| kubectl delete workflowstepdefinitions --all -n vela-system || true | |
| kubectl delete policydefinitions --all -n vela-system || true | |
| make tidy | |
| kubevela-source/bin/vela def apply-module . --conflict=overwrite | |
| - name: Install Ginkgo | |
| run: | | |
| make install-ginkgo | |
| echo "$(go env GOPATH)/bin" >> $GITHUB_PATH | |
| - name: Run Parity Tests | |
| run: | | |
| echo "Running dry-run parity tests..." | |
| make test-e2e-parity \ | |
| BASELINE_DIR=/tmp/cue-baseline \ | |
| TESTDATA_PATH=${{ github.workspace }}/test/builtin-definition-example | |
| - name: Summary | |
| if: always() | |
| run: | | |
| echo "## Dry-Run Parity Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Compares defkit-generated definitions against built-in CUE definitions" >> $GITHUB_STEP_SUMMARY | |
| echo "using \`vela dry-run\` output." >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Baseline Files" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| ls -1 /tmp/cue-baseline/ 2>/dev/null | wc -l | xargs -I {} echo "Total baseline files: {}" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY |