refactor: codebase consistency fixes and test coverage (#179)

Author: mchmarny

.github/actions/kwok-test/action.yml

@@ -13,13 +13,12 @@
 # limitations under the License.
 
 name: 'KWOK Cluster Test'
-description: 'Test recipes using KWOK simulated nodes in a shared Kind cluster'
+description: 'Test a single recipe using KWOK simulated nodes in a Kind cluster'
 
 inputs:
   recipe:
-    description: 'Recipe name to test (empty = all testable recipes)'
-    required: false
-    default: ''
+    description: 'Recipe name to test'
+    required: true
   go_version:
     description: 'Go version to install'
     required: true
@@ -156,49 +155,42 @@ runs:
         make build
         ls -la dist/
 
-    - name: Run KWOK recipe tests
+    - name: Run KWOK recipe test
       id: validate
       shell: bash
       env:
-        KWOK_CLUSTER: aicr-kwok-test
         KIND_NODE_IMAGE: ${{ inputs.kind_node_image }}
       run: |
-        bash kwok/scripts/run-all-recipes.sh ${{ inputs.recipe }}
+        KWOK_CLUSTER=aicr-kwok-test bash kwok/scripts/run-all-recipes.sh ${{ inputs.recipe }}
 
     - name: Collect debug artifacts
       if: failure()
       shell: bash
-      env:
-        KWOK_CLUSTER: aicr-kwok-test
       run: |
         mkdir -p /tmp/kwok-debug-artifacts
-        kubectl get all --all-namespaces > /tmp/kwok-debug-artifacts/all-resources.txt || true
-        kubectl get nodes -o wide > /tmp/kwok-debug-artifacts/nodes.txt || true
-        kubectl get nodes -l type=kwok -o yaml > /tmp/kwok-debug-artifacts/kwok-nodes.yaml || true
-        kubectl get events --all-namespaces --sort-by='.lastTimestamp' > /tmp/kwok-debug-artifacts/events.txt || true
-        kubectl get pods --all-namespaces -o wide > /tmp/kwok-debug-artifacts/pods.txt || true
-
-        # Get pending pod details
-        for ns in $(kubectl get ns -o jsonpath='{.items[*].metadata.name}'); do
-          for pod in $(kubectl get pods -n "$ns" --field-selector=status.phase=Pending -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do
-            kubectl describe pod -n "$ns" "$pod" > "/tmp/kwok-debug-artifacts/pending-${ns}-${pod}.txt" 2>/dev/null || true
-          done
+        # Collect from whichever cluster is still running
+        for cluster in $(kind get clusters 2>/dev/null); do
+          kind export kubeconfig --name "$cluster" 2>/dev/null || continue
+          kubectl get all --all-namespaces > "/tmp/kwok-debug-artifacts/${cluster}-resources.txt" || true
+          kubectl get nodes -o wide > "/tmp/kwok-debug-artifacts/${cluster}-nodes.txt" || true
+          kubectl get events --all-namespaces --sort-by='.lastTimestamp' > "/tmp/kwok-debug-artifacts/${cluster}-events.txt" || true
+          kubectl get pods --all-namespaces -o wide > "/tmp/kwok-debug-artifacts/${cluster}-pods.txt" || true
         done
 
     - name: Export Kind logs
       if: failure()
       shell: bash
-      env:
-        KWOK_CLUSTER: aicr-kwok-test
       run: |
         mkdir -p /tmp/kwok-kind-logs
-        kind export logs /tmp/kwok-kind-logs --name "${KWOK_CLUSTER}" || true
+        for cluster in $(kind get clusters 2>/dev/null); do
+          kind export logs "/tmp/kwok-kind-logs/${cluster}" --name "$cluster" || true
+        done
 
     - name: Upload debug artifacts
       if: failure() && inputs.upload_artifacts == 'true'
       uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6.0.0
       with:
-        name: kwok-debug-${{ github.run_id }}
+        name: kwok-debug-${{ inputs.recipe }}-${{ github.run_id }}
         path: |
           /tmp/kwok-debug-artifacts/
           /tmp/kwok-kind-logs/
@@ -207,8 +199,8 @@ runs:
     - name: Cleanup
       if: always()
       shell: bash
-      env:
-        KWOK_CLUSTER: aicr-kwok-test
       run: |
-        kind delete cluster --name "${KWOK_CLUSTER}" || true
+        for cluster in $(kind get clusters 2>/dev/null); do
+          kind delete cluster --name "$cluster" || true
+        done
         docker system prune -f || true

.github/workflows/kwok-recipes.yaml

@@ -46,10 +46,59 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  discover:
+    name: Discover Recipes
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    outputs:
+      recipes: ${{ steps.find.outputs.recipes }}
+    steps:
+      - name: Checkout overlays
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
+          sparse-checkout: |
+            recipes/overlays
+          sparse-checkout-cone-mode: false
+
+      - name: Find testable recipes
+        id: find
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          # If a specific recipe was requested via workflow_dispatch, use it
+          if [[ -n "${{ github.event.inputs.recipe }}" ]]; then
+            recipes=$(jq -nc '[$recipe]' --arg recipe "${{ github.event.inputs.recipe }}")
+            echo "recipes=${recipes}" >> "$GITHUB_OUTPUT"
+            echo "Using specific recipe: ${{ github.event.inputs.recipe }}"
+            exit 0
+          fi
+
+          # Scan overlays for recipes with a cloud service criteria
+          recipes="[]"
+          for overlay in recipes/overlays/*.yaml; do
+            name=$(basename "$overlay" .yaml)
+            # yq is pre-installed on ubuntu-latest runners
+            service=$(yq eval '.spec.criteria.service // ""' "$overlay" 2>/dev/null || true)
+            if [[ -n "$service" && "$service" != "null" && "$service" != "any" ]]; then
+              recipes=$(echo "$recipes" | jq -c --arg r "$name" '. + [$r]')
+            fi
+          done
+
+          echo "recipes=${recipes}" >> "$GITHUB_OUTPUT"
+          echo "Discovered $(echo "$recipes" | jq 'length') recipe(s): $(echo "$recipes" | jq -c '.')"
+
   test:
-    name: KWOK Recipe Tests
+    name: KWOK (${{ matrix.recipe }})
+    needs: discover
+    if: ${{ needs.discover.outputs.recipes != '[]' && needs.discover.outputs.recipes != '' }}
     runs-on: ubuntu-latest
-    timeout-minutes: 45
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        recipe: ${{ fromJSON(needs.discover.outputs.recipes) }}
     steps:
       - name: Checkout Code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -60,10 +109,10 @@ jobs:
         id: versions
         uses: ./.github/actions/load-versions
 
-      - name: Run KWOK tests
+      - name: Run KWOK test
         uses: ./.github/actions/kwok-test
         with:
-          recipe: ${{ github.event.inputs.recipe }}
+          recipe: ${{ matrix.recipe }}
           go_version: ${{ steps.versions.outputs.go }}
           kind_version: ${{ steps.versions.outputs.kind }}
           helm_version: ${{ steps.versions.outputs.helm }}
@@ -85,6 +134,8 @@ jobs:
 
           if [[ "${{ needs.test.result }}" == "success" ]]; then
             echo "All recipe validations passed" >> $GITHUB_STEP_SUMMARY
+          elif [[ "${{ needs.test.result }}" == "skipped" ]]; then
+            echo "No recipes to test" >> $GITHUB_STEP_SUMMARY
           elif [[ "${{ needs.test.result }}" == "cancelled" ]]; then
             echo "Recipe validations were cancelled" >> $GITHUB_STEP_SUMMARY
             exit 1

.gitignore

@@ -123,6 +123,7 @@ tags
 # Claude
 .claude/settings.local.json
 docs/plans
+.worktrees/
 
 # Validator generator scaffolding output
 pkg/validator/checks/**/*_README.md

Makefile

@@ -479,10 +479,6 @@ endif
 kwok-test-all: build ## Run all KWOK recipe tests in a shared cluster
 	@bash kwok/scripts/run-all-recipes.sh
 
-.PHONY: kwok-test-all-parallel
-kwok-test-all-parallel: build ## Run all KWOK recipe tests in parallel across multiple clusters
-	@bash kwok/scripts/run-all-recipes-parallel.sh
-
 # =============================================================================
 # Combined Development Targets
 # =============================================================================
@@ -547,7 +543,6 @@ help-full: ## Displays commands grouped by category
 	@echo "  make kwok-status    Show KWOK cluster and node status"
 	@echo "  make kwok-e2e       Full KWOK workflow (RECIPE=<name>)"
 	@echo "  make kwok-test-all  Run all recipes in shared cluster"
-	@echo "  make kwok-test-all-parallel  Run all recipes in parallel clusters (faster)"
 	@echo ""
 	@echo "\033[1m=== Code Maintenance ===\033[0m"
 	@echo "  make tidy           Format code and update dependencies"