ci(kwok): add flux-git lane with in-cluster gitea (#1290)

Signed-off-by: Christopher Haar <christopher.haar@upbound.io>
Co-authored-by: Mark Chmarny <mchmarny@users.noreply.github.com>

Author: haarchri

.github/actions/kwok-test/action.yml

@@ -20,7 +20,7 @@ inputs:
     description: 'Recipe name to test'
     required: true
   deployer:
-    description: 'Deployer to exercise: helm | argocd-oci | argocd-helm-oci | flux-oci'
+    description: 'Deployer to exercise: helm | argocd-oci | argocd-helm-oci | flux-oci | flux-git'
     required: false
     default: 'helm'
   go_version:
@@ -50,7 +50,7 @@ inputs:
     required: false
     default: 'v4.52.4'
   flux_version:
-    description: 'Flux CLI version to install (required for flux-oci deployer)'
+    description: 'Flux CLI version to install (required for the flux-oci and flux-git deployers)'
     required: false
     default: ''
   chainsaw_version:
@@ -111,7 +111,7 @@ runs:
         flux_version: '${{ inputs.flux_version }}'
         install_goreleaser: 'true'
         goreleaser_version: '${{ inputs.goreleaser_version }}'
-        # chainsaw drives the argocd-* / flux-oci sync gates from
+        # chainsaw drives the argocd-* / flux-* sync gates from
         # kwok/scripts/validate-scheduling.sh (issue #962 migration).
         install_chainsaw: 'true'
         chainsaw_version: '${{ inputs.chainsaw_version }}'
@@ -150,11 +150,12 @@ runs:
           kubectl get applications -n argocd -o yaml > "/tmp/kwok-debug-artifacts/${cluster}-argo-apps.yaml" 2>/dev/null || true
           kubectl logs -n argocd deploy/argocd-repo-server --tail=500 > "/tmp/kwok-debug-artifacts/${cluster}-argo-reposerver.log" 2>/dev/null || true
           kubectl logs -n argocd statefulset/argocd-application-controller --tail=500 > "/tmp/kwok-debug-artifacts/${cluster}-argo-appcontroller.log" 2>/dev/null || true
-          kubectl get ocirepositories,kustomizations,helmreleases -A -o yaml > "/tmp/kwok-debug-artifacts/${cluster}-flux-resources.yaml" 2>/dev/null || true
+          kubectl get ocirepositories,gitrepositories,kustomizations,helmreleases -A -o yaml > "/tmp/kwok-debug-artifacts/${cluster}-flux-resources.yaml" 2>/dev/null || true
           kubectl logs -n flux-system deploy/source-controller --tail=500 > "/tmp/kwok-debug-artifacts/${cluster}-flux-source-controller.log" 2>/dev/null || true
           kubectl logs -n flux-system deploy/kustomize-controller --tail=500 > "/tmp/kwok-debug-artifacts/${cluster}-flux-kustomize-controller.log" 2>/dev/null || true
           kubectl logs -n flux-system deploy/helm-controller --tail=500 > "/tmp/kwok-debug-artifacts/${cluster}-flux-helm-controller.log" 2>/dev/null || true
           kubectl logs -n aicr-registry deploy/registry --tail=200 > "/tmp/kwok-debug-artifacts/${cluster}-registry.log" 2>/dev/null || true
+          kubectl logs -n aicr-registry deploy/gitea --tail=200 > "/tmp/kwok-debug-artifacts/${cluster}-gitea.log" 2>/dev/null || true
         done
 
     - name: Export Kind logs

.github/workflows/kwok-recipes.yaml

@@ -228,7 +228,7 @@ jobs:
           # shard's matrix under the limit. Keep this deployer list in sync
           # with the test-tier1 matrix above and the input doc in
           # .github/actions/kwok-test/action.yml.
-          deployers='["helm","argocd-oci","argocd-helm-oci","flux-oci"]'
+          deployers='["helm","argocd-oci","argocd-helm-oci","flux-oci","flux-git"]'
           readonly TIER3_BATCH_SIZE=200  # headroom under GitHub's 256 cap
 
           # Fail closed if the batch size is ever raised past the hard limit —
@@ -275,7 +275,7 @@ jobs:
       fail-fast: false
       matrix:
         recipe: ${{ fromJSON(needs.discover.outputs.tier1) }}
-        deployer: [helm, argocd-oci, argocd-helm-oci, flux-oci]
+        deployer: [helm, argocd-oci, argocd-helm-oci, flux-oci, flux-git]
     steps:
       - name: Checkout Code
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3

.settings.yaml

@@ -122,6 +122,11 @@ testing_tools:
   # bundle consumes (OCIRepository -> Kustomization -> HelmRelease).
   # renovate: datasource=github-releases depName=fluxcd/flux2 depType=testing_tools
   flux_version: 'v2.8.8'
+  # In-cluster Git server for the KWOK flux-git deployer lane (issue #963).
+  # Rootless variant so the admin-user bootstrap can `kubectl exec gitea
+  # admin user create` directly without su gymnastics.
+  # renovate: datasource=docker depName=gitea/gitea depType=testing_tools
+  gitea_image: 'gitea/gitea:1.26.2-rootless'
   # renovate: datasource=github-releases depName=hauler-dev/hauler depType=testing_tools
   hauler: 'v1.4.3'
   # renovate: datasource=github-releases depName=zarf-dev/zarf depType=testing_tools

Makefile

@@ -781,14 +781,14 @@ kwok-test-all: build ## Run all KWOK recipe tests in a shared cluster
 	@bash kwok/scripts/run-all-recipes.sh
 
 .PHONY: kwok-test-deployer
-kwok-test-deployer: build ## Validate scheduling under a specific deployer (RECIPE=… DEPLOYER=helm|argocd-oci|argocd-helm-oci|flux-oci)
+kwok-test-deployer: build ## Validate scheduling under a specific deployer (RECIPE=… DEPLOYER=helm|argocd-oci|argocd-helm-oci|flux-oci|flux-git)
 ifndef RECIPE
 	@echo "Error: RECIPE is required"
 	@echo "Usage: make kwok-test-deployer RECIPE=eks-training DEPLOYER=argocd-oci"
 	@exit 1
 endif
 ifndef DEPLOYER
-	@echo "Error: DEPLOYER is required (helm | argocd-oci | argocd-helm-oci | flux-oci)"
+	@echo "Error: DEPLOYER is required (helm | argocd-oci | argocd-helm-oci | flux-oci | flux-git)"
 	@exit 1
 endif
 	@echo "Validating $(RECIPE) under deployer=$(DEPLOYER)"

docs/contributor/tests.md

@@ -241,8 +241,8 @@ real hardware. CI uses it to validate two things per recipe:
    the overlay expects.
 2. **Deployer output correctness.** The same recipe is re-rendered
    through every output adapter — `helm`, `argocd-oci`,
-   `argocd-helm-oci`, `flux-oci` — and each renders to a working
-   bundle the GitOps controllers can reconcile.
+   `argocd-helm-oci`, `flux-oci`, `flux-git` — and each renders to a
+   working bundle the GitOps controllers can reconcile.
 
 KWOK nodes have no kubelet, so pods never actually run. KWOK testing
 **does not exercise** runtime validators (NCCL, inference-perf) or
@@ -255,31 +255,35 @@ simulated reflection of production shape, not a relaxed substitute.
 
 For the design rationale and the spike findings that justify the
 chart pin and Repository-secret shape, see
-[ADR-008](https://github.com/NVIDIA/aicr/blob/main/docs/design/008-kwok-deployer-matrix.md).
+[ADR-008](https://github.com/NVIDIA/aicr/blob/main/docs/design/008-kwok-deployer-matrix.md);
+for the Git-source lanes (in-cluster Gitea, `flux-git`), see
+[ADR-010](https://github.com/NVIDIA/aicr/blob/main/docs/design/010-kwok-git-source-lanes.md).
 For cluster-level KWOK setup (node profiles, recipe auto-discovery),
 see [kwok/README.md](https://github.com/NVIDIA/aicr/blob/main/kwok/README.md).
 
 ### Deployer Coverage Matrix
 
 | Tier | Trigger | Deployers exercised |
 |------|---------|----------------------|
-| Tier 1 — generic overlays | every PR + push | `helm`, `argocd-oci`, `argocd-helm-oci`, `flux-oci` |
+| Tier 1 — generic overlays | every PR + push | `helm`, `argocd-oci`, `argocd-helm-oci`, `flux-oci`, `flux-git` |
 | Tier 2 — diff-aware accelerator overlays | PR only, conditional on changed files | `helm` only |
-| Tier 3 — full overlay set | push to `main` + nightly schedule | `helm`, `argocd-oci`, `argocd-helm-oci`, `flux-oci` |
+| Tier 3 — full overlay set | push to `main` + nightly schedule | `helm`, `argocd-oci`, `argocd-helm-oci`, `flux-oci`, `flux-git` |
 
 | Lane | Pull artifact | Apply manifests | Reconcile to Ready |
 |---|---|---|---|
 | `helm` | n/a (filesystem) | `helm install` | pods scheduled |
 | `argocd-oci` | repo-server OCI pull | Argo CD sync | `Synced+Healthy` |
 | `argocd-helm-oci` | `helm pull` OCI | wrapper chart install | `Synced+Healthy` |
 | `flux-oci` | source-controller OCI pull | kustomize-controller apply | all HelmReleases `Ready=True` + ArtifactGenerators Ready |
+| `flux-git` | source-controller Git clone (in-cluster Gitea) | kustomize-controller apply | GitRepositories Ready + all HelmReleases `Ready=True` |
 
 Tier 2 stays `helm`-only because its job is to verify accelerator-specific
 overlays still render correctly when their inputs change. The deployer
 shape is orthogonal — re-running through Argo CD would only re-exercise
 template rendering, which Tier 1 and Tier 3 already cover on the generic
-overlays. For filesystem (Git-source) round-trip coverage of `argocd` /
-`flux`, see [#963](https://github.com/NVIDIA/aicr/issues/963).
+overlays. The `flux-git` lane covers the filesystem (Git-source)
+round-trip half of [#963](https://github.com/NVIDIA/aicr/issues/963);
+the `argocd-git` half is a follow-up on the same Gitea infrastructure.
 
 ### Running KWOK Locally
 
@@ -293,11 +297,12 @@ make kwok-test-deployer RECIPE=eks-training DEPLOYER=argocd-oci
 ```
 
 Valid `DEPLOYER` values: `helm`, `argocd-oci`, `argocd-helm-oci`,
-`flux-oci`. The target invokes
+`flux-oci`, `flux-git`. The target invokes
 `kwok/scripts/run-all-recipes.sh --deployer <name> <recipe>`, which
 calls `install-infra.sh` once with `DEPLOYER` exported (in-cluster
 `registry:2` always; Argo CD for `argocd-*`; Flux 2 controllers for
-`flux-oci`), then runs `validate-scheduling.sh` for the recipe.
+`flux-*`; Gitea additionally for `flux-git`), then runs
+`validate-scheduling.sh` for the recipe.
 
 **Registry host port.** The Kind cluster exposes the in-cluster
 `registry:2` Service on **host port 5500** (`kwok/kind-config.yaml`'s
@@ -308,11 +313,21 @@ macOS. Linux runners have 5500 free too. The in-cluster NodePort
 independent — Argo CD's repo-server reaches the registry via Service
 DNS (`registry.aicr-registry.svc.cluster.local:5000`) regardless.
 
+**Gitea host port (flux-git).** The same pattern exposes the
+in-cluster Gitea on **host port 3300** (NodePort `30300`) so the
+runner can `git push` the filesystem bundle; 3300 avoids Gitea's
+default 3000, commonly held by Grafana / local dev servers. Flux's
+source-controller clones via Service DNS
+(`gitea.aicr-registry.svc.cluster.local:3000`). Clusters created
+before the 3300 mapping existed must be recreated
+(`kind delete cluster --name aicr-kwok-test`) — `install-infra.sh`
+exit code 71 is the telltale.
+
 **Sweeping all deployers locally.** `make kwok-test-all` defaults to
 `helm`; there is no matrix-aware make target. Loop in shell:
 
 ```bash
-for d in helm argocd-oci argocd-helm-oci flux-oci; do
+for d in helm argocd-oci argocd-helm-oci flux-oci flux-git; do
   make kwok-test-deployer RECIPE=eks-training DEPLOYER="$d" || break
 done
 
@@ -336,6 +351,9 @@ and local loops can branch on failure mode without parsing logs.
 | `install-infra.sh` | 60 | Flux install manifest apply failed |
 | `install-infra.sh` | 61 | Flux controller not Ready within 180 s |
 | `install-infra.sh` | 62 | Flux CRDs not Established within 60 s |
+| `install-infra.sh` | 70 | Gitea Deployment not Ready within 120 s |
+| `install-infra.sh` | 71 | Gitea not reachable on host port within 60 s (cluster likely predates the 3300 port mapping) |
+| `install-infra.sh` | 72 | Gitea admin user bootstrap failed |
 | `validate-scheduling.sh` | 50 | GitOps sync deadline hit |
 | `run-all-recipes.sh` | 50 | Three consecutive GitOps sync timeouts; ADR-008 3-strike rule tripped |
 
@@ -353,9 +371,16 @@ independent.
 |----------|---------|---------|
 | `KWOK_ARGOCD_SYNC_TIMEOUT` | `300` s | Deadline for all child Argo CD Applications to reach `Synced+Healthy` |
 | `KWOK_ARGOCD_ROOT_GRACE` | `30` s | Grace period for the root Application before deadline counting starts |
-| `KWOK_FLUX_SYNC_TIMEOUT` | `300` s | Deadline for OCIRepository fetch + Kustomization apply + HelmReleases `Ready=True` + ArtifactGenerators Ready |
+| `KWOK_FLUX_SYNC_TIMEOUT` | `500` s | Deadline for source fetch (OCIRepository or GitRepository) + Kustomization apply + HelmReleases `Ready=True` + ArtifactGenerators Ready |
 | `KWOK_FLUX_ROOT_GRACE` | `30` s | Grace period for the outer Kustomization before deadline counting starts |
 
+The `flux-git` lane additionally honors `KWOK_GITEA_HOST_PORT`
+(default `3300`), `KWOK_GITEA_USER` (default `aicr`), and
+`KWOK_GITEA_PASSWORD` (default `aicr-kwok-ci`) — shared between
+`install-infra.sh` (Gitea install + admin bootstrap) and
+`validate-scheduling.sh` (`git push`). The password is a CI-only
+credential for the ephemeral in-cluster Gitea, not a secret.
+
 On a clean local Kind cluster `Synced+Healthy` lands in ~30 s; the
 300-second default exists to absorb CI variance. If a local run trips
 code 50 but the cluster is otherwise healthy, raise the relevant
@@ -369,8 +394,9 @@ When `kwok-test` fails, it uploads an artifact named
 
 - `<cluster>-resources.txt`, `<cluster>-nodes.txt`, `<cluster>-pods.txt`, `<cluster>-events.txt`
 - `<cluster>-argo-apps.yaml` plus the repo-server and application-controller logs (argocd lanes)
-- `<cluster>-flux-resources.yaml` (OCIRepositories, Kustomizations, HelmReleases, ArtifactGenerators, ExternalArtifacts) plus source-, kustomize-, and helm-controller logs (flux-oci lane)
+- `<cluster>-flux-resources.yaml` (OCIRepositories, GitRepositories, Kustomizations, HelmReleases, ArtifactGenerators, ExternalArtifacts) plus source-, kustomize-, and helm-controller logs (flux-* lanes)
 - `<cluster>-registry.log` — last 200 lines of the in-cluster `registry:2`
+- `<cluster>-gitea.log` — last 200 lines of the in-cluster Gitea (flux-git lane)
 
 Start with the repo-server log (Argo CD) or source-controller log
 (Flux) for OCI-pull failures. Application-controller / kustomize-controller
@@ -385,8 +411,9 @@ add a new value (say, `argocd-git`):
 1. Add a `case` branch in `kwok/scripts/validate-scheduling.sh`'s
    `resolve_argocd_root_app()` (or `resolve_flux_root_names()` for
    Flux-reconciled lanes), plus branches in `generate_bundle` and
-   `deploy_bundle`. Reuse the existing `argocd-oci` / `flux-oci`
-   branches as templates.
+   `deploy_bundle`. Reuse the existing `argocd-oci` / `flux-oci` /
+   `flux-git` branches as templates (`flux-git` is the closest model
+   for Git-source lanes — Gitea install, dual-view URLs, push-to-create).
 2. Extend the `DEPLOYER` allowlist in
    `kwok/scripts/run-all-recipes.sh` (`case "$DEPLOYER" in` in `main()`).
 3. Extend the `case "${DEPLOYER}"` branches in `install-infra.sh`'s
@@ -478,4 +505,5 @@ the pre-push gate is local.
 - [validator.md](validator.md) — validator engine, chainsaw checks, container-per-validator pattern
 - [CLAUDE.md](https://github.com/NVIDIA/aicr/blob/main/.claude/CLAUDE.md) — coding rules and anti-patterns table
 - [ADR-008](https://github.com/NVIDIA/aicr/blob/main/docs/design/008-kwok-deployer-matrix.md) — KWOK deployer matrix rationale
+- [ADR-010](https://github.com/NVIDIA/aicr/blob/main/docs/design/010-kwok-git-source-lanes.md) — Git-source lanes (Gitea, flux-git)
 - [kwok/README.md](https://github.com/NVIDIA/aicr/blob/main/kwok/README.md) — KWOK cluster setup and node profiles

docs/design/008-kwok-deployer-matrix.md

@@ -8,6 +8,12 @@ Implementation tracked under [#843](https://github.com/NVIDIA/aicr/issues/843).
 The matrix dimensions, in-cluster topology, OCI flow, and acceptance
 criteria described below are intent, not current behavior.
 
+Extended by [ADR-010](010-kwok-git-source-lanes.md) [#963](https://github.com/NVIDIA/aicr/issues/963),
+which adds Git-source lanes (`flux-git` via in-cluster Gitea) for the
+filesystem-bundle round-trip. The "OCI over Git source" rationale
+below scoped the *first* lanes to OCI; ADR-010 supplements rather
+than reverses it.
+
 ## Problem
 
 The KWOK CI workflow validates bundle deployment for the default