syz-cluster: facilitate automatic deployment

syz-cluster/Makefile

@@ -64,17 +64,44 @@ restart-spanner: build-db-mgmt
 	minikube addons enable cloud-spanner;
 	./run-local.sh db-mgmt migrate
 
-
-SUBSTITUTE_VARS := '$$IMAGE_PREFIX $$IMAGE_TAG $$SPANNER_DATABASE_URI $$BLOB_STORAGE_GCS_BUCKET $$WORKFLOW_ARTIFACTS_BUCKET'
+# It takes some time for K8S to set up Argo CRDs, so if we `kubectl apply` everything at once, it will not
+# work on the first attempt.
+# The following two targets facilitate CI/CD:
+# - k8s-config-argo installs Argo workflows.
+# - k8s-config-argo-wait awaits untill all CRDs are ready.
+k8s-config-argo:
+	@kubectl kustomize ./overlays/common/argo/
+
+k8s-config-argo-wait:
+	@kubectl wait --for condition=established --timeout=60s crd/cronworkflows.argoproj.io
+	@kubectl wait --for condition=established --timeout=60s crd/workflows.argoproj.io
+
+escape_sed_replacement = \
+    $(shell printf '%s' '$(subst ','\'',$(1))' | \
+    sed -e 's~\\~\\\\~g' \
+        -e 's~&~\\&~g' \
+        -e 's~\$$~\\$$~g' \
+        -e 's~/~\\/~g' \
+    )
+
+SED_EXPRESSIONS := \
+    -e "s~\$${IMAGE_PREFIX}~$(call escape_sed_replacement,${IMAGE_PREFIX})~g" \
+    -e "s~\$${IMAGE_TAG}~$(call escape_sed_replacement,${IMAGE_TAG})~g" \
+    -e "s~\$${SPANNER_DATABASE_URI}~$(call escape_sed_replacement,${SPANNER_DATABASE_URI})~g" \
+    -e "s~\$${WORKFLOW_ARTIFACTS_BUCKET}~$(call escape_sed_replacement,${WORKFLOW_ARTIFACTS_BUCKET})~g" \
+    -e "s~\$${BLOB_STORAGE_GCS_BUCKET}~$(call escape_sed_replacement,${BLOB_STORAGE_GCS_BUCKET})~g" \
 
 k8s-config-dev:
-	@kubectl kustomize ./overlays/minikube/ | IMAGE_PREFIX=${IMAGE_PREFIX} IMAGE_TAG=${IMAGE_TAG} envsubst ${SUBSTITUTE_VARS}
+	@kubectl kustomize ./overlays/minikube/ | sed $(SED_EXPRESSIONS)
 
 k8s-config-gke: ensure-spanner-database-uri-env ensure-blob-storage-env ensure-workflow-artifacts-bucket
-	@kubectl kustomize ./overlays/gke/ | IMAGE_PREFIX=${IMAGE_PREFIX} IMAGE_TAG=${IMAGE_TAG} envsubst ${SUBSTITUTE_VARS}
+	@kubectl kustomize ./overlays/gke/ | sed $(SED_EXPRESSIONS)
 
 migrate-job.yaml: ensure-spanner-database-uri-env
-	@cat db-mgmt/migrate-job.yaml | IMAGE_PREFIX=${IMAGE_PREFIX} IMAGE_TAG=${IMAGE_TAG} envsubst
+	@cat db-mgmt/migrate-job.yaml | sed $(SED_EXPRESSIONS)
+
+fetch-kernels-once.yaml:
+	@cat kernel-disk/fetch-kernels-once.yaml
 
 ensure-spanner-database-uri-env:
 	@if [ -z "${SPANNER_DATABASE_URI}" ]; then \

syz-cluster/README.md

@@ -21,11 +21,13 @@ $ make build-all
 ```
 $ make restart-spanner
 $ kubectl create namespace argo
+$ make k8s-config-argo | kubectl apply -f -
+$ make k8s-config-argo-wait
 $ make k8s-config-dev | kubectl apply -f -
 ```
 5. (Optional) Pre-fetch the kernel git repository:
 ```
-$ argo submit --from cronwf/fetch-kernels-cron
+$ make fetch-kernels-once.yaml | kubectl create -f -
 ```
 
 ## Developmental tips

syz-cluster/controller/service.yaml

@@ -12,3 +12,4 @@ spec:
   - protocol: TCP
     port: 8080
     targetPort: 8080
+  type: ClusterIP

syz-cluster/dashboard/service.yaml

@@ -12,4 +12,4 @@ spec:
   - protocol: TCP
     port: 80
     targetPort: 8081
-  type: LoadBalancer
+  type: ClusterIP

syz-cluster/kernel-disk/fetch-kernels-cron.yaml

@@ -11,65 +11,5 @@ spec:
   concurrencyPolicy: "Replace"
   startingDeadlineSeconds: 0
   workflowSpec:
-    entrypoint: main
-    podMetadata:
-      labels:
-        tier: workflow
-    serviceAccountName: argo-executor-ksa
-    templates:
-    - name: main
-      parallelism: 1
-      steps:
-        - - name: query-trees
-            template: query-trees-template
-        - - name: iterate-trees
-            template: process-tree
-            arguments:
-              parameters:
-                - name: tree
-                  value: "{{item}}"
-            withParam: "{{=jsonpath(steps['query-trees'].outputs.result, '$.trees')}}"
-            continueOn:
-              failed: true
-    - name: query-trees-template
-      http:
-        url: "http://controller-service:8080/trees"
-        method: "GET"
-    - name: process-tree
-      inputs:
-        parameters:
-          - name: tree
-      volumes:
-        - name: git-repo
-          persistentVolumeClaim:
-            claimName: base-kernel-repo-pv-claim
-      container:
-        image: alpine/git:latest
-        imagePullPolicy: IfNotPresent
-        volumeMounts:
-          - name: git-repo
-            mountPath: /repo.git
-        resources:
-          requests:
-            cpu: 4
-            memory: 8G
-          limits:
-            cpu: 8
-            memory: 16G
-        command:
-          - "/bin/sh"
-          - "-c"
-          - |
-            cd /repo.git
-            if [ ! -d "refs" ]; then
-              git init --bare
-            fi
-            NAME="{{=jsonpath(inputs.parameters.tree, '$.name')}}"
-            REPO="{{=jsonpath(inputs.parameters.tree, '$.URL')}}"
-            BRANCH="{{=jsonpath(inputs.parameters.tree, '$.branch')}}"
-            echo "${NAME}: ${REPO}/${BRANCH}"
-            if ! git config --get remote.${NAME}.url > /dev/null; then
-              git remote add ${NAME} ${REPO}
-            fi
-            git fetch ${NAME} ${BRANCH} --tags
-            git tag -f ${NAME}-head ${NAME}/${BRANCH}
+    workflowTemplateRef:
+      name: fetch-kernels-workflow-template

syz-cluster/kernel-disk/fetch-kernels-once.yaml

@@ -0,0 +1,10 @@
+# Copyright 2025 syzkaller project authors. All rights reserved.
+# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: fetch-kernels-manual-
+spec:
+  workflowTemplateRef:
+    name: fetch-kernels-workflow-template

syz-cluster/kernel-disk/fetch-kernels-template.yaml

@@ -0,0 +1,70 @@
+# Copyright 2025 syzkaller project authors. All rights reserved.
+# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: fetch-kernels-workflow-template
+spec:
+  entrypoint: main
+  podMetadata:
+    labels:
+      tier: workflow
+  serviceAccountName: argo-executor-ksa
+  templates:
+    - name: main
+      parallelism: 1
+      steps:
+        - - name: query-trees
+            template: query-trees-template
+        - - name: iterate-trees
+            template: process-tree
+            arguments:
+              parameters:
+                - name: tree
+                  value: "{{item}}"
+            withParam: "{{=jsonpath(steps['query-trees'].outputs.result, '$.trees')}}"
+            continueOn:
+              failed: true
+    - name: query-trees-template
+      http:
+        url: "http://controller-service:8080/trees"
+        method: "GET"
+    - name: process-tree
+      inputs:
+        parameters:
+          - name: tree
+      volumes:
+        - name: git-repo
+          persistentVolumeClaim:
+            claimName: base-kernel-repo-pv-claim
+      container:
+        image: alpine/git:latest
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+          - name: git-repo
+            mountPath: /repo.git
+        resources:
+          requests:
+            cpu: 4
+            memory: 8G
+          limits:
+            cpu: 8
+            memory: 16G
+        command:
+          - "/bin/sh"
+          - "-c"
+          - |
+            cd /repo.git
+            if [ ! -d "refs" ]; then
+              git init --bare
+            fi
+            NAME="{{=jsonpath(inputs.parameters.tree, '$.name')}}"
+            REPO="{{=jsonpath(inputs.parameters.tree, '$.URL')}}"
+            BRANCH="{{=jsonpath(inputs.parameters.tree, '$.branch')}}"
+            echo "${NAME}: ${REPO}/${BRANCH}"
+            if ! git config --get remote.${NAME}.url > /dev/null; then
+              git remote add ${NAME} ${REPO}
+            fi
+            git fetch ${NAME} ${BRANCH} --tags
+            git tag -f ${NAME}-head ${NAME}/${BRANCH}

syz-cluster/kernel-disk/kustomization.yaml

@@ -2,4 +2,5 @@
 # Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 
 resources:
+  - fetch-kernels-template.yaml
   - fetch-kernels-cron.yaml

syz-cluster/overlays/common/argo/kustomization.yaml

@@ -0,0 +1,10 @@
+# Copyright 2025 syzkaller project authors. All rights reserved.
+# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+resources:
+  - https://github.com/argoproj/argo-workflows/releases/download/v3.6.2/install.yaml
+  - workflow-roles.yaml
+
+patchesStrategicMerge:
+  - patch-argo-controller.yaml
+  - patch-workflow-controller-configmap.yaml

syz-cluster/overlays/common/kustomization.yaml

@@ -2,7 +2,6 @@
 # Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 
 resources:
-  - https://github.com/argoproj/argo-workflows/releases/download/v3.6.2/install.yaml
   - ../../controller
   - ../../dashboard
   - ../../series-tracker
@@ -13,11 +12,6 @@ resources:
   - network-policy-controller.yaml
   - network-policy-git-access.yaml
   - network-policy-web-dashboard.yaml
-  - workflow-roles.yaml
-
-patchesStrategicMerge:
-  - patch-argo-controller.yaml
-  - patch-workflow-controller-configmap.yaml
 
 patches:
   - target:

syz-cluster/overlays/gke/global-config.yaml

@@ -11,7 +11,3 @@ data:
     loreArchives:
       - netdev
       - linux-ext4
-    emailReporting:
-      name: test-name
-      docs: http://docs/
-      supportEmail: name@email.com

syz-cluster/overlays/gke/kustomization.yaml

@@ -31,3 +31,11 @@ patches:
         path: /spec/templates/0/nodeSelector
         value:
           cloud.google.com/gke-nodepool: nested-vm-pool
+  - target:
+      kind: Service
+      name: web-dashboard-service
+    patch: |-
+      - op: replace
+        path: /metadata/annotations
+        value:
+          cloud.google.com/neg: '{"exposed_ports": {"80":{"name": "web-dashboard"}}}'

syz-cluster/overlays/minikube/kustomization.yaml

@@ -4,6 +4,7 @@
 resources:
   - service-accounts.yaml
   - kernel-disk-pvc.yaml
+  - ../common/argo
   - ../common
   - global-config.yaml
   - global-config-env.yaml

syz-cluster/reporter-server/service.yaml

@@ -12,3 +12,4 @@ spec:
   - protocol: TCP
     port: 8080
     targetPort: 8080
+  type: ClusterIP