Use private registry for CI tests.

thallgren · thallgren · commit f1efc5f5b44e · 2023-12-05T12:55:40.000+01:00
Signed-off-by: Thomas Hallgren &lt;thomas@datawire.io&gt;
diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml
@@ -3,6 +3,11 @@ on:
   pull_request_target:
     types:
       - labeled
+
+env:
+  A8R_AGENT_REGISTRY: 'localhost:5000'
+  KUBECONFIG: ${{ github.workspace }}/kubeconfig.yaml
+
 jobs:
   build-image:
     runs-on: ubuntu-latest
@@ -22,69 +27,35 @@ jobs:
         uses: actions/setup-go@v3
         with:
           go-version: stable
-      - name: Build image
-        run: |
-          make image-tar
-      - name: Upload image
-        uses: actions/upload-artifact@v3
-        with:
-          name: image
-          path: build-output/ambassador-agent-image.tar
-  test:
-    runs-on: ubuntu-latest
-    needs: build-image
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          ref: "${{ github.event.pull_request.head.sha }}"
-      - name: Get last run
-        uses: ./.github/actions/rerun-cache
-        id: get_last_run
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: stable
       - name: Setup kubectl
         uses: azure/setup-kubectl@v3
         id: kubectl
-        if: steps.get_last_run.outputs.passed != 'success'
-      - name: Download image
-        uses: actions/download-artifact@v3
-        if: steps.get_last_run.outputs.passed != 'success'
-        with:
-          name: image
-      - name: Prepare kluster
-        uses: ./.github/actions/prepare-kluster
-        if: steps.get_last_run.outputs.passed != 'success'
+      - name: Provision Kubeception cluster
         id: kluster
-        with:
-          token: ${{ secrets.DEV_AMBASSADOR_AGENT_KUBECEPTION_TOKEN }}
-          image-tar: ambassador-agent-image.tar
-          github-sha: ${{ github.event.pull_request.head.sha }}
+        uses: datawire/infra-actions/provision-cluster@v0.2.6
+        with:
+          distribution: Kubeception
+          version: 1.26
+          kubeconfig: ${{ env.KUBECONFIG }}
+          kubeceptionToken: ${{ secrets.DEV_AMBASSADOR_AGENT_KUBECEPTION_TOKEN }}
+          gkeCredentials: '{"project_id": "foo"}'
+      - name: Add a private image registry to the cluster
+        run: make private-registry
+      - name: Build image
+        id: build
+        run: |
+          make push-image
+          echo "version=$(cat build-output/version.txt)" >> $GITHUB_OUTPUT
       - name: Run tests
-        if: steps.get_last_run.outputs.passed != 'success'
         uses: nick-invision/retry@v2
+        env:
+          A8R_AGENT_VERSION: ${{ steps.build.outputs.version }}
+          AMBASSADOR_AGENT_DOCKER_IMAGE: localhost:5000/ambassador-agent:${{ steps.build.outputs.version }}
+          KAT_SERVER_DOCKER_IMAGE: docker.io/datawiredev/kat-server:3.0.1-0.20220817135951-2cb28ef4f415
         with:
           max_attempts: 3
           timeout_minutes: 15
           command: |
-            export DTEST_KUBECONFIG=${{ steps.kluster.outputs.kubeconfig }}
-            export KUBECONFIG=${{ steps.kluster.outputs.kubeconfig }}
-            export AMBASSADOR_AGENT_DOCKER_IMAGE=datawiredev/ambassador-agent:dev-latest
-            export KAT_SERVER_DOCKER_IMAGE=docker.io/datawiredev/kat-server:3.0.1-0.20220817135951-2cb28ef4f415
             make go-integration-test
       - name: Mark test as succesfull
         run: echo "::set-output name=run_result::success" > run_result
-      - name: Cleanup kluster
-        uses: ./.github/actions/cleanup-kluster
-        if: always()
-        with:
-          token: ${{ secrets.DEV_AMBASSADOR_AGENT_KUBECEPTION_TOKEN }}
-          github-sha: ${{ github.event.pull_request.head.sha }}
-
-
-
-
-      
diff --git a/Makefile b/Makefile
@@ -106,11 +106,13 @@ push-ximage: image
 	docker buildx build --build-arg A8R_AGENT_VERSION=$(A8R_AGENT_VERSION) --push --platform=linux/amd64,linux/arm64 --tag=$(IMAGE) .
 
 .PHONY: push-image
-push-image: image
+push-image:
 	if docker pull $(IMAGE); then \
 	  printf "Failure: Tag already exists\n"; \
 	  exit 1; \
 	fi
+	mkdir -p $(BUILDDIR)
+	echo $(IMAGE_VERSION) > $(BUILDDIR)/version.txt
 	docker build --build-arg A8R_AGENT_VERSION=$(A8R_AGENT_VERSION) --tag=$(IMAGE) .
 	docker push $(IMAGE)
 
@@ -130,10 +132,45 @@ go-unit-test:
 	go test ./cmd/... -race
 	go test ./pkg/... -race
 
+TOOLSDIR = tools
+
+tools/helm = $(TOOLSDIR)/bin/helm
+HELM_VERSION=$(shell go mod edit -json | jq -r '.Require[] | select (.Path == "helm.sh/helm/v3") | .Version')
+HELM_TGZ = https://get.helm.sh/helm-$(HELM_VERSION)-$(GOHOSTOS)-$(GOHOSTARCH).tar.gz
+$(BUILDDIR)/$(notdir $(HELM_TGZ)):
+	mkdir -p $(@D)
+	curl -sfL $(HELM_TGZ) -o $@
+
+%/helm: $(BUILDDIR)/$(notdir $(HELM_TGZ))
+	mkdir -p $(@D)
+	tar -C $(@D) -zxmf $< --strip-components=1 $(GOHOSTOS)-$(GOHOSTARCH)/helm
+
+# Ensure that the Helm repo index is up to date
+HELM_REPO_NAME ?= datawire
+HELM_REPO_URL ?= https://app.getambassador.io
+
+$(BUILDDIR)/.helm-update-ts: $(tools/helm)
+	mkdir -p $(BUILDDIR)
+	$(tools/helm) repo add --force-update $(HELM_REPO_NAME) $(HELM_REPO_URL)
+	$(tools/helm) repo update
+	touch $@
+
 .PHONY: apply
-apply: push-image
-	helm install ambassador-agent ./helm/ambassador-agent -n ambassador --set image.fullImageOverride=$(IMAGE) --set logLevel=DEBUG --set cloudConnectToken=$(APIKEY)
+apply: push-image $(tools/helm)
+	$(tools/helm) install ambassador-agent ./helm/ambassador-agent -n ambassador --set image.fullImageOverride=$(IMAGE) --set logLevel=DEBUG --set cloudConnectToken=$(APIKEY)
 
 .PHONY: delete
 delete:
-	helm delete ambassador-agent -n ambassador
+	$(tools/helm) delete ambassador-agent -n ambassador
+
+.PHONY: private-registry
+private-registry: $(tools/helm) ## (Test) Add a private docker registry to the current k8s cluster and make it available on localhost:5000.
+	mkdir -p $(BUILDDIR)
+	$(tools/helm) repo add twuni https://helm.twun.io
+	$(tools/helm) repo update
+	$(tools/helm) install --set image.tag=2.8.1,configData.storage.cache.blobdescriptor=inmemory,configData.storage.cache.blobdescriptorsize=10000 docker-registry twuni/docker-registry
+	kubectl apply -f build-aux/private-reg-proxy.yaml
+	kubectl rollout status -w daemonset/private-registry-proxy
+	sleep 5
+	kubectl wait --for=condition=ready pod --all
+	kubectl port-forward daemonset/private-registry-proxy 5000:5000 &
diff --git a/build-aux/private-reg-proxy.yaml b/build-aux/private-reg-proxy.yaml
@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: private-registry-proxy
+  labels:
+   app: private-registry-proxy
+spec:
+  selector:
+    matchLabels:
+      app: private-registry-proxy
+  template:
+    metadata:
+      labels:
+        app: private-registry-proxy
+    spec:
+      containers:
+        - name: tcp-proxy
+          image: quay.io/bentoml/proxy-to-service:v2
+          args:
+            - tcp
+            - "5000"
+            - docker-registry.default.svc.cluster.local
+          ports:
+            - containerPort: 5000
+              hostPort: 5000
+              name: tcp
+              protocol: TCP
