WIP: Local kind registry

timflannagan · timflannagan · commit b46b43c2cc11 · 2025-10-31T15:02:04.000-04:00
Signed-off-by: timflannagan &lt;timflannagan@gmail.com&gt;
diff --git a/.github/actions/kube-agent-gateway-conformance-tests/action.yaml b/.github/actions/kube-agent-gateway-conformance-tests/action.yaml
@@ -39,21 +39,9 @@ runs:
   - name: Install kgateway via helm
     shell: bash
     run: |
-      # Check if additional Helm values file is provided
-      ADDITIONAL_VALUES=""
-      if [[ -n "${HELM_ADDITIONAL_VALUES}" && -f "${HELM_ADDITIONAL_VALUES}" ]]; then
-        ADDITIONAL_VALUES="-f ${HELM_ADDITIONAL_VALUES}"
-        echo "Using additional Helm values from: ${HELM_ADDITIONAL_VALUES}"
-      fi
-      
       if [[ -z "${{ matrix.version }}" ]]; then
-        # If matrix.version is empty, use the local chart path specified in the Makefile.
-          helm upgrade -i -n kgateway-system kgateway-crds ./install/helm/kgateway-crds/ \
-          --create-namespace
-        helm upgrade -i -n kgateway-system kgateway ./install/helm/kgateway/ \
-          --create-namespace \
-          --set image.tag=${VERSION} --set image.registry=ghcr.io/kgateway-dev \
-          ${ADDITIONAL_VALUES}
+        # If matrix.version is empty, use the local chart path via Makefile target
+        make deploy-kgateway
       else
         # TODO(tim): this will require changes once the new helm chart is integrated
         # and published with the release pipeline.
diff --git a/.github/actions/kube-gateway-api-conformance-tests/action.yaml b/.github/actions/kube-gateway-api-conformance-tests/action.yaml
@@ -40,12 +40,8 @@ runs:
     shell: bash
     run: |
       if [[ -z "${{ matrix.version }}" ]]; then
-        # If matrix.version is empty, use the local chart path specified in the Makefile.
-          helm upgrade -i -n kgateway-system kgateway-crds ./install/helm/kgateway-crds/ \
-          --create-namespace
-        helm upgrade -i -n kgateway-system kgateway ./install/helm/kgateway/ \
-          --create-namespace \
-          --set image.tag=${VERSION} --set image.registry=ghcr.io/kgateway-dev
+        # If matrix.version is empty, use the local chart path via Makefile target
+        make deploy-kgateway
       else
         # TODO(tim): this will require changes once the new helm chart is integrated
         # and published with the release pipeline.
diff --git a/.github/actions/kube-gateway-api-load-tests/action.yaml b/.github/actions/kube-gateway-api-load-tests/action.yaml
@@ -21,17 +21,9 @@ runs:
     - name: Install KGateway via Helm
       shell: bash
       run: |
-        # Install using local chart paths (development/testing scenario)
-        go tool helm upgrade --install kgateway-crds ./install/helm/kgateway-crds/ \
-          --namespace kgateway-system --create-namespace \
-          --wait --timeout 5m
-        
-        go tool helm upgrade --install kgateway ./install/helm/kgateway/ \
-          --namespace kgateway-system --create-namespace \
-          --set image.tag=${VERSION:-1.0.0-ci1} \
-          --set image.registry=ghcr.io/kgateway-dev \
-          --set inferenceExtension.enabled=true \
-          --wait --timeout 5m
+        # TODO(tim): confirm is this needs to --set the image registry.
+        make deploy-kgateway \
+          HELM_ADDITIONAL_FLAGS="--set inferenceExtension.enabled=true --wait --timeout 5m"
 
     - name: Wait for KGateway deployment to be ready
       shell: bash
@@ -40,4 +32,4 @@ runs:
 
     - name: Run load tests
       shell: bash
-      run: make run-load-tests
+      run: make run-load-tests
diff --git a/.github/actions/kube-inference-extension-conformance-tests/action.yaml b/.github/actions/kube-inference-extension-conformance-tests/action.yaml
@@ -39,15 +39,8 @@ runs:
     shell: bash
     run: |
       if [[ -z "${{ matrix.version }}" ]]; then
-        # If matrix.version is empty, use the local chart path specified in the Makefile.
-          helm upgrade -i -n kgateway-system kgateway-crds ./install/helm/kgateway-crds/ \
-          --create-namespace \
-          --set inferenceExtension.enabled=true
-        helm upgrade -i -n kgateway-system kgateway ./install/helm/kgateway/ \
-          --create-namespace \
-          --set image.tag=${VERSION} \
-          --set image.registry=ghcr.io/kgateway-dev \
-          --set inferenceExtension.enabled=true
+        # If matrix.version is empty, use the local chart path via Makefile target
+        make deploy-kgateway HELM_ADDITIONAL_FLAGS="--set inferenceExtension.enabled=true"
       else
         # TODO(tim): this will require changes once the new helm chart is integrated
         # and published with the release pipeline.
diff --git a/Makefile b/Makefile
@@ -28,13 +28,14 @@ help: ## Output the self-documenting make targets
 ROOTDIR := $(shell pwd)
 OUTPUT_DIR ?= $(ROOTDIR)/_output
 
-export IMAGE_REGISTRY ?= ghcr.io/kgateway-dev
+# Image registry for container images. Defaults to localhost:5001 for local development with kind
+# Override via environment variable for releases (e.g., IMAGE_REGISTRY=ghcr.io/kgateway-dev)
+IMAGE_REGISTRY ?= localhost:5001
+export IMAGE_REGISTRY
 
 # Kind of a hack to make sure _output exists
 z := $(shell mkdir -p $(OUTPUT_DIR))
 
-BUILDX_BUILD ?= docker buildx build -q
-
 # A semver resembling 1.0.1-dev. Most calling GHA jobs customize this. Exported for use in goreleaser.yaml.
 VERSION ?= 1.0.1-dev
 export VERSION
@@ -62,7 +63,9 @@ else
 	endif
 endif
 
-PLATFORM := --platform=linux/$(GOARCH)
+BUILDX_BUILDER_NAME ?= kind-builder
+BUILDX_BUILD ?= docker buildx build -q
+BUILDX_BUILD_ARGS ?= --push --builder $(BUILDX_BUILDER_NAME) --platform=linux/$(GOARCH)
 
 GOOS ?= $(shell uname -s | tr '[:upper:]' '[:lower:]')
 
@@ -417,8 +420,9 @@ kgateway: $(CONTROLLER_OUTPUT_DIR)/kgateway-linux-$(GOARCH)
 $(CONTROLLER_OUTPUT_DIR)/Dockerfile: cmd/kgateway/Dockerfile
 	cp $< $@
 
+.PHONY: $(CONTROLLER_OUTPUT_DIR)/.docker-stamp-$(VERSION)-$(GOARCH)
 $(CONTROLLER_OUTPUT_DIR)/.docker-stamp-$(VERSION)-$(GOARCH): $(CONTROLLER_OUTPUT_DIR)/kgateway-linux-$(GOARCH) $(CONTROLLER_OUTPUT_DIR)/Dockerfile
-	$(BUILDX_BUILD) --load $(PLATFORM) $(CONTROLLER_OUTPUT_DIR) -f $(CONTROLLER_OUTPUT_DIR)/Dockerfile \
+	$(BUILDX_BUILD) $(BUILDX_BUILD_ARGS) $(CONTROLLER_OUTPUT_DIR) -f $(CONTROLLER_OUTPUT_DIR)/Dockerfile \
 		--build-arg GOARCH=$(GOARCH) \
 		--build-arg ENVOY_IMAGE=$(ENVOY_IMAGE) \
 		-t $(IMAGE_REGISTRY)/$(CONTROLLER_IMAGE_REPO):$(VERSION)
@@ -445,8 +449,9 @@ sds: $(SDS_OUTPUT_DIR)/sds-linux-$(GOARCH)
 $(SDS_OUTPUT_DIR)/Dockerfile.sds: cmd/sds/Dockerfile
 	cp $< $@
 
+.PHONY: $(SDS_OUTPUT_DIR)/.docker-stamp-$(VERSION)-$(GOARCH)
 $(SDS_OUTPUT_DIR)/.docker-stamp-$(VERSION)-$(GOARCH): $(SDS_OUTPUT_DIR)/sds-linux-$(GOARCH) $(SDS_OUTPUT_DIR)/Dockerfile.sds
-	$(BUILDX_BUILD) --load $(PLATFORM) $(SDS_OUTPUT_DIR) -f $(SDS_OUTPUT_DIR)/Dockerfile.sds \
+	$(BUILDX_BUILD) $(BUILDX_BUILD_ARGS) $(SDS_OUTPUT_DIR) -f $(SDS_OUTPUT_DIR)/Dockerfile.sds \
 		--build-arg GOARCH=$(GOARCH) \
 		--build-arg BASE_IMAGE=$(ALPINE_BASE_IMAGE) \
 		-t $(IMAGE_REGISTRY)/$(SDS_IMAGE_REPO):$(VERSION)
@@ -486,8 +491,9 @@ $(ENVOYINIT_OUTPUT_DIR)/Dockerfile.envoyinit: $(ENVOYINIT_DOCKERFILE) $(RUSTFORM
 $(ENVOYINIT_OUTPUT_DIR)/docker-entrypoint.sh: cmd/envoyinit/docker-entrypoint.sh
 	cp $< $@
 
+.PHONY: $(ENVOYINIT_OUTPUT_DIR)/.docker-stamp-$(VERSION)-$(GOARCH)
 $(ENVOYINIT_OUTPUT_DIR)/.docker-stamp-$(VERSION)-$(GOARCH): $(ENVOYINIT_OUTPUT_DIR)/envoyinit-linux-$(GOARCH) $(ENVOYINIT_OUTPUT_DIR)/Dockerfile.envoyinit $(ENVOYINIT_OUTPUT_DIR)/docker-entrypoint.sh
-	$(BUILDX_BUILD) --load $(PLATFORM) $(ENVOYINIT_OUTPUT_DIR) -f $(ENVOYINIT_OUTPUT_DIR)/Dockerfile.envoyinit \
+	$(BUILDX_BUILD) $(BUILDX_BUILD_ARGS) $(ENVOYINIT_OUTPUT_DIR) -f $(ENVOYINIT_OUTPUT_DIR)/Dockerfile.envoyinit \
 		--build-arg GOARCH=$(GOARCH) \
 		--build-arg ENVOY_IMAGE=$(ENVOY_IMAGE) \
 		--build-arg RUST_BUILD_ARCH=$(RUST_BUILD_ARCH) \
@@ -561,6 +567,8 @@ GORELEASER_ARGS ?= --snapshot --clean
 GORELEASER_TIMEOUT ?= 60m
 GORELEASER_CURRENT_TAG ?= $(VERSION)
 
+# When running locally, we need to set the IMAGE_REGISTRY to override the default
+# localhost:5001 local registry.
 .PHONY: release
 release: ## Create a release using goreleaser
 	GORELEASER_CURRENT_TAG=$(GORELEASER_CURRENT_TAG) $(GORELEASER) release $(GORELEASER_ARGS) --timeout $(GORELEASER_TIMEOUT)
@@ -577,9 +585,15 @@ INSTALL_NAMESPACE ?= kgateway-system
 # This version should stay in sync with `hack/kind/setup-kind.sh`.
 CLUSTER_NODE_VERSION ?= v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a
 
+.PHONY: kind-create-builder
+kind-create-builder: ## Create a buildx builder for pushing to local registry
+	@if ! docker buildx ls | grep -q $(BUILDX_BUILDER_NAME); then \
+		docker buildx create --name $(BUILDX_BUILDER_NAME) --driver docker-container --driver-opt network=host --bootstrap --use; \
+	fi
+
 .PHONY: kind-create
-kind-create: ## Create a KinD cluster
-	$(KIND) get clusters | grep $(CLUSTER_NAME) || $(KIND) create cluster --name $(CLUSTER_NAME) --image kindest/node:$(CLUSTER_NODE_VERSION)
+kind-create: ## Create a KinD cluster with a local registry
+	KIND_CLUSTER_NAME=$(CLUSTER_NAME) KIND_IMAGE_VERSION=$(CLUSTER_NODE_VERSION) ./hack/kind/setup-local-registry.sh
 
 CONFORMANCE_CHANNEL ?= experimental
 CONFORMANCE_VERSION ?= v1.4.0
@@ -606,10 +620,10 @@ deploy-kgateway: package-kgateway-charts deploy-kgateway-crd-chart deploy-kgatew
 setup-base: kind-create gw-api-crds gie-crds metallb ## Setup the base infrastructure (kind cluster, CRDs, and MetalLB)
 
 .PHONY: setup
-setup: setup-base kind-build-and-load package-kgateway-charts ## Setup the complete infrastructure (base setup plus images and charts)
+setup: setup-base docker-build package-kgateway-charts ## Setup the complete infrastructure
 
 .PHONY: run
-run: setup deploy-kgateway  ## Set up complete development environment
+run: setup deploy-kgateway ## Set up complete development environment
 
 .PHONY: undeploy
 undeploy: undeploy-kgateway undeploy-kgateway-crds ## Undeploy the application from the cluster
@@ -622,21 +636,19 @@ undeploy-kgateway: ## Undeploy the core chart from the cluster
 undeploy-kgateway-crds: ## Undeploy the CRD chart from the cluster
 	$(HELM) uninstall kgateway-crds --namespace $(INSTALL_NAMESPACE) || true
 
+.PHONY: docker-build ## Build and push all docker images to local registry
+docker-build: kind-create-builder
+docker-build: kgateway-docker
+docker-build: envoy-wrapper-docker
+docker-build: sds-docker
+
 #----------------------------------------------------------------------------------
 # Build assets for kubernetes e2e tests
 #----------------------------------------------------------------------------------
 
 kind-setup: ## Set up the KinD cluster. Deprecated: use kind-create instead.
 	VERSION=${VERSION} CLUSTER_NAME=${CLUSTER_NAME} ./hack/kind/setup-kind.sh
 
-kind-load-%:
-	$(KIND) load docker-image $(IMAGE_REGISTRY)/$*:$(VERSION) --name $(CLUSTER_NAME)
-
-# Build an image and load it into the KinD cluster
-# Depends on: IMAGE_REGISTRY, VERSION, CLUSTER_NAME
-# Envoy image may be specified via ENVOY_IMAGE on the command line or at the top of this file
-kind-build-and-load-%: %-docker kind-load-% ; ## Use to build specified image and load it into kind
-
 # Update the docker image used by a deployment
 # This works for most of our deployments because the deployment name and container name both match
 # NOTE TO DEVS:
@@ -649,24 +661,6 @@ kind-set-image-%:
 	kubectl patch deployment $* -n $(INSTALL_NAMESPACE) -p '{"spec": {"template":{"metadata":{"annotations":{"kgateway-kind-last-update":"$(shell date)"}}}} }'
 	kubectl rollout resume deployment $* -n $(INSTALL_NAMESPACE)
 
-# Reload an image in KinD
-# This is useful to developers when changing a single component
-# You can reload an image, which means it will be rebuilt and reloaded into the kind cluster, and the deployment
-# will be updated to reference it
-# Depends on: IMAGE_REGISTRY, VERSION, INSTALL_NAMESPACE , CLUSTER_NAME
-# Envoy image may be specified via ENVOY_IMAGE on the command line or at the top of this file
-kind-reload-%: kind-build-and-load-% kind-set-image-% ; ## Use to build specified image, load it into kind, and restart its deployment
-
-.PHONY: kind-build-and-load ## Use to build all images and load them into kind
-kind-build-and-load: kind-build-and-load-kgateway
-kind-build-and-load: kind-build-and-load-envoy-wrapper
-kind-build-and-load: kind-build-and-load-sds
-
-.PHONY: kind-load ## Use to load all images into kind
-kind-load: kind-load-kgateway
-kind-load: kind-load-envoy-wrapper
-kind-load: kind-load-sds
-
 #----------------------------------------------------------------------------------
 # Load Testing
 #----------------------------------------------------------------------------------
diff --git a/hack/helm/dev.yaml b/hack/helm/dev.yaml
@@ -8,3 +8,5 @@ controller:
     # rollingUpdate:
     #   maxSurge: 100%
     #   maxUnavailable: 100%
+image:
+  registry: localhost:5001
diff --git a/hack/kind/cluster.yaml b/hack/kind/cluster.yaml
@@ -1,13 +1,8 @@
-kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
-kubeadmConfigPatches:
-  - |
-    apiVersion: kubeadm.k8s.io/v1beta3
-    kind: ClusterConfiguration
-    metadata:
-      name: config
-  - |
-    apiVersion: kubeadm.k8s.io/v1beta3
-    kind: InitConfiguration
-    metadata:
-      name: config
+kind: Cluster
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry]
+    config_path = "/etc/containerd/certs.d"
+networking:
+  apiServerAddress: "127.0.0.1"
diff --git a/hack/kind/setup-kind.sh b/hack/kind/setup-kind.sh
@@ -42,11 +42,8 @@ function create_kind_cluster_or_skip() {
     return
   fi
 
-  echo "creating cluster ${CLUSTER_NAME}"
-  $KIND create cluster \
-    --name "$CLUSTER_NAME" \
-    --image "kindest/node:$CLUSTER_NODE_VERSION" \
-    --config="$SCRIPT_DIR/cluster.yaml"
+  echo "creating cluster ${CLUSTER_NAME} with local registry"
+  KIND_CLUSTER_NAME=$CLUSTER_NAME KIND_IMAGE_VERSION=$CLUSTER_NODE_VERSION $SCRIPT_DIR/setup-local-registry.sh
   echo "Finished setting up cluster $CLUSTER_NAME"
 
   # so that you can just build the kind image alone if needed
@@ -56,18 +53,15 @@ function create_kind_cluster_or_skip() {
   fi
 }
 
-# 1. Create a kind cluster (or skip creation if a cluster with name=CLUSTER_NAME already exists)
+# 1. Create a kind cluster with local registry (or skip creation if a cluster with name=CLUSTER_NAME already exists)
 # This config is roughly based on: https://kind.sigs.k8s.io/docs/user/ingress/
 create_kind_cluster_or_skip
 
 if [[ $SKIP_DOCKER == 'true' ]]; then
-  # TODO(tim): refactor the Makefile & CI scripts so we're loading local
-  # charts to real helm repos, and then we can remove this block.
   echo "SKIP_DOCKER=true, not building images or chart"
 else
-  # 2. Make all the docker images and load them to the kind cluster
-  VERSION=$VERSION CLUSTER_NAME=$CLUSTER_NAME make kind-build-and-load
-
+  # 2. Create buildx builder and build all docker images, pushing to local registry
+  VERSION=$VERSION make docker-build
   # 3. Build the test helm chart, ensuring we have a chart in the `_test` folder
   VERSION=$VERSION make package-kgateway-charts
 
diff --git a/hack/kind/setup-local-registry.sh b/hack/kind/setup-local-registry.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+
+KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:-kind}
+KIND_IMAGE_VERSION=${KIND_IMAGE_VERSION:-v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a}
+
+# 1. Create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \
+    registry:2
+fi
+
+# 2. Create kind cluster with containerd registry config dir enabled
+#
+# NOTE: the containerd config patch is not necessary with images from kind v0.27.0+
+# It may enable some older images to work similarly.
+# If you're only supporting newer releases, you can just use `kind create cluster` here.
+#
+# See:
+# https://github.com/kubernetes-sigs/kind/issues/2875
+# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration
+# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
+if kind get clusters | grep -qx "${KIND_CLUSTER_NAME}"; then
+  echo "Kind cluster '${KIND_CLUSTER_NAME}' already exists; skipping create."
+else
+  kind create cluster --name "${KIND_CLUSTER_NAME}" \
+    --config hack/kind/cluster.yaml \
+    --image="kindest/node:${KIND_IMAGE_VERSION}"
+fi
+
+echo "Adding registry config to nodes"
+# 3. Add the registry config to the nodes
+#
+# This is necessary because localhost resolves to loopback addresses that are
+# network-namespace local.
+# In other words: localhost in the container is not localhost on the host.
+#
+# We want a consistent name that works from both ends, so we tell containerd to
+# alias localhost:${reg_port} to the registry container when pulling images
+REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
+for node in $(kind get nodes --name "${KIND_CLUSTER_NAME}"); do
+  docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
+  cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
+[host."http://${reg_name}:5000"]
+EOF
+done
+
+echo "Connecting registry to cluster network"
+# 4. Connect the registry to the cluster network if not already connected
+# This allows kind to bootstrap the network but ensures they're on the same network
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+  docker network connect "kind" "${reg_name}"
+fi
+
+# 5. Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
