Skip to content

Commit 4f33856

Browse files
committed
fix: replicate BRRTRouter's Kind registry setup verbatim
- Update kind-config.yaml to mirror localhost:5001 to kind-registry:5001 - Update registry startup to use -p 5001:5001 with REGISTRY_HTTP_ADDR - Simplify Tiltfile to use localhost:5001 everywhere (containerd handles mirroring) - Remove REGISTRY_CLUSTER and use single registry reference pattern - Remove separate containerd configuration step (handled by kind-config.yaml) This matches BRRTRouter's working implementation exactly.
1 parent eaa9eb6 commit 4f33856

3 files changed

Lines changed: 40 additions & 126 deletions

File tree

.github/workflows/tilt-ci.yaml

Lines changed: 5 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -89,48 +89,12 @@ jobs:
8989
9090
- name: Start local registry for Kind (localhost:5001)
9191
run: |
92-
# Use BRRTRouter's approach: always remove and recreate to prevent cached images
93-
# This ensures a clean registry for each CI run
94-
# Using port 5001 to avoid conflict with macOS AirPlay Receiver (port 5000)
95-
echo "Checking for existing kind-registry container..."
96-
if docker ps -a --format '{{.Names}}' | grep -q '^kind-registry$'; then
97-
echo "Removing existing kind-registry container..."
98-
docker rm -f kind-registry || true
99-
# Wait for port to be released
100-
echo "Waiting for port 5001 to be released..."
101-
sleep 3
92+
docker ps -a
93+
RUNNING=$(docker inspect -f '{{.State.Running}}' kind-registry 2>/dev/null || echo "false")
94+
if [ "$RUNNING" != "true" ]; then
95+
docker rm -f kind-registry 2>/dev/null || true
96+
docker run -d --restart=always -p 5001:5001 -e REGISTRY_HTTP_ADDR=0.0.0.0:5001 --name kind-registry registry:2
10297
fi
103-
104-
# Try to start registry
105-
# Docker will fail with a clear error if port is unavailable
106-
echo "Starting registry on port 5001..."
107-
if ! docker run -d --restart=always -p 5001:5001 -e REGISTRY_HTTP_ADDR=0.0.0.0:5001 --name kind-registry registry:2; then
108-
echo "❌ Failed to start registry on port 5001"
109-
echo "Checking for containers using port 5001:"
110-
docker ps --format "table {{.Names}}\t{{.Ports}}" | grep 5001 || echo "No Docker containers found using port 5001"
111-
echo ""
112-
echo "Possible solutions:"
113-
echo " 1. Stop any service using port 5001"
114-
exit 1
115-
fi
116-
117-
# Verify registry is running and accessible
118-
echo "Verifying registry is running..."
119-
sleep 2
120-
if ! docker ps --format '{{.Names}}' | grep -q '^kind-registry$'; then
121-
echo "❌ Registry container started but is not running"
122-
echo "Container logs:"
123-
docker logs kind-registry || true
124-
exit 1
125-
fi
126-
127-
# Test registry accessibility
128-
if ! curl -s http://127.0.0.1:5001/v2/ >/dev/null 2>&1; then
129-
echo "⚠️ Warning: Registry started but may not be accessible yet"
130-
echo "This is often normal - registry may need a moment to initialize"
131-
fi
132-
133-
echo "✅ Registry 'kind-registry' created and running on port 5001 (fresh, no cached images)"
13498
13599
- name: Create Kind cluster (with local registry mirror)
136100
run: |
@@ -157,32 +121,10 @@ jobs:
157121
158122
- name: Connect registry to kind network
159123
run: |
160-
# Use BRRTRouter's approach: simple connect (idempotent)
161124
docker network connect kind kind-registry 2>/dev/null || true
162125
kubectl get nodes
163126
kubectl get pods -A || true
164127
165-
- name: Configure containerd registry mirror
166-
env:
167-
REGISTRY_NAME: kind-registry
168-
run: python3 scripts/configure_containerd_registry.py
169-
170-
- name: Verify registry accessibility from Kind nodes
171-
run: |
172-
# Simple verification: test registry API from a Kind node
173-
NODE_NAME=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
174-
if [ -z "$NODE_NAME" ]; then
175-
echo "⚠️ No Kind nodes found (cluster may still be starting)"
176-
exit 0
177-
fi
178-
179-
# Test registry API via container name (Docker DNS resolves kind-registry)
180-
if docker exec $NODE_NAME sh -c "curl -s --connect-timeout 5 http://kind-registry:5001/v2/" 2>&1 | grep -q "{}"; then
181-
echo "✅ Registry is accessible from Kind nodes"
182-
else
183-
echo "⚠️ Registry may not be immediately accessible (containerd config will handle it)"
184-
fi
185-
186128
- name: Verify cluster
187129
run: |
188130
echo "=== Cluster Info ==="

Tiltfile

Lines changed: 11 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,9 @@
3232
allow_k8s_contexts(['kind-kube-startup-cpu-boost'])
3333

3434
# Configure default registry for Kind cluster
35-
# Explicitly set registry to avoid auto-detection from ConfigMap
36-
# The registry is set up by scripts/setup_kind.py as 'kind-registry' on localhost:5001
37-
# host_from_cluster uses the container name 'kind-registry:5001' which Docker DNS resolves
38-
# Note: Both host and container use port 5001 to avoid conflict with macOS AirPlay Receiver (port 5000)
39-
default_registry(
40-
'localhost:5001', # Registry host as seen from local machine (port 5001)
41-
host_from_cluster='kind-registry:5001' # Registry host as seen from within Kind cluster (container port 5001)
42-
)
35+
# Containerd mirrors localhost:5001 to kind-registry:5000 automatically
36+
# We use localhost:5001 everywhere - containerd handles the translation
37+
default_registry('localhost:5001')
4338

4439
# Suppress warning for custom_build image that uses full registry path
4540
# We use custom_build with explicit registry path, so Tilt won't find the image name in manifests
@@ -115,15 +110,13 @@ local_resource(
115110
BINARY_PATH = 'bin/manager'
116111
IMAGE_NAME = 'kube-startup-cpu-boost'
117112
# Registry as seen from host (for docker push)
113+
# Containerd will mirror localhost:5001 to kind-registry:5000 automatically
118114
REGISTRY_HOST = 'localhost:5001'
119-
# Registry as seen from inside Kind cluster (for Kubernetes image pull)
120-
REGISTRY_CLUSTER = 'kind-registry:5001'
121-
FULL_IMAGE_NAME_HOST = '%s/%s' % (REGISTRY_HOST, IMAGE_NAME)
122-
FULL_IMAGE_NAME_CLUSTER = '%s/%s' % (REGISTRY_CLUSTER, IMAGE_NAME)
115+
FULL_IMAGE_NAME = '%s/%s' % (REGISTRY_HOST, IMAGE_NAME)
123116
# Use unique tag per build to prevent Kubernetes from using cached images
124117
# Git hash ensures each build has a unique image tag
125118
IMAGE_TAG = 'tilt-' + str(local('git rev-parse --short HEAD 2>/dev/null || echo unknown')).strip()
126-
IMAGE_REF = '%s:%s' % (FULL_IMAGE_NAME_HOST, IMAGE_TAG)
119+
IMAGE_REF = '%s:%s' % (FULL_IMAGE_NAME, IMAGE_TAG)
127120

128121
# Build and push Docker image using local_resource
129122
# This explicitly builds and pushes the image, ensuring it's available before deployment
@@ -207,7 +200,7 @@ custom_build(
207200
# which matches the kustomize-to-helm conversion that helmify performs.
208201
# The chart structure is compatible with both helmify generation and manual helm template usage.
209202
k8s_yaml(
210-
local('helm template kube-startup-cpu-boost %s/charts/kube-startup-cpu-boost --namespace kube-startup-cpu-boost-system --set controllerManager.manager.image.repository=%s --set controllerManager.manager.image.tag=%s' % (PROJECT_DIR, FULL_IMAGE_NAME_CLUSTER, IMAGE_TAG))
203+
local('helm template kube-startup-cpu-boost %s/charts/kube-startup-cpu-boost --namespace kube-startup-cpu-boost-system --set controllerManager.manager.image.repository=%s --set controllerManager.manager.image.tag=%s' % (PROJECT_DIR, FULL_IMAGE_NAME, IMAGE_TAG))
211204
)
212205

213206
# Configure controller manager resource
@@ -256,11 +249,10 @@ local_resource(
256249
# Build Docker image for the demo Java app
257250
# Uses the Dockerfile in demo-app directory
258251
DEMO_APP_IMAGE_NAME = 'spring-demo-app'
259-
DEMO_APP_FULL_IMAGE_NAME_HOST = '%s/%s' % (REGISTRY_HOST, DEMO_APP_IMAGE_NAME)
260-
DEMO_APP_FULL_IMAGE_NAME_CLUSTER = '%s/%s' % (REGISTRY_CLUSTER, DEMO_APP_IMAGE_NAME)
252+
DEMO_APP_FULL_IMAGE_NAME = '%s/%s' % (REGISTRY_HOST, DEMO_APP_IMAGE_NAME)
261253
# Use unique tag per build
262254
DEMO_APP_IMAGE_TAG = 'tilt-' + str(local('git rev-parse --short HEAD 2>/dev/null || echo unknown')).strip()
263-
DEMO_APP_IMAGE_REF = '%s:%s' % (DEMO_APP_FULL_IMAGE_NAME_HOST, DEMO_APP_IMAGE_TAG)
255+
DEMO_APP_IMAGE_REF = '%s:%s' % (DEMO_APP_FULL_IMAGE_NAME, DEMO_APP_IMAGE_TAG)
264256

265257
# Build and push Docker image for demo app
266258
local_resource(
@@ -287,7 +279,7 @@ local_resource(
287279
# Note: We don't add resource_deps here because custom_build doesn't support it
288280
# Instead, the k8s_resource for spring-demo-app has resource_deps that includes docker-build-demo-app
289281
custom_build(
290-
DEMO_APP_FULL_IMAGE_NAME_HOST,
282+
DEMO_APP_FULL_IMAGE_NAME,
291283
'docker tag %s $EXPECTED_REF && docker push $EXPECTED_REF' % DEMO_APP_IMAGE_REF,
292284
deps=[], # No file deps - image is already built by docker-build-demo-app
293285
tag=DEMO_APP_IMAGE_TAG,
@@ -315,7 +307,7 @@ custom_build(
315307
# deployment waits for docker-build-demo-app, even if YAML is generated early
316308
# The image tag is computed at Tiltfile load time, so it will be consistent
317309
k8s_yaml(
318-
local('kubectl kustomize %s/demo-app | python3 -c \'import sys, yaml; docs = list(yaml.safe_load_all(sys.stdin)); filtered = [d for d in docs if d.get("kind") != "StartupCPUBoost"]; print(yaml.dump_all(filtered, default_flow_style=False))\' | sed "s|ghcr.io/google/spring-demo-app:latest|%s:%s|g"' % (PROJECT_DIR, DEMO_APP_FULL_IMAGE_NAME_CLUSTER, DEMO_APP_IMAGE_TAG)),
310+
local('kubectl kustomize %s/demo-app | python3 -c \'import sys, yaml; docs = list(yaml.safe_load_all(sys.stdin)); filtered = [d for d in docs if d.get("kind") != "StartupCPUBoost"]; print(yaml.dump_all(filtered, default_flow_style=False))\' | sed "s|ghcr.io/google/spring-demo-app:latest|%s:%s|g"' % (PROJECT_DIR, DEMO_APP_FULL_IMAGE_NAME, DEMO_APP_IMAGE_TAG)),
319311
)
320312

321313
# Wait for webhook service to be ready

kind-config.yaml

Lines changed: 24 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,32 @@
1-
# Copyright 2023 Google LLC
2-
#
3-
# Licensed under the Apache License, Version 2.0 (the "License");
4-
# you may not use this file except in compliance with the License.
5-
# You may obtain a copy of the License at
6-
#
7-
# https://www.apache.org/licenses/LICENSE-2.0
8-
#
9-
# Unless required by applicable law or agreed to in writing, software
10-
# distributed under the License is distributed on an "AS IS" BASIS,
11-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12-
# See the License for the specific language governing permissions and
13-
# limitations under the License.
14-
1+
# kind cluster configuration for kube-startup-cpu-boost local development
2+
# Creates a single-node cluster with port mappings for services
153
kind: Cluster
164
apiVersion: kind.x-k8s.io/v1alpha4
175
name: kube-startup-cpu-boost
18-
# Configure containerd to use local registry mirror
19-
# This maps localhost:5001 to kind-registry:5001 (container name, Docker DNS resolves it)
20-
# Note: Both host and container use port 5001 to avoid conflict with macOS AirPlay Receiver (port 5000)
6+
# Use stable Kubernetes version (v1.33.1 may have issues on some systems)
7+
# Uncomment to use a specific version:
8+
# nodes:
9+
# - role: control-plane
10+
# image: kindest/node:v1.29.2
11+
# Enable local registry support for project images
12+
# See: https://kind.sigs.k8s.io/docs/user/local-registry/
2113
containerdConfigPatches:
2214
- |-
2315
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5001"]
2416
endpoint = ["http://kind-registry:5001"]
25-
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."kind-registry:5001"]
26-
endpoint = ["http://kind-registry:5001"]
27-
[plugins."io.containerd.grpc.v1.cri".registry.configs."kind-registry:5001"]
28-
[plugins."io.containerd.grpc.v1.cri".registry.configs."kind-registry:5001".tls]
29-
insecure_skip_verify = true
30-
[plugins."io.containerd.grpc.v1.cri".registry.configs."localhost:5001"]
31-
[plugins."io.containerd.grpc.v1.cri".registry.configs."localhost:5001".tls]
32-
insecure_skip_verify = true
3317
nodes:
34-
- role: control-plane
35-
image: kindest/node:v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a
36-
kubeadmConfigPatches:
37-
# Explicitly use v1beta3 for Kubernetes 1.34.3 (v1beta4 expected for 1.36+)
38-
- |
39-
kind: InitConfiguration
40-
apiVersion: kubeadm.k8s.io/v1beta3
41-
nodeRegistration:
42-
kubeletExtraArgs:
43-
node-labels: "ingress-ready=true"
44-
networking:
45-
# Use unique pod and service subnets to avoid conflicts with other Kind clusters
46-
# Allocation: kube-startup-cpu-boost cluster
47-
# - Pod Subnet: 10.206.0.0/16 (unique, avoids default 10.244.0.0/16, dcops 10.204.0.0/16)
48-
# - Service Subnet: 10.207.0.0/16 (unique, avoids default 10.96.0.0/12, dcops 10.205.0.0/16)
49-
podSubnet: "10.206.0.0/16"
50-
serviceSubnet: "10.207.0.0/16"
51-
featureGates:
52-
InPlacePodVerticalScaling: true
18+
- role: control-plane
19+
# Resource limits for local development
20+
kubeadmConfigPatches:
21+
- |
22+
kind: ClusterConfiguration
23+
apiVersion: kubeadm.k8s.io/v1beta3
24+
apiServer:
25+
extraArgs:
26+
# Increase API server timeout for debugging
27+
request-timeout: "300s"
28+
controllerManager:
29+
extraArgs:
30+
# Faster reconciliation for development
31+
node-monitor-period: "2s"
32+
node-monitor-grace-period: "16s"

0 commit comments

Comments
 (0)