Skip to content

Commit 79bcdbb

Browse files
matthias-hornematthias-horne
authored
Enhance Makefile to easily deploy to a remote cluster (#1751)
* add step to build helm charts * push helm charts to remote registry * push docker images to remote registry * create Extension for remote * ignore build artifacts * apply extension manifest * add all-in-one step * add pullSecretRef for Helm charts * fix values and always pull images * add documentation * change yq quotes to work also on Linux
1 parent 76020d4 commit 79bcdbb

4 files changed

Lines changed: 156 additions & 27 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
**/dev
66
/bin
77
hack/tools/bin
8+
/remote
89

910
*.coverprofile
1011
*.html

‎Makefile‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,79 @@ docker-image-admission:
120120
.PHONY: docker-images
121121
docker-images: docker-image-provider docker-image-admission
122122

123+
.PHONY: docker-push-provider
124+
docker-push-provider: $(KUBECTL)
125+
$(eval REGISTRY_URL := $(shell $(KUBECTL) cluster-info | head -1 | grep -oP 'https://\K[^:]+' | sed 's/^api\./reg./'))
126+
@docker tag $(IMAGE_PREFIX)/$(NAME):$(VERSION) $(REGISTRY_URL)/$(NAME):$(VERSION)
127+
@docker push $(REGISTRY_URL)/$(NAME):$(VERSION)
128+
129+
.PHONY: docker-push-admission
130+
docker-push-admission: $(KUBECTL)
131+
$(eval REGISTRY_URL := $(shell $(KUBECTL) cluster-info | head -1 | grep -oP 'https://\K[^:]+' | sed 's/^api\./reg./'))
132+
@docker tag $(IMAGE_PREFIX)/$(ADMISSION_NAME):$(VERSION) $(REGISTRY_URL)/$(ADMISSION_NAME):$(VERSION)
133+
@docker push $(REGISTRY_URL)/$(ADMISSION_NAME):$(VERSION)
134+
135+
.PHONY: docker-push
136+
docker-push: docker-push-provider docker-push-admission
137+
138+
.PHONY: helm-chart-provider
139+
helm-chart-provider: $(HELM)
140+
@mkdir -p remote
141+
@$(HELM) package ./charts/$(EXTENSION_PREFIX)-$(NAME) --version $(VERSION) --app-version $(VERSION) --destination remote
142+
143+
.PHONY: helm-chart-admission
144+
helm-chart-admission: $(HELM)
145+
@mkdir -p remote
146+
@$(HELM) package ./charts/$(EXTENSION_PREFIX)-$(ADMISSION_NAME)/charts/application --version $(VERSION) --app-version $(VERSION) --destination remote
147+
@$(HELM) package ./charts/$(EXTENSION_PREFIX)-$(ADMISSION_NAME)/charts/runtime --version $(VERSION) --app-version $(VERSION) --destination remote
148+
149+
.PHONY: helm-charts
150+
helm-charts: helm-chart-provider helm-chart-admission
151+
152+
.PHONY: helm-push-provider
153+
helm-push-provider: $(HELM) $(KUBECTL)
154+
$(eval REGISTRY_URL := $(shell $(KUBECTL) cluster-info | head -1 | grep -oP 'https://\K[^:]+' | sed 's/^api\./reg./'))
155+
@$(HELM) push remote/$(EXTENSION_PREFIX)-$(NAME)-$(VERSION).tgz oci://$(REGISTRY_URL)
156+
157+
.PHONY: helm-push-admission
158+
helm-push-admission: $(HELM) $(KUBECTL)
159+
$(eval REGISTRY_URL := $(shell $(KUBECTL) cluster-info | head -1 | grep -oP 'https://\K[^:]+' | sed 's/^api\./reg./'))
160+
@$(HELM) push remote/$(ADMISSION_NAME)-application-$(VERSION).tgz oci://$(REGISTRY_URL)
161+
@$(HELM) push remote/$(ADMISSION_NAME)-runtime-$(VERSION).tgz oci://$(REGISTRY_URL)
162+
163+
.PHONY: helm-push
164+
helm-push: helm-push-provider helm-push-admission
165+
166+
.PHONY: extension-manifest
167+
extension-manifest: $(KUBECTL) $(YQ)
168+
$(eval REGISTRY_URL := $(shell $(KUBECTL) cluster-info | head -1 | grep -oP 'https://\K[^:]+' | sed 's/^api\./reg./'))
169+
@mkdir -p remote
170+
@$(YQ) eval ".spec.deployment.admission.runtimeCluster.helm.ociRepository.ref = \"$(REGISTRY_URL)/$(ADMISSION_NAME)-runtime:$(VERSION)\" | \
171+
.spec.deployment.admission.runtimeCluster.helm.ociRepository.pullSecretRef.name = \"gardener-images\" | \
172+
.spec.deployment.admission.virtualCluster.helm.ociRepository.ref = \"$(REGISTRY_URL)/$(ADMISSION_NAME)-application:$(VERSION)\" | \
173+
.spec.deployment.admission.virtualCluster.helm.ociRepository.pullSecretRef.name = \"gardener-images\" | \
174+
.spec.deployment.extension.helm.ociRepository.ref = \"$(REGISTRY_URL)/$(EXTENSION_PREFIX)-$(NAME):$(VERSION)\" | \
175+
.spec.deployment.extension.helm.ociRepository.pullSecretRef.name = \"gardener-images\" | \
176+
.spec.deployment.admission.values.image.repository = \"$(REGISTRY_URL)/$(ADMISSION_NAME)\" | \
177+
.spec.deployment.admission.values.image.tag = \"$(VERSION)\" | \
178+
.spec.deployment.admission.values.image.pullPolicy = \"Always\" | \
179+
.spec.deployment.extension.values.image.repository = \"$(REGISTRY_URL)/$(NAME)\" | \
180+
.spec.deployment.extension.values.image.tag = \"$(VERSION)\" | \
181+
.spec.deployment.extension.values.image.pullPolicy = \"Always\" | \
182+
.spec.deployment.extension.runtimeClusterValues.image.repository = \"$(REGISTRY_URL)/$(NAME)\" | \
183+
.spec.deployment.extension.runtimeClusterValues.image.tag = \"$(VERSION)\" | \
184+
.spec.deployment.extension.runtimeClusterValues.image.pullPolicy = \"Always\"" \
185+
example/extension.yaml > remote/extension.yaml
186+
@echo "Created remote/extension.yaml with registry $(REGISTRY_URL)"
187+
188+
.PHONY: extension-apply
189+
extension-apply: $(KUBECTL)
190+
@$(KUBECTL) apply -f remote/extension.yaml
191+
192+
.PHONY: deploy-remote
193+
deploy-remote: docker-images docker-push helm-charts helm-push extension-manifest extension-apply
194+
@echo "Successfully deployed extension to remote cluster"
195+
123196
#####################################################################
124197
# Rules for verification, formatting, linting, testing and cleaning #
125198
#####################################################################

‎docs/development/local-setup.md‎

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎docs/development/remote-setup.md‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# Remote Cluster Setup
2+
3+
This guide describes how to build and deploy the AWS provider extension to a remote Kubernetes cluster for development and testing.
4+
5+
## Prerequisites
6+
7+
A Gardener installation deployed remotely following the [Gardener Remote Setup Guide](https://gardener.cloud/docs/gardener/deployment/getting_started_remotely/)
8+
9+
**Important:** Ensure your `kubectl` context is set to the correct remote cluster before running any commands. The deployment process automatically detects the cluster's registry URL from the active kubeconfig.
10+
11+
## Building and Deploying
12+
13+
### Deploy Everything
14+
15+
To build Docker images, Helm charts, and deploy the extension in one command:
16+
17+
```bash
18+
make deploy-remote
19+
```
20+
21+
This will:
22+
1. Build the provider and admission Docker images
23+
2. Push images to the cluster's registry
24+
3. Package the Helm charts
25+
4. Push charts to the cluster's registry
26+
5. Generate the extension manifest with correct registry references
27+
6. Apply the extension to the cluster
28+
29+
### Individual Steps
30+
31+
You can also run individual steps:
32+
33+
```bash
34+
# Build Docker images
35+
make docker-images
36+
37+
# Push Docker images
38+
make docker-push
39+
40+
# Build Helm charts
41+
make helm-charts
42+
43+
# Push Helm charts
44+
make helm-push
45+
46+
# Generate extension manifest
47+
make extension-manifest
48+
49+
# Apply extension to cluster
50+
make extension-apply
51+
```
52+
53+
## Platform-Specific Notes
54+
55+
### ARM Macs (Apple Silicon)
56+
57+
When building on ARM Macs (M1/M2/M3), you must specify the target platform as `linux/amd64` since Kubernetes clusters typically run on AMD64 architecture:
58+
59+
```bash
60+
TARGET_PLATFORMS=linux/amd64 make deploy-remote
61+
```
62+
63+
Or for individual steps:
64+
65+
```bash
66+
TARGET_PLATFORMS=linux/amd64 make docker-images
67+
make docker-push helm-charts helm-push extension-manifest extension-apply
68+
```
69+
70+
## Generated Artifacts
71+
72+
All build artifacts are placed in the `remote/` directory (gitignored):
73+
- `remote/*.tgz` - Helm chart packages
74+
- `remote/extension.yaml` - Generated extension manifest with registry URLs
75+
76+
## Registry Configuration
77+
78+
The deployment automatically derives the registry URL from your cluster's API server:
79+
- API server: `api.example.shoot.dev.k8s-hana.ondemand.com`
80+
- Registry: `reg.example.shoot.dev.k8s-hana.ondemand.com`
81+
82+
The extension manifest is configured to pull images using the `gardener-images` secret.

0 commit comments

Comments
 (0)