Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 54 additions & 23 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,22 @@ GCP_ONLY ?= false
GCP_PROJECT ?= k8s-skaffold
GKE_CLUSTER_NAME ?= integration-tests
GKE_ZONE ?= us-central1-a
GKE_REGION=us-central1

# Set registry/auth/cluster location based on GCP_PROJECT
ifeq ($(GCP_PROJECT),skaffold-ci-cd)
# Presubmit environment: skaffold-ci-cd project with Artifact Registry
Comment on lines +39 to +40
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The new logic for GCP_PROJECT=skaffold-ci-cd relies on AR_REGION and GKE_REGION variables. However, these variables are not defined with default values, nor are there any checks to ensure they are set. If they are not provided in the environment, they will be empty, leading to malformed values for IMAGE_REPO_BASE and GKE_LOCATION_FLAG, which can cause silent failures or hard-to-debug issues. It's safer to add checks to ensure these required variables are set.

ifeq ($(GCP_PROJECT),skaffold-ci-cd)
  ifndef AR_REGION
    $(error AR_REGION must be set when GCP_PROJECT is skaffold-ci-cd)
  endif
  ifndef GKE_REGION
    $(error GKE_REGION must be set when GCP_PROJECT is skaffold-ci-cd)
  endif
  # Presubmit environment: skaffold-ci-cd project with Artifact Registry

IMAGE_REPO_BASE := $(AR_REGION)-docker.pkg.dev/$(GCP_PROJECT)
GCLOUD_AUTH_CONFIG := $(AR_REGION)-docker.pkg.dev
GKE_LOCATION_FLAG := --region $(GKE_REGION)
$(info Using Artifact Registry config for project: $(GCP_PROJECT))
else
# k8s-skaffold project with GCR
IMAGE_REPO_BASE := gcr.io/$(GCP_PROJECT)
GCLOUD_AUTH_CONFIG := gcr.io
GKE_LOCATION_FLAG := --zone $(GKE_ZONE)
$(info Using GCR config for project: $(GCP_PROJECT))
endif

SUPPORTED_PLATFORMS = linux-amd64 darwin-amd64 windows-amd64.exe linux-arm64 darwin-arm64
BUILD_PACKAGE = $(REPOPATH)/v2/cmd/skaffold
Expand Down Expand Up @@ -142,9 +158,16 @@ integration-tests:
ifeq ($(GCP_ONLY),true)
gcloud container clusters get-credentials \
$(GKE_CLUSTER_NAME) \
--zone $(GKE_ZONE) \
$(GKE_LOCATION_FLAG) \
--project $(GCP_PROJECT)
gcloud auth configure-docker us-central1-docker.pkg.dev

# Conditional Docker authentication: ONLY when GCR is used
ifneq ($(GCP_PROJECT),skaffold-ci-cd)
@echo "Configuring Docker for GCR: $(GCLOUD_AUTH_CONFIG)"
gcloud auth configure-docker $(GCLOUD_AUTH_CONFIG) -q
else
@echo "Docker auth is handled in the build script for skaffold-ci-cd"
endif
endif
@ GCP_ONLY=$(GCP_ONLY) GKE_CLUSTER_NAME=$(GKE_CLUSTER_NAME) ./hack/gotest.sh -v $(REPOPATH)/v2/integration -timeout 50m $(INTEGRATION_TEST_ARGS)

Expand All @@ -157,17 +180,17 @@ release: $(BUILD_DIR)/VERSION
--build-arg VERSION=$(VERSION) \
-f deploy/skaffold/Dockerfile \
--target release \
-t gcr.io/$(GCP_PROJECT)/skaffold:$(VERSION) \
-t gcr.io/$(GCP_PROJECT)/skaffold:latest \
-t $(IMAGE_REPO_BASE)/skaffold:$(VERSION) \
-t $(IMAGE_REPO_BASE)/skaffold:latest \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

critical

This line has two issues:

  1. It uses spaces for indentation, while the surrounding lines and Makefile conventions use tabs. This is inconsistent.
  2. It is missing a trailing \ to continue the docker build command on the next line. This will cause the build to fail.

Please fix the indentation and add the line continuation character.

		-t $(IMAGE_REPO_BASE)/skaffold:latest \

.

.PHONY: release-build
release-build:
docker build \
-f deploy/skaffold/Dockerfile \
--target release \
-t gcr.io/$(GCP_PROJECT)/skaffold:edge \
-t gcr.io/$(GCP_PROJECT)/skaffold:$(COMMIT) \
-t $(IMAGE_REPO_BASE)/skaffold:edge \
-t $(IMAGE_REPO_BASE)/skaffold:$(COMMIT) \
.

.PHONY: release-lts
Expand All @@ -176,53 +199,58 @@ release-lts: $(BUILD_DIR)/VERSION
--build-arg VERSION=$(VERSION) \
-f deploy/skaffold/Dockerfile.lts \
--target release \
-t gcr.io/$(GCP_PROJECT)/skaffold:lts \
-t gcr.io/$(GCP_PROJECT)/skaffold:$(VERSION)-lts \
-t gcr.io/$(GCP_PROJECT)/skaffold:$(SCANNING_MARKER)-lts \
-t $(IMAGE_REPO_BASE)/skaffold:lts \
-t $(IMAGE_REPO_BASE)/skaffold:$(VERSION)-lts \
-t $(IMAGE_REPO_BASE)/skaffold:$(SCANNING_MARKER)-lts \
.

.PHONY: release-lts-build
release-lts-build:
docker build \
-f deploy/skaffold/Dockerfile.lts \
--target release \
-t gcr.io/$(GCP_PROJECT)/skaffold:edge-lts \
-t gcr.io/$(GCP_PROJECT)/skaffold:$(COMMIT)-lts \
-t $(IMAGE_REPO_BASE)/skaffold:edge-lts \
-t $(IMAGE_REPO_BASE)/skaffold:$(COMMIT)-lts \
.

.PHONY: clean
clean:
rm -rf $(BUILD_DIR) hack/bin $(EMBEDDED_FILES_CHECK) fs/assets/schemas_generated/

# Runs a script to calculate a hash/digest of the build dependencies. Store it
# in DEPS_DIGEST. Then push the dependency image to GCR/AR.
.PHONY: build_deps
build_deps:
$(eval DEPS_DIGEST := $(shell ./hack/skaffold-deps-sha1.sh))
docker build \
-f deploy/skaffold/Dockerfile.deps \
-t gcr.io/$(GCP_PROJECT)/build_deps:$(DEPS_DIGEST) \
-t $(IMAGE_REPO_BASE)/build_deps:$(DEPS_DIGEST) \
deploy/skaffold
docker push gcr.io/$(GCP_PROJECT)/build_deps:$(DEPS_DIGEST)
docker push $(IMAGE_REPO_BASE)/build_deps:$(DEPS_DIGEST)


skaffold-builder-ci:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

critical

The skaffold-builder-ci target uses the DEPS_DIGEST variable in a --cache-from flag, but DEPS_DIGEST is not defined within this target's scope. This will result in DEPS_DIGEST being empty, causing the cache to be missed or pointing to an incorrect image tag. You should define DEPS_DIGEST within the skaffold-builder-ci target.

skaffold-builder-ci:
	$(eval DEPS_DIGEST := $(shell ./hack/skaffold-deps-sha1.sh))

docker build \
--cache-from gcr.io/$(GCP_PROJECT)/build_deps \
--cache-from $(IMAGE_REPO_BASE)/build_deps:$(DEPS_DIGEST) \
-f deploy/skaffold/Dockerfile.deps \
-t gcr.io/$(GCP_PROJECT)/build_deps \
-t $(IMAGE_REPO_BASE)/build_deps \
.
time docker build \
-f deploy/skaffold/Dockerfile \
--target builder \
-t gcr.io/$(GCP_PROJECT)/skaffold-builder \
--cache-from $(IMAGE_REPO_BASE)/build_deps \
-t $(IMAGE_REPO_BASE)/skaffold-builder \
.

.PHONY: skaffold-builder
skaffold-builder:
time docker build \
-f deploy/skaffold/Dockerfile \
--target builder \
-t gcr.io/$(GCP_PROJECT)/skaffold-builder \
-t $(IMAGE_REPO_BASE)/skaffold-builder \
.

# Run integration tests within a local kind (Kubernetes IN Docker) cluster.
.PHONY: integration-in-kind
integration-in-kind: skaffold-builder
echo '{}' > /tmp/docker-config
Expand All @@ -237,7 +265,7 @@ integration-in-kind: skaffold-builder
-e INTEGRATION_TEST_ARGS=$(INTEGRATION_TEST_ARGS) \
-e IT_PARTITION=$(IT_PARTITION) \
--network kind \
gcr.io/$(GCP_PROJECT)/skaffold-builder \
$(IMAGE_REPO_BASE)/skaffold-builder \
sh -eu -c ' \
if ! kind get clusters | grep -q kind; then \
trap "kind delete cluster" 0 1 2 15; \
Expand All @@ -262,7 +290,7 @@ integration-in-k3d: skaffold-builder
-v $(CURDIR)/hack/maven/settings.xml:/root/.m2/settings.xml \
-e INTEGRATION_TEST_ARGS=$(INTEGRATION_TEST_ARGS) \
-e IT_PARTITION=$(IT_PARTITION) \
gcr.io/$(GCP_PROJECT)/skaffold-builder \
$(IMAGE_REPO_BASE)/skaffold-builder \
sh -eu -c ' \
if ! k3d cluster list | grep -q k3s-default; then \
trap "k3d cluster delete" 0 1 2 15; \
Expand All @@ -276,27 +304,30 @@ integration-in-k3d: skaffold-builder
make integration \
'

# The `gcloud auth configure-docker` below is needed this starts a separate
# container (us-central1-docker.pkg.dev/skaffold-ci-cd/skaffold-builder) to run
# the tests. This inner container doesn't inherit the Docker configuration from
# the host.
.PHONY: integration-in-docker
integration-in-docker: skaffold-builder-ci
docker run --rm \
-v /var/run/docker.sock:/var/run/docker.sock \
-v $(HOME)/.config/gcloud:/root/.config/gcloud \
-v $(GOOGLE_APPLICATION_CREDENTIALS):$(GOOGLE_APPLICATION_CREDENTIALS) \
-v $(CURDIR)/hack/maven/settings.xml:/root/.m2/settings.xml \
-e GCP_ONLY=$(GCP_ONLY) \
-e GCP_PROJECT=$(GCP_PROJECT) \
-e GKE_CLUSTER_NAME=$(GKE_CLUSTER_NAME) \
-e GKE_ZONE=$(GKE_ZONE) \
-e DOCKER_CONFIG=/root/.docker \
-e GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \
-e INTEGRATION_TEST_ARGS=$(INTEGRATION_TEST_ARGS) \
-e IT_PARTITION=$(IT_PARTITION) \
gcr.io/$(GCP_PROJECT)/skaffold-builder \
make integration-tests
$(IMAGE_REPO_BASE)/skaffold-builder \
sh -c "gcloud auth configure-docker us-central1-docker.pkg.dev -q && make integration-tests"

.PHONY: submit-build-trigger
submit-build-trigger:
gcloud builds submit . \
--project=$(GCP_PROJECT) \
--config=deploy/cloudbuild.yaml \
--substitutions="_RELEASE_BUCKET=$(RELEASE_BUCKET),COMMIT_SHA=$(COMMIT)"

Expand Down
28 changes: 14 additions & 14 deletions deploy/skaffold/Dockerfile.deps
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
ARG ARCH=amd64

# Download kubectl
FROM alpine:3.21.2 as download-kubectl
FROM mirror.gcr.io/library/alpine:3.21.2 as download-kubectl
ARG ARCH
# https://dl.k8s.io/release/stable.txt
ENV KUBECTL_VERSION v1.33.0
Expand All @@ -25,7 +25,7 @@ RUN wget -O kubectl "${KUBECTL_URL}" && sha256sum -c kubectl.${ARCH}.sha256
RUN chmod +x kubectl

# Download helm (see https://github.com/helm/helm/releases/latest)
FROM alpine:3.21.2 as download-helm
FROM mirror.gcr.io/library/alpine:3.21.2 as download-helm
ARG ARCH
RUN echo arch=$ARCH
ENV HELM_VERSION v3.17.3
Expand All @@ -35,7 +35,7 @@ RUN wget -O helm.tar.gz "${HELM_URL}" && sha256sum -c helm.${ARCH}.sha256
RUN tar -xvf helm.tar.gz --strip-components 1

# Download kustomize (see https://github.com/kubernetes-sigs/kustomize/releases/latest)
FROM alpine:3.21.2 as download-kustomize
FROM mirror.gcr.io/library/alpine:3.21.2 as download-kustomize
ARG ARCH
ENV KUSTOMIZE_VERSION 5.6.0
ENV KUSTOMIZE_URL https://storage.googleapis.com/skaffold/deps/kustomize/v${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_${ARCH}.tar.gz
Expand All @@ -44,7 +44,7 @@ RUN wget -O kustomize.tar.gz "${KUSTOMIZE_URL}" && sha256sum -c kustomize.${ARCH
RUN tar -xvf kustomize.tar.gz

# Download kpt (see https://github.com/kptdev/kpt/releases/latest)
FROM alpine:3.21.2 as download-kpt
FROM mirror.gcr.io/library/alpine:3.21.2 as download-kpt
ARG ARCH
ENV KPT_VERSION 1.0.0-beta.57
ENV KPT_URL https://storage.googleapis.com/skaffold/deps/kpt/v${KPT_VERSION}/kpt_linux_amd64
Expand All @@ -53,7 +53,7 @@ RUN wget -O kpt "${KPT_URL}" && sha256sum -c kpt.${ARCH}.sha256
RUN chmod +x kpt

# Download kompose (see https://github.com/kubernetes/kompose/releases/latest)
FROM alpine:3.21.2 as download-kompose
FROM mirror.gcr.io/library/alpine:3.21.2 as download-kompose
ARG ARCH
ENV KOMPOSE_VERSION v1.35.0
ENV KOMPOSE_URL https://storage.googleapis.com/skaffold/deps/kompose/${KOMPOSE_VERSION}/kompose-linux-amd64
Expand All @@ -62,7 +62,7 @@ RUN wget -O kompose "${KOMPOSE_URL}" && sha256sum -c kompose.${ARCH}.sha256
RUN chmod +x kompose

# Download container-structure-test (https://github.com/GoogleContainerTools/container-structure-test/releases/latest)
FROM alpine:3.21.2 as download-container-structure-test
FROM mirror.gcr.io/library/alpine:3.21.2 as download-container-structure-test
ARG ARCH
ENV CONTAINER_STRUCTURE_TEST_VERSION v1.19.3
ENV CONTAINER_STRUCTURE_TEST_URL https://storage.googleapis.com/skaffold/deps/container-structure-test/${CONTAINER_STRUCTURE_TEST_VERSION}/container-structure-test-linux-${ARCH}
Expand All @@ -71,7 +71,7 @@ RUN wget -O container-structure-test "${CONTAINER_STRUCTURE_TEST_URL}" && sha512
RUN chmod +x container-structure-test

# Download kind (see https://github.com/kubernetes-sigs/kind/releases/latest)
FROM alpine:3.21.2 as download-kind
FROM mirror.gcr.io/library/alpine:3.21.2 as download-kind
ARG ARCH
ENV KIND_VERSION v0.27.0
ENV KIND_URL https://storage.googleapis.com/skaffold/deps/kind/${KIND_VERSION}/kind-linux-${ARCH}
Expand All @@ -80,7 +80,7 @@ RUN wget -O kind "${KIND_URL}" && sha512sum -c kind.${ARCH}.sha512
RUN chmod +x kind

# Download k3d (see https://github.com/k3d-io/k3d/releases/latest)
FROM alpine:3.21.2 as download-k3d
FROM mirror.gcr.io/library/alpine:3.21.2 as download-k3d
ARG ARCH
ENV K3D_VERSION v5.8.3
ENV K3D_URL https://storage.googleapis.com/skaffold/deps/k3d/${K3D_VERSION}/k3d-linux-amd64
Expand All @@ -89,7 +89,7 @@ RUN wget -O k3d "${K3D_URL}" && sha256sum -c k3d.${ARCH}.sha256
RUN chmod +x k3d

# Download gcloud (see https://cloud.google.com/sdk/docs/release-notes)
FROM alpine:3.21.2 as download-gcloud
FROM mirror.gcr.io/library/alpine:3.21.2 as download-gcloud
ARG ARCH
ENV GCLOUD_VERSION 520.0.0
ENV GCLOUD_URL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_VERSION}-linux-GCLOUDARCH.tar.gz
Expand All @@ -102,7 +102,7 @@ RUN \
RUN tar -zxf gcloud.tar.gz

# Download bazel (see https://github.com/bazelbuild/bazel/releases/latest)
FROM alpine:3.21.2 as download-bazel
FROM mirror.gcr.io/library/alpine:3.21.2 as download-bazel
ARG ARCH
ENV BAZEL_VERSION 8.2.1
ENV BAZEL_URL https://storage.googleapis.com/skaffold/deps/bazel/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-BAZELARCH
Expand All @@ -113,17 +113,17 @@ RUN \
sha256sum -c bazel.${ARCH}.sha256
RUN chmod +x bazel

FROM ubuntu:24.04 as runtime_deps
FROM mirror.gcr.io/library/ubuntu:24.04 as runtime_deps

RUN apt-get update && \
apt-get install --no-install-recommends --no-install-suggests -y \
git python3 unzip && \
rm -rf /var/lib/apt/lists/*

# https://docs.docker.com/engine/release-notes
COPY --from=docker:28.1.1 /usr/local/bin/docker /usr/local/bin/
COPY --from=mirror.gcr.io/library/docker:28.1.1 /usr/local/bin/docker /usr/local/bin/
# https://hub.docker.com/r/docker/buildx-bin/tags
COPY --from=docker/buildx-bin:0.23.0 /buildx /usr/libexec/docker/cli-plugins/docker-buildx
COPY --from=mirror.gcr.io/docker/buildx-bin:0.23.0 /buildx /usr/libexec/docker/cli-plugins/docker-buildx
COPY --from=download-kubectl kubectl /usr/local/bin/
COPY --from=download-helm helm /usr/local/bin/
COPY --from=download-kustomize kustomize /usr/local/bin/
Expand Down Expand Up @@ -163,5 +163,5 @@ RUN apt-get update && apt-get install --no-install-recommends --no-install-sugge
jq \
apt-transport-https && \
rm -rf /var/lib/apt/lists/*
COPY --from=golang:1.25.5 /usr/local/go /usr/local/go
COPY --from=mirror.gcr.io/library/golang:1.25.5 /usr/local/go /usr/local/go
ENV PATH /usr/local/go/bin:/root/go/bin:$PATH
6 changes: 2 additions & 4 deletions hack/kokoro/presubmit.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.

export DOCKER_NAMESPACE=gcr.io/k8s-skaffold
source $KOKORO_GFILE_DIR/common.sh

# Changes the current directory to where Kokoro has checked out the GitHub repository.
pushd $KOKORO_ARTIFACTS_DIR/github/skaffold >/dev/null
GCP_ONLY=true make integration-in-docker
GCP_ONLY=true GCP_PROJECT=skaffold-ci-cd AR_REGION=us-central1 GKE_REGION=us-central1 make integration-in-docker
popd

2 changes: 1 addition & 1 deletion integration/examples/compose/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.18 as builder
FROM mirror.gcr.io/library/golang:1.18 as builder
WORKDIR /code
COPY main.go .
COPY go.mod .
Expand Down
2 changes: 1 addition & 1 deletion integration/examples/cross-platform-builds/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.18 as builder
FROM mirror.gcr.io/library/golang:1.18 as builder
WORKDIR /code
COPY main.go .
COPY go.mod .
Expand Down
2 changes: 1 addition & 1 deletion integration/examples/custom-tests/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.18 as builder
FROM mirror.gcr.io/library/golang:1.18 as builder
WORKDIR /code
COPY main.go .
COPY go.mod .
Expand Down
2 changes: 1 addition & 1 deletion integration/examples/docker-deploy/bert/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.18 as builder
FROM mirror.gcr.io/library/golang:1.18 as builder
WORKDIR /code
COPY main.go .
COPY go.mod .
Expand Down
2 changes: 1 addition & 1 deletion integration/examples/docker-deploy/ernie/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.18 as builder
FROM mirror.gcr.io/library/golang:1.18 as builder
WORKDIR /code
COPY main.go .
COPY go.mod .
Expand Down
2 changes: 1 addition & 1 deletion integration/examples/gcb-kaniko/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.18 as builder
FROM mirror.gcr.io/library/golang:1.18 as builder
WORKDIR /code
COPY main.go .
COPY go.mod .
Expand Down
2 changes: 1 addition & 1 deletion integration/examples/generate-pipeline/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.18 as builder
FROM mirror.gcr.io/library/golang:1.18 as builder
WORKDIR /code
COPY main.go .
COPY go.mod .
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.18 as builder
FROM mirror.gcr.io/library/golang:1.18 as builder
WORKDIR /code
COPY main.go .
COPY go.mod .
Expand Down
2 changes: 1 addition & 1 deletion integration/examples/getting-started/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.18 as builder
FROM mirror.gcr.io/library/golang:1.18 as builder
WORKDIR /code
COPY main.go .
COPY go.mod .
Expand Down
2 changes: 1 addition & 1 deletion integration/examples/google-cloud-build/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.18 as builder
FROM mirror.gcr.io/library/golang:1.18 as builder
WORKDIR /code
COPY main.go .
COPY go.mod .
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.19 as builder
FROM mirror.gcr.io/library/golang:1.23 as builder

WORKDIR /workspace

Expand Down
4 changes: 2 additions & 2 deletions integration/examples/grpc-e2e-tests/service/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
FROM golang:1.19 as builder
FROM mirror.gcr.io/library/golang:1.23 as builder

WORKDIR /workspace

COPY . .

RUN go build -o bin/visitorcounter ./server
RUN go mod tidy && go build -o bin/visitorcounter ./server

FROM gcr.io/distroless/base:latest
COPY --from=builder /workspace/bin/visitorcounter visitorcounter
Expand Down
Loading
Loading