Skip to content

Commit 8d537fa

Browse files
committed
refactor kop script
1 parent aa36fbe commit 8d537fa

File tree

3 files changed

+131
-93
lines changed

3 files changed

+131
-93
lines changed

e2e/scenarios/kops-simple

Lines changed: 11 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ if [[ -z "${WORKDIR:-}" ]]; then
5252
fi
5353
mkdir -p "${WORKDIR}"
5454

55+
# Common Utilities
56+
source "${REPO_ROOT}/tools/kops-common.sh"
57+
5558
# Ensure we have a project; get one from boskos if one not provided in GCP_PROJECT
5659
source "${REPO_ROOT}"/test/boskos.sh
5760
if [[ -z "${GCP_PROJECT:-}" ]]; then
@@ -62,14 +65,8 @@ if [[ -z "${GCP_PROJECT:-}" ]]; then
6265
fi
6366
echo "GCP_PROJECT=${GCP_PROJECT}"
6467

65-
# Ensure we have an SSH key; needed to dump the node information to artifacts/
66-
if [[ -z "${SSH_PRIVATE_KEY:-}" ]]; then
67-
echo "SSH_PRIVATE_KEY not set, creating one"
68-
69-
SSH_PRIVATE_KEY="${WORKDIR}/google_compute_engine"
70-
gcloud compute --project="${GCP_PROJECT}" config-ssh --ssh-key-file="${SSH_PRIVATE_KEY}"
71-
export KUBE_SSH_USER="${USER}"
72-
fi
68+
# SSH Key Setup
69+
setup_ssh_key
7370
echo "SSH_PRIVATE_KEY=${SSH_PRIVATE_KEY}"
7471

7572
# Build kubetest-2 kOps support
@@ -99,33 +96,9 @@ echo "CLOUD_PROVIDER=${CLOUD_PROVIDER}"
9996
#Set cloud provider location
10097
GCP_LOCATION="${GCP_LOCATION:-us-central1}"
10198

102-
# KOPS_STATE_STORE holds metadata about the clusters we create
103-
if [[ -z "${KOPS_STATE_STORE:-}" ]]; then
104-
KOPS_STATE_STORE="gs://kops-state-${GCP_PROJECT}"
105-
# Ensure the bucket exists
106-
gsutil ls -p "${GCP_PROJECT}" "${KOPS_STATE_STORE}" || gsutil mb -p "${GCP_PROJECT}" -l "${GCP_LOCATION}" "${KOPS_STATE_STORE}"
107-
108-
# Setting ubla off so that kOps can automatically set ACLs for the default serviceACcount
109-
gsutil ubla set off "${KOPS_STATE_STORE}"
110-
111-
# Grant storage.admin on the bucket to our ServiceAccount
112-
SA=$(gcloud config list --format 'value(core.account)')
113-
gsutil iam ch serviceAccount:${SA}:admin "${KOPS_STATE_STORE}"
114-
fi
99+
# Setup KOPS_STATE_STORE
100+
setup_kops_state_store
115101
echo "KOPS_STATE_STORE=${KOPS_STATE_STORE}"
116-
export KOPS_STATE_STORE
117-
118-
# IMAGE_REPO is used to upload images
119-
if [[ -z "${IMAGE_REPO:-}" ]]; then
120-
IMAGE_REPO="gcr.io/${GCP_PROJECT}"
121-
fi
122-
echo "IMAGE_REPO=${IMAGE_REPO}"
123-
124-
cd ${REPO_ROOT}
125-
if [[ -z "${IMAGE_TAG:-}" ]]; then
126-
IMAGE_TAG=$(git rev-parse --short HEAD)-$(date +%Y%m%dT%H%M%S)
127-
fi
128-
echo "IMAGE_TAG=${IMAGE_TAG}"
129102

130103
# Build and push cloud-controller-manager
131104
cd ${REPO_ROOT}
@@ -138,11 +111,8 @@ unset KUBE_ROOT
138111
echo "git status:"
139112
git status
140113

141-
echo "Configuring docker auth with gcloud"
142-
gcloud auth configure-docker
143-
144-
echo "Building and pushing images"
145-
IMAGE_REPO=${IMAGE_REPO} IMAGE_TAG=${IMAGE_TAG} tools/push-images
114+
# Build CCM and Generate Manifest
115+
build_and_push_ccm
146116

147117
if [[ -z "${ADMIN_ACCESS:-}" ]]; then
148118
ADMIN_ACCESS="0.0.0.0/0" # Or use your IPv4 with /32
@@ -159,13 +129,9 @@ fi
159129
# Workaround for test-infra#24747
160130
create_args="${create_args} --gce-service-account=default"
161131

162-
# Add our manifest
163-
cp "${REPO_ROOT}/deploy/packages/default/manifest.yaml" "${WORKDIR}/cloud-provider-gcp.yaml"
164-
sed -i -e "s@k8scloudprovidergcp/cloud-controller-manager:latest@${IMAGE_REPO}/cloud-controller-manager:${IMAGE_TAG}@g" "${WORKDIR}/cloud-provider-gcp.yaml"
165-
create_args="${create_args} --add=${WORKDIR}/cloud-provider-gcp.yaml"
132+
# Add our (common generated) manifest
133+
create_args="${create_args} ${ADD_MANIFEST_ARG}"
166134

167-
# Enable cluster addons, this enables us to replace the built-in manifest
168-
KOPS_FEATURE_FLAGS="ClusterAddons,${KOPS_FEATURE_FLAGS:-}"
169135
echo "KOPS_FEATURE_FLAGS=${KOPS_FEATURE_FLAGS}"
170136

171137
# Note that these arguments for kubetest2 and kOps, not (for example) the arguments passed to the cloud-provider-gcp

tools/kops-common.sh

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2025 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# tools/kops-common.sh
18+
# Shared functions for Kops deployment scripts
19+
20+
# Configures the KOPS_STATE_STORE bucket.
21+
# Requires: GCP_PROJECT, GCP_LOCATION
22+
# Sets/Exports: KOPS_STATE_STORE
23+
function setup_kops_state_store() {
24+
if [[ -z "${KOPS_STATE_STORE:-}" ]]; then
25+
KOPS_STATE_STORE="gs://kops-state-${GCP_PROJECT}"
26+
echo "KOPS_STATE_STORE not set, using default: ${KOPS_STATE_STORE}"
27+
fi
28+
export KOPS_STATE_STORE
29+
30+
# Ensure bucket exists
31+
if ! gsutil ls -p "${GCP_PROJECT}" "${KOPS_STATE_STORE}" >/dev/null 2>&1; then
32+
echo "Creating state store bucket: ${KOPS_STATE_STORE}"
33+
gsutil mb -p "${GCP_PROJECT}" -l "${GCP_LOCATION}" "${KOPS_STATE_STORE}"
34+
gsutil ubla set off "${KOPS_STATE_STORE}"
35+
36+
# Grant storage.admin to the current service account (useful for CI/Boskos)
37+
local SA
38+
SA=$(gcloud config list --format 'value(core.account)')
39+
if [[ -n "${SA}" ]]; then
40+
echo "Granting admin access to ${SA}"
41+
gsutil iam ch serviceAccount:${SA}:admin "${KOPS_STATE_STORE}" || echo "Warning: Failed to grant IAM, possibly already owner or not a service account."
42+
fi
43+
fi
44+
}
45+
46+
# Configures SSH keys for cluster access.
47+
# Requires: GCP_PROJECT, WORKDIR/REPO_ROOT
48+
# Sets/Exports: KUBE_SSH_USER, SSH_PRIVATE_KEY_PATH (or SSH_PRIVATE_KEY for kubetest2 compatibility)
49+
function setup_ssh_key() {
50+
# Accept SSH_PRIVATE_KEY_PATH (dev script) or SSH_PRIVATE_KEY (CI script)
51+
local KEY_PATH="${SSH_PRIVATE_KEY_PATH:-${SSH_PRIVATE_KEY:-}}"
52+
53+
if [[ -z "${KEY_PATH}" ]]; then
54+
# Default location
55+
KEY_PATH="${REPO_ROOT}/google_compute_engine"
56+
echo "SSH key path not set, using default: ${KEY_PATH}"
57+
58+
if [[ ! -f "${KEY_PATH}" ]]; then
59+
echo "Generaing/Configuring SSH key..."
60+
gcloud compute config-ssh --project="${GCP_PROJECT}" --ssh-key-file="${KEY_PATH}" --quiet
61+
fi
62+
export KUBE_SSH_USER="${USER}"
63+
fi
64+
65+
# Normalize variables for both scripts
66+
export SSH_PRIVATE_KEY="${KEY_PATH}"
67+
export SSH_PRIVATE_KEY_PATH="${KEY_PATH}"
68+
export KUBE_SSH_PUBLIC_KEY_PATH="${KEY_PATH}.pub"
69+
70+
echo "SSH Key configured: ${KEY_PATH}"
71+
}
72+
73+
# Builds CCM image and generates the manifest with arguments.
74+
# Requires: REPO_ROOT, GCP_PROJECT, CLUSTER_NAME, WORKDIR
75+
# Sets/Exports: ADD_MANIFEST_ARG, KOPS_FEATURE_FLAGS
76+
function build_and_push_ccm() {
77+
echo "Building Local CCM..."
78+
79+
# Setup image tags
80+
if [[ -z "${IMAGE_REPO:-}" ]]; then
81+
IMAGE_REPO="gcr.io/${GCP_PROJECT}"
82+
fi
83+
if [[ -z "${IMAGE_TAG:-}" ]]; then
84+
IMAGE_TAG=$(git rev-parse --short HEAD)-$(date +%Y%m%dT%H%M%S)
85+
fi
86+
87+
# Configure docker auth
88+
gcloud auth configure-docker --quiet
89+
90+
# Build and Push
91+
echo "Pushing image to ${IMAGE_REPO}/cloud-controller-manager:${IMAGE_TAG}"
92+
IMAGE_REPO=${IMAGE_REPO} IMAGE_TAG=${IMAGE_TAG} "${REPO_ROOT}/tools/push-images"
93+
94+
# Prepare Manifest
95+
local MANIFEST_DIR="${WORKDIR:-${REPO_ROOT}/_tmp/${CLUSTER_NAME}}"
96+
if [[ ! -d "${MANIFEST_DIR}" ]]; then
97+
mkdir -p "${MANIFEST_DIR}"
98+
fi
99+
100+
echo "Generating manifest in ${MANIFEST_DIR}..."
101+
cp "${REPO_ROOT}/deploy/packages/default/manifest.yaml" "${MANIFEST_DIR}/cloud-provider-gcp.yaml"
102+
sed -i -e "s@k8scloudprovidergcp/cloud-controller-manager:latest@${IMAGE_REPO}/cloud-controller-manager:${IMAGE_TAG}@g" "${MANIFEST_DIR}/cloud-provider-gcp.yaml"
103+
104+
# Inject CCM args
105+
# We replace "args: [] ..." with the actual list of arguments required for CCM to run.
106+
sed -i -e "s|args: \[\] .*|args:\n - --cloud-provider=gcp\n - --leader-elect=true\n - --use-service-account-credentials\n - --allocate-node-cidrs=true\n - --configure-cloud-routes=true\n - --cluster-name=${CLUSTER_NAME}|" "${MANIFEST_DIR}/cloud-provider-gcp.yaml"
107+
108+
echo "Manifest generated at ${MANIFEST_DIR}/cloud-provider-gcp.yaml"
109+
110+
# Export for use in calling script
111+
export ADD_MANIFEST_ARG="--add=${MANIFEST_DIR}/cloud-provider-gcp.yaml"
112+
export KOPS_FEATURE_FLAGS="ClusterAddons,${KOPS_FEATURE_FLAGS:-}"
113+
}

tools/kops_lkg_ccm.sh

Lines changed: 7 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -84,29 +84,15 @@ NODE_SIZE="${NODE_SIZE:-e2-medium}"
8484
MASTER_SIZE="${MASTER_SIZE:-e2-medium}"
8585
DELETE_CLUSTER="${DELETE_CLUSTER:-true}"
8686

87-
# Setup KOPS_STATE_STORE
88-
if [[ -z "${KOPS_STATE_STORE:-}" ]]; then
89-
KOPS_STATE_STORE="gs://kops-state-${GCP_PROJECT}"
90-
fi
91-
export KOPS_STATE_STORE
87+
# Common Utilities
88+
source "${REPO_ROOT}/tools/kops-common.sh"
9289

93-
# Ensure bucket exists
94-
if ! gsutil ls -p "${GCP_PROJECT}" "${KOPS_STATE_STORE}" >/dev/null 2>&1; then
95-
gsutil mb -p "${GCP_PROJECT}" -l "${GCP_LOCATION}" "${KOPS_STATE_STORE}"
96-
gsutil ubla set off "${KOPS_STATE_STORE}"
97-
fi
90+
# Setup KOPS_STATE_STORE
91+
setup_kops_state_store
9892

9993
# SSH Key Setup
100-
if [[ -z "${SSH_PRIVATE_KEY_PATH:-}" ]]; then
101-
SSH_PRIVATE_KEY_PATH="${REPO_ROOT}/google_compute_engine"
102-
if [[ ! -f "${SSH_PRIVATE_KEY_PATH}" ]]; then
103-
gcloud compute config-ssh --project="${GCP_PROJECT}" --ssh-key-file="${SSH_PRIVATE_KEY_PATH}" --quiet
104-
fi
105-
export KUBE_SSH_USER="${USER}"
106-
fi
107-
export KUBE_SSH_PUBLIC_KEY_PATH="${SSH_PRIVATE_KEY_PATH}.pub"
94+
setup_ssh_key
10895

109-
# Cleanup trap
11096
# Cleanup trap
11197
function cleanup {
11298
if [[ "${DELETE_CLUSTER}" == "true" ]]; then
@@ -162,36 +148,9 @@ esac
162148
# Build Local CCM if needed
163149
ADD_MANIFEST_ARG=""
164150
if [[ "${BUILD_LOCAL_CCM}" == "true" ]]; then
165-
echo "Building Local CCM..."
166-
167-
# Setup image tags
168-
if [[ -z "${IMAGE_REPO:-}" ]]; then
169-
IMAGE_REPO="gcr.io/${GCP_PROJECT}"
170-
fi
171-
if [[ -z "${IMAGE_TAG:-}" ]]; then
172-
IMAGE_TAG=$(git rev-parse --short HEAD)-$(date +%Y%m%dT%H%M%S)
173-
fi
174-
175-
# Configure docker auth
176-
gcloud auth configure-docker --quiet
177-
178-
# Build and Push
179-
IMAGE_REPO=${IMAGE_REPO} IMAGE_TAG=${IMAGE_TAG} "${REPO_ROOT}/tools/push-images"
180-
181-
# Prepare Manifest
151+
# Create temp dir for manifest used by build_and_push_ccm
182152
WORKDIR="${REPO_ROOT}/_tmp/${CLUSTER_NAME}"
183-
mkdir -p "${WORKDIR}"
184-
cp "${REPO_ROOT}/deploy/packages/default/manifest.yaml" "${WORKDIR}/cloud-provider-gcp.yaml"
185-
sed -i -e "s@k8scloudprovidergcp/cloud-controller-manager:latest@${IMAGE_REPO}/cloud-controller-manager:${IMAGE_TAG}@g" "${WORKDIR}/cloud-provider-gcp.yaml"
186-
187-
# Inject CCM args
188-
# We replace "args: [] ..." with the actual list of arguments required for CCM to run.
189-
sed -i -e "s|args: \[\] .*|args:\n - --cloud-provider=gcp\n - --leader-elect=true\n - --use-service-account-credentials\n - --allocate-node-cidrs=true\n - --configure-cloud-routes=true\n - --cluster-name=${CLUSTER_NAME}|" "${WORKDIR}/cloud-provider-gcp.yaml"
190-
191-
ADD_MANIFEST_ARG="--add=${WORKDIR}/cloud-provider-gcp.yaml"
192-
193-
# Enable addons
194-
export KOPS_FEATURE_FLAGS="ClusterAddons,${KOPS_FEATURE_FLAGS:-}"
153+
build_and_push_ccm
195154
fi
196155

197156
# Setup admin access

0 commit comments

Comments
 (0)