diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index a80698bd..1356a01f 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -2,6 +2,7 @@ AADSTS artifactory bak bitnami +containerregistry curlimages deepcopy deletecollection @@ -21,8 +22,8 @@ rolearn selfsigned servicemonitor servicemonitors -SResources spiffe +SResources SVIDs tekton tpl diff --git a/pkg/utils/k8s/update_fields.go b/pkg/utils/k8s/update_fields.go index 66d0d3aa..2e4cfa0c 100644 --- a/pkg/utils/k8s/update_fields.go +++ b/pkg/utils/k8s/update_fields.go @@ -37,6 +37,7 @@ func UpdateCronJobFields(obj, desired *batchv1.CronJob) { ps.InitContainers = dps.InitContainers ps.Containers = dps.Containers ps.Volumes = dps.Volumes + ps.ImagePullSecrets = dps.ImagePullSecrets } // UpdateDeploymentFields copies managed fields from desired to obj, @@ -54,6 +55,7 @@ func UpdateDeploymentFields(obj, desired *appsv1.Deployment) { ps.ServiceAccountName = dps.ServiceAccountName ps.Containers = dps.Containers ps.Volumes = dps.Volumes + ps.ImagePullSecrets = dps.ImagePullSecrets } // UpdateDaemonSetFields copies managed fields from desired to obj, @@ -73,4 +75,5 @@ func UpdateDaemonSetFields(obj, desired *appsv1.DaemonSet) { ps.Tolerations = dps.Tolerations ps.Containers = dps.Containers ps.Volumes = dps.Volumes + ps.ImagePullSecrets = dps.ImagePullSecrets } diff --git a/pkg/utils/k8s/update_fields_test.go b/pkg/utils/k8s/update_fields_test.go new file mode 100644 index 00000000..dda106b0 --- /dev/null +++ b/pkg/utils/k8s/update_fields_test.go @@ -0,0 +1,125 @@ +// Copyright Mondoo, Inc. 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package k8s + +import ( + "testing" + + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestUpdateCronJobFields_ImagePullSecrets(t *testing.T) { + desired := &batchv1.CronJob{ + Spec: batchv1.CronJobSpec{ + Schedule: "*/5 * * * *", + JobTemplate: batchv1.JobTemplateSpec{ + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "test", Image: "test:latest"}}, + ImagePullSecrets: []corev1.LocalObjectReference{ + {Name: "my-secret"}, + {Name: "another-secret"}, + }, + }, + }, + }, + }, + }, + } + + obj := &batchv1.CronJob{} + UpdateCronJobFields(obj, desired) + + assert.Equal(t, desired.Spec.Schedule, obj.Spec.Schedule) + assert.Equal(t, desired.Spec.JobTemplate.Spec.Template.Spec.Containers, obj.Spec.JobTemplate.Spec.Template.Spec.Containers) + assert.Equal(t, desired.Spec.JobTemplate.Spec.Template.Spec.ImagePullSecrets, obj.Spec.JobTemplate.Spec.Template.Spec.ImagePullSecrets) +} + +func TestUpdateCronJobFields_PreservesUnmanagedFields(t *testing.T) { + obj := &batchv1.CronJob{ + Spec: batchv1.CronJobSpec{ + JobTemplate: batchv1.JobTemplateSpec{ + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + DNSPolicy: corev1.DNSClusterFirst, + SchedulerName: "custom-scheduler", + }, + }, + }, + }, + }, + } + + desired := &batchv1.CronJob{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "test"}}, + Spec: batchv1.CronJobSpec{ + Schedule: "*/10 * * * *", + JobTemplate: batchv1.JobTemplateSpec{ + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "test"}}, + }, + }, + }, + }, + }, + } + + UpdateCronJobFields(obj, desired) + + // Managed fields are updated + assert.Equal(t, "*/10 * * * *", obj.Spec.Schedule) + // Unmanaged fields are preserved + assert.Equal(t, corev1.DNSClusterFirst, obj.Spec.JobTemplate.Spec.Template.Spec.DNSPolicy) + assert.Equal(t, "custom-scheduler", obj.Spec.JobTemplate.Spec.Template.Spec.SchedulerName) +} + +func TestUpdateDeploymentFields_ImagePullSecrets(t *testing.T) { + desired := &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "test", Image: "test:latest"}}, + ImagePullSecrets: []corev1.LocalObjectReference{ + {Name: "my-secret"}, + }, + }, + }, + }, + } + + obj := &appsv1.Deployment{} + UpdateDeploymentFields(obj, desired) + + assert.Equal(t, desired.Spec.Template.Spec.ImagePullSecrets, obj.Spec.Template.Spec.ImagePullSecrets) + assert.Equal(t, desired.Spec.Template.Spec.Containers, obj.Spec.Template.Spec.Containers) +} + +func TestUpdateDaemonSetFields_ImagePullSecrets(t *testing.T) { + desired := &appsv1.DaemonSet{ + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "test", Image: "test:latest"}}, + ImagePullSecrets: []corev1.LocalObjectReference{ + {Name: "my-secret"}, + }, + }, + }, + }, + } + + obj := &appsv1.DaemonSet{} + UpdateDaemonSetFields(obj, desired) + + assert.Equal(t, desired.Spec.Template.Spec.ImagePullSecrets, obj.Spec.Template.Spec.ImagePullSecrets) + assert.Equal(t, desired.Spec.Template.Spec.Containers, obj.Spec.Template.Spec.Containers) +} diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 789ee1c2..c288acd7 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -6,8 +6,9 @@ End-to-end tests that deploy the Mondoo operator to a real GKE cluster and verif - **Fresh Deploy** (`run-fresh-deploy.sh`): Builds the operator from the current branch, deploys to a GKE cluster, configures scanning, and verifies everything works. - **Upgrade** (`run-upgrade.sh`): Installs a released baseline version first, verifies it, then upgrades to the current branch and verifies again. +- **Registry Mirroring & Proxy** (`run-registry-mirroring.sh`): Deploys with an Artifact Registry mirror repo and optional Squid proxy. Verifies image references are rewritten, `imagePullSecrets` are propagated, and proxy env vars are set. -Both tests pause for manual verification at each verify step (check Mondoo console for assets/scan results). Press Enter to continue or Ctrl+C to abort. +All tests pause for manual verification at each verify step (check Mondoo console for assets/scan results). Press Enter to continue or Ctrl+C to abort. ## Prerequisites @@ -17,6 +18,8 @@ Both tests pause for manual verification at each verify step (check Mondoo conso - `docker` - `kubectl` - Go toolchain (for building the operator) +- `jq` (for registry mirroring test verification) +- `crane` CLI (for registry mirroring test): `go install github.com/google/go-containerregistry/cmd/crane@latest` ### Mondoo credentials @@ -57,6 +60,8 @@ terraform apply \ | `mondoo_org_id` | yes | - | Mondoo organization ID | | `region` | no | `europe-west3` | GCP region | | `autopilot` | no | `true` | `true` for Autopilot, `false` for Standard cluster | +| `enable_mirror_test` | no | `false` | Create a mirror AR repo for registry mirroring/imagePullSecrets tests | +| `enable_proxy_test` | no | `false` | Provision a Squid proxy VM for proxy tests (requires `enable_mirror_test`) | You can also set these in a `terraform.tfvars` file. @@ -94,6 +99,32 @@ What it does: 6. Upgrades to the current branch image via local Helm chart 7. Waits, verifies again, pauses for manual check +### Registry Mirroring & Proxy + +```bash +# Provision infrastructure with mirror AR repo (and optional proxy VM) +cd tests/e2e/terraform +terraform apply -var="project_id=MY_PROJECT" -var="mondoo_org_id=MY_ORG" \ + -var="enable_mirror_test=true" -var="enable_proxy_test=true" + +# Run the test +cd tests/e2e +./run-registry-mirroring.sh +``` + +What it does: +1. Loads Terraform outputs (including mirror AR repo and proxy IP if enabled) +2. Builds the operator image and pushes to Artifact Registry +3. Creates `imagePullSecret` from the mirror repo's GCP service account key +4. Uses `crane` to copy cnspec image from ghcr.io into the mirror AR repo +5. Deploys an nginx test workload +6. Helm installs the operator with `registryMirrors`, `imagePullSecrets`, and proxy `--values` +7. Applies MondooAuditConfig, waits 90s for reconciliation +8. Runs standard verification checks +9. Runs mirroring-specific checks: image refs rewritten, pull secrets propagated, proxy env vars set +10. Checks Squid proxy access logs (if proxy enabled) +11. Pauses for manual verification in the Mondoo console + ## Cleanup Remove all test resources from the cluster (everything except Terraform infra): @@ -115,23 +146,29 @@ terraform destroy -var="project_id=MY_PROJECT" -var="mondoo_org_id=MY_ORG" ``` tests/e2e/ ├── README.md -├── run-fresh-deploy.sh # Fresh deploy test orchestrator -├── run-upgrade.sh # Upgrade test orchestrator +├── run-fresh-deploy.sh # Fresh deploy test orchestrator +├── run-upgrade.sh # Upgrade test orchestrator +├── run-registry-mirroring.sh # Registry mirroring & proxy test orchestrator ├── terraform/ -│ ├── versions.tf # Provider requirements -│ ├── variables.tf # Input variables -│ ├── main.tf # GKE cluster + Artifact Registry -│ ├── mondoo.tf # Mondoo space + service account -│ └── outputs.tf # Outputs consumed by scripts +│ ├── versions.tf # Provider requirements +│ ├── variables.tf # Input variables +│ ├── main.tf # GKE cluster + Artifact Registry +│ ├── mondoo.tf # Mondoo space + service account +│ ├── proxy.tf # Squid proxy VM (optional) +│ └── outputs.tf # Outputs consumed by scripts ├── scripts/ -│ ├── common.sh # Logging, TF output loading, wait helpers -│ ├── build-and-push.sh # Build operator image, push to AR -│ ├── deploy-operator.sh # Helm install from local chart -│ ├── deploy-baseline.sh # Helm install released version -│ ├── deploy-test-workload.sh # Deploy nginx for scanning -│ ├── apply-mondoo-config.sh # Create secret + apply MondooAuditConfig -│ ├── verify.sh # Automated checks + manual verification pause -│ └── cleanup.sh # Remove all test resources from cluster +│ ├── common.sh # Logging, TF output loading, wait helpers +│ ├── build-and-push.sh # Build operator image, push to AR +│ ├── deploy-operator.sh # Helm install from local chart +│ ├── deploy-operator-mirroring.sh # Helm install with mirror/proxy values +│ ├── deploy-baseline.sh # Helm install released version +│ ├── deploy-test-workload.sh # Deploy nginx for scanning +│ ├── apply-mondoo-config.sh # Create secret + apply MondooAuditConfig +│ ├── setup-mirror-registry.sh # Create imagePullSecret for mirror AR repo +│ ├── populate-mirror-registry.sh # Copy cnspec image into mirror AR repo via crane +│ ├── verify.sh # Automated checks + manual verification pause +│ ├── verify-mirroring.sh # Mirroring/proxy-specific verification +│ └── cleanup.sh # Remove all test resources from cluster └── manifests/ ├── mondoo-audit-config.yaml.tpl # Standard cluster config (nodes enabled) ├── mondoo-audit-config-autopilot.yaml.tpl # Autopilot config (nodes disabled) diff --git a/tests/e2e/run-registry-mirroring.sh b/tests/e2e/run-registry-mirroring.sh new file mode 100755 index 00000000..b9a0be55 --- /dev/null +++ b/tests/e2e/run-registry-mirroring.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Copyright Mondoo, Inc. 2026 +# SPDX-License-Identifier: BUSL-1.1 + +# Test: Registry Mirroring, imagePullSecrets & Proxy +# Builds operator, deploys with AR-based mirror registry and optional proxy, +# verifies image references, pull secrets, and proxy configuration. +# +# Prerequisites: +# - Terraform infrastructure provisioned with enable_mirror_test=true +# - crane CLI installed (go install github.com/google/go-containerregistry/cmd/crane@latest) +# - gcloud authenticated, docker, helm, kubectl available +# - For proxy testing: also set enable_proxy_test=true in terraform +# +# Usage: +# ./run-registry-mirroring.sh + +set -euo pipefail + +E2E_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${E2E_ROOT}/scripts/common.sh" + +info "==========================================" +info " Test: Registry Mirroring & Proxy" +info "==========================================" + +# Step 1: Load Terraform outputs (includes mirror and proxy outputs) +load_tf_outputs + +# Validate that mirror test infra is provisioned +if [[ "${ENABLE_MIRROR_TEST}" != "true" ]]; then + die "This test requires enable_mirror_test=true in Terraform. Run: terraform apply -var='enable_mirror_test=true'" +fi + +# Step 2: Build and push operator image +info "--- Step: Build and Push ---" +source "${E2E_ROOT}/scripts/build-and-push.sh" + +# Step 3: Create imagePullSecret for mirror AR repo +info "--- Step: Setup Mirror Registry Credentials ---" +source "${E2E_ROOT}/scripts/setup-mirror-registry.sh" + +# Step 4: Populate mirror AR repo with cnspec image +info "--- Step: Populate Mirror Registry ---" +source "${E2E_ROOT}/scripts/populate-mirror-registry.sh" + +# Step 5: Deploy test workload +info "--- Step: Deploy Test Workload ---" +source "${E2E_ROOT}/scripts/deploy-test-workload.sh" + +# Step 6: Set proxy env vars from Terraform if enabled +if [[ "${ENABLE_PROXY_TEST}" == "true" ]]; then + info "Proxy testing enabled — Squid proxy at ${SQUID_PROXY_IP}" +else + info "Proxy testing disabled — skipping proxy configuration" +fi + +# Step 7: Deploy operator with mirroring/proxy configuration +info "--- Step: Deploy Operator with Mirroring ---" +source "${E2E_ROOT}/scripts/deploy-operator-mirroring.sh" + +# Step 8: Apply MondooAuditConfig +info "--- Step: Apply Mondoo Config ---" +source "${E2E_ROOT}/scripts/apply-mondoo-config.sh" + +# Step 9: Wait for operator to reconcile +info "Waiting 90s for operator to reconcile with mirrored images..." +sleep 90 + +# Step 10: Standard verification +info "--- Step: Standard Verify ---" +source "${E2E_ROOT}/scripts/verify.sh" + +# Step 11: Mirroring-specific verification +info "--- Step: Verify Mirroring & Proxy ---" +source "${E2E_ROOT}/scripts/verify-mirroring.sh" + +info "" +info "==========================================" +info " Test: Registry Mirroring & Proxy - COMPLETE" +info "==========================================" diff --git a/tests/e2e/scripts/cleanup.sh b/tests/e2e/scripts/cleanup.sh index f92f9597..779dbadf 100755 --- a/tests/e2e/scripts/cleanup.sh +++ b/tests/e2e/scripts/cleanup.sh @@ -66,4 +66,8 @@ fi info "Removing mondoo Helm repo..." helm repo remove mondoo 2>/dev/null || true +# Mirror registry resources (from registry mirroring tests) +info "Deleting mirror-registry-creds secret..." +kubectl delete secret mirror-registry-creds -n "${NAMESPACE}" --ignore-not-found + info "=== Cleanup complete ===" diff --git a/tests/e2e/scripts/common.sh b/tests/e2e/scripts/common.sh index 52771493..43b7c61e 100755 --- a/tests/e2e/scripts/common.sh +++ b/tests/e2e/scripts/common.sh @@ -43,6 +43,17 @@ load_tf_outputs() { export TARGET_KUBECONFIG_PATH="$(cd "${TF_DIR}" && realpath "$(terraform output -raw target_kubeconfig_path)")" fi + export ENABLE_MIRROR_TEST="$(terraform output -raw enable_mirror_test 2>/dev/null || echo "false")" + if [[ "${ENABLE_MIRROR_TEST}" == "true" ]]; then + export MIRROR_REGISTRY="$(terraform output -raw mirror_registry_repo)" + export MIRROR_SA_KEY_B64="$(terraform output -raw mirror_sa_key_b64)" + fi + + export ENABLE_PROXY_TEST="$(terraform output -raw enable_proxy_test 2>/dev/null || echo "false")" + if [[ "${ENABLE_PROXY_TEST}" == "true" ]]; then + export SQUID_PROXY_IP="$(terraform output -raw squid_proxy_ip)" + fi + cd ->/dev/null # Use gcloud to get cluster credentials (auto-refreshing auth) diff --git a/tests/e2e/scripts/deploy-operator-mirroring.sh b/tests/e2e/scripts/deploy-operator-mirroring.sh new file mode 100755 index 00000000..26f7be2f --- /dev/null +++ b/tests/e2e/scripts/deploy-operator-mirroring.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copyright Mondoo, Inc. 2026 +# SPDX-License-Identifier: BUSL-1.1 + +# Deploy the operator from the local Helm chart with mirror/proxy configuration +# Uses a values-override file to avoid Helm --set escaping issues with dots in keys + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +: "${IMAGE_REPO:?IMAGE_REPO must be set}" +: "${IMAGE_TAG:?IMAGE_TAG must be set}" +: "${NAMESPACE:?NAMESPACE must be set}" +: "${MIRROR_REGISTRY:?MIRROR_REGISTRY must be set}" + +# Proxy settings (optional) +SQUID_PROXY_IP="${SQUID_PROXY_IP:-}" + +info "Deploying operator with mirroring configuration..." +info " Mirror registry: ${MIRROR_REGISTRY}" +if [[ -n "${SQUID_PROXY_IP}" ]]; then + info " Proxy: http://${SQUID_PROXY_IP}:3128" +fi + +# Build proxy values +PROXY_HTTP="" +PROXY_HTTPS="" +PROXY_NO="" +PROXY_CONTAINER="" +if [[ -n "${SQUID_PROXY_IP}" ]]; then + PROXY_HTTP="http://${SQUID_PROXY_IP}:3128" + PROXY_HTTPS="http://${SQUID_PROXY_IP}:3128" + PROXY_CONTAINER="http://${SQUID_PROXY_IP}:3128" + + # Get both the external and in-cluster Kubernetes API server IPs. + # External: from kubeconfig (used by kubectl from outside) + # Internal: the kubernetes.default ClusterIP (used by pods via KUBERNETES_SERVICE_HOST) + K8S_API_EXTERNAL=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed -E 's|https?://||;s|:[0-9]+$||') + K8S_API_INTERNAL=$(kubectl get svc kubernetes -n default -o jsonpath='{.spec.clusterIP}') + info "Kubernetes API server IPs: external=${K8S_API_EXTERNAL}, internal=${K8S_API_INTERNAL}" + PROXY_NO="10.0.0.0/8,172.16.0.0/12,.cluster.local,.svc,localhost,127.0.0.1,${K8S_API_EXTERNAL},${K8S_API_INTERNAL}" +fi + +# Generate values override file +# Using a file avoids Helm --set escaping issues with dots in registry mirror keys +VALUES_FILE=$(mktemp /tmp/mondoo-mirror-values-XXXXXX.yaml) +trap 'rm -f "${VALUES_FILE}"' EXIT + +cat > "${VALUES_FILE}" </dev/null | grep mondoo || true); do + info "Adopting existing CRD for Helm: ${crd}" + kubectl label "${crd}" app.kubernetes.io/managed-by=Helm --overwrite + kubectl annotate "${crd}" meta.helm.sh/release-name=mondoo-operator meta.helm.sh/release-namespace="${NAMESPACE}" --overwrite +done + +helm upgrade --install mondoo-operator "${REPO_ROOT}/charts/mondoo-operator" \ + --namespace "${NAMESPACE}" --create-namespace \ + --set controllerManager.manager.image.repository="${IMAGE_REPO}" \ + --set controllerManager.manager.image.tag="${IMAGE_TAG}" \ + --set controllerManager.manager.imagePullPolicy=Always \ + --values "${VALUES_FILE}" \ + --wait --timeout 5m + +wait_for_deployment "${NAMESPACE}" "mondoo-operator-controller-manager" + +info "Operator deployed with mirroring configuration." diff --git a/tests/e2e/scripts/populate-mirror-registry.sh b/tests/e2e/scripts/populate-mirror-registry.sh new file mode 100755 index 00000000..41ab8da9 --- /dev/null +++ b/tests/e2e/scripts/populate-mirror-registry.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Copyright Mondoo, Inc. 2026 +# SPDX-License-Identifier: BUSL-1.1 + +# Copy cnspec image from ghcr.io into the mirror Artifact Registry repo. +# Uses crane locally with gcloud auth (caller is already authenticated to GCP). +# +# The operator image is pulled directly from the main AR repo (set via Helm +# --set values) and does NOT go through the mirror. Only images the operator +# spawns (cnspec) get resolved through registryMirrors. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +: "${MIRROR_REGISTRY:?MIRROR_REGISTRY must be set}" +: "${REGION:?REGION must be set}" + +# cnspec image constants (must match pkg/utils/mondoo/container_image_resolver.go) +CNSPEC_IMAGE="ghcr.io/mondoohq/mondoo-operator/cnspec" +CNSPEC_TAG="12-rootless" + +CNSPEC_SRC="${CNSPEC_IMAGE}:${CNSPEC_TAG}" +# Mirror path preserves the ghcr.io path structure so registryMirrors mapping works: +# ghcr.io/mondoohq/mondoo-operator/cnspec:tag -> MIRROR_REGISTRY/mondoohq/mondoo-operator/cnspec:tag +CNSPEC_MIRROR="${MIRROR_REGISTRY}/mondoohq/mondoo-operator/cnspec:${CNSPEC_TAG}" + +info "Populating mirror registry..." +info " cnspec: ${CNSPEC_SRC} -> ${CNSPEC_MIRROR}" + +# Verify crane is available +if ! command -v crane &>/dev/null; then + die "crane CLI is required. Install with: go install github.com/google/go-containerregistry/cmd/crane@latest" +fi + +# Configure crane auth for the mirror AR repo (uses gcloud credentials) +info "Configuring crane auth for Artifact Registry..." +crane auth login "${REGION}-docker.pkg.dev" -u oauth2accesstoken -p "$(gcloud auth print-access-token)" + +# Copy cnspec image from ghcr.io (public) to mirror AR repo +info "Copying cnspec image to mirror..." +crane copy "${CNSPEC_SRC}" "${CNSPEC_MIRROR}" + +info "Mirror registry populated successfully." diff --git a/tests/e2e/scripts/setup-mirror-registry.sh b/tests/e2e/scripts/setup-mirror-registry.sh new file mode 100755 index 00000000..7697a925 --- /dev/null +++ b/tests/e2e/scripts/setup-mirror-registry.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Copyright Mondoo, Inc. 2026 +# SPDX-License-Identifier: BUSL-1.1 + +# Create the imagePullSecret for the mirror Artifact Registry repo. +# The mirror AR repo and a read-only service account are provisioned by Terraform. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +: "${NAMESPACE:?NAMESPACE must be set}" +: "${MIRROR_REGISTRY:?MIRROR_REGISTRY must be set}" +: "${MIRROR_SA_KEY_B64:?MIRROR_SA_KEY_B64 must be set}" +: "${REGION:?REGION must be set}" + +info "Setting up mirror registry credentials..." + +# The mirror AR repo URL is REGION-docker.pkg.dev, extract the server +MIRROR_SERVER="${REGION}-docker.pkg.dev" + +# Decode the GCP SA key (Terraform outputs it as base64) +SA_KEY_JSON=$(echo "${MIRROR_SA_KEY_B64}" | base64 -d) + +# Create docker-registry secret in operator namespace for imagePullSecrets +info "Creating mirror-registry-creds secret in ${NAMESPACE}..." +kubectl create namespace "${NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - +kubectl create secret docker-registry mirror-registry-creds \ + --namespace "${NAMESPACE}" \ + --docker-server="${MIRROR_SERVER}" \ + --docker-username="_json_key" \ + --docker-password="${SA_KEY_JSON}" \ + --dry-run=client -o yaml | kubectl apply -f - + +info "Mirror registry credentials created for ${MIRROR_SERVER}" diff --git a/tests/e2e/scripts/verify-mirroring.sh b/tests/e2e/scripts/verify-mirroring.sh new file mode 100755 index 00000000..9892ee9a --- /dev/null +++ b/tests/e2e/scripts/verify-mirroring.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Copyright Mondoo, Inc. 2026 +# SPDX-License-Identifier: BUSL-1.1 + +# Verify mirroring, imagePullSecrets, and proxy configuration + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +: "${NAMESPACE:?NAMESPACE must be set}" +: "${MIRROR_REGISTRY:?MIRROR_REGISTRY must be set}" + +SQUID_PROXY_IP="${SQUID_PROXY_IP:-}" +ENABLE_PROXY_TEST="${ENABLE_PROXY_TEST:-false}" + +PASS=0 +FAIL=0 +WARN_COUNT=0 + +check() { + local desc="$1" + shift + if "$@" >/dev/null 2>&1; then + info "PASS: ${desc}" + PASS=$((PASS + 1)) + else + err "FAIL: ${desc}" + FAIL=$((FAIL + 1)) + fi +} + +check_warn() { + local desc="$1" + shift + if "$@" >/dev/null 2>&1; then + info "PASS: ${desc}" + PASS=$((PASS + 1)) + else + warn "WARN: ${desc}" + WARN_COUNT=$((WARN_COUNT + 1)) + fi +} + +info "=== Mirroring & Proxy Verification ===" + +# Check MondooOperatorConfig has registryMirrors +check "MondooOperatorConfig has registryMirrors" \ + bash -c "kubectl get mondoooperatorconfigs.k8s.mondoo.com -n '${NAMESPACE}' -o jsonpath='{.items[0].spec.registryMirrors}' | grep -q '${MIRROR_REGISTRY}'" + +# Check MondooOperatorConfig has imagePullSecrets +check "MondooOperatorConfig has imagePullSecrets" \ + bash -c "kubectl get mondoooperatorconfigs.k8s.mondoo.com -n '${NAMESPACE}' -o jsonpath='{.items[0].spec.imagePullSecrets[0].name}' | grep -q 'mirror-registry-creds'" + +# Check CronJob pod specs have imagePullSecrets +check "CronJob pods have imagePullSecrets" \ + bash -c "kubectl get cronjobs -n '${NAMESPACE}' -l mondoo_cr=mondoo-client -o json | jq -e '.items[0].spec.jobTemplate.spec.template.spec.imagePullSecrets[] | select(.name==\"mirror-registry-creds\")'" + +# Check CronJob container images reference the mirror registry +check "CronJob images use mirror registry" \ + bash -c "kubectl get cronjobs -n '${NAMESPACE}' -l mondoo_cr=mondoo-client -o json | jq -e '.items[0].spec.jobTemplate.spec.template.spec.containers[0].image' | grep -q '${MIRROR_REGISTRY}'" + +# Check operator deployment has imagePullSecrets +check "Operator deployment has imagePullSecrets" \ + bash -c "kubectl get deployment mondoo-operator-controller-manager -n '${NAMESPACE}' -o json | jq -e '.spec.template.spec.imagePullSecrets[] | select(.name==\"mirror-registry-creds\")'" + +# Proxy-specific checks +if [[ "${ENABLE_PROXY_TEST}" == "true" && -n "${SQUID_PROXY_IP}" ]]; then + info "" + info "--- Proxy Checks ---" + + # Check MondooOperatorConfig has proxy settings + check "MondooOperatorConfig has httpProxy" \ + bash -c "kubectl get mondoooperatorconfigs.k8s.mondoo.com -n '${NAMESPACE}' -o jsonpath='{.items[0].spec.httpProxy}' | grep -q '${SQUID_PROXY_IP}'" + + check "MondooOperatorConfig has httpsProxy" \ + bash -c "kubectl get mondoooperatorconfigs.k8s.mondoo.com -n '${NAMESPACE}' -o jsonpath='{.items[0].spec.httpsProxy}' | grep -q '${SQUID_PROXY_IP}'" + + # Check that CronJob containers have HTTP_PROXY env vars + check "CronJob containers have HTTP_PROXY env var" \ + bash -c "kubectl get cronjobs -n '${NAMESPACE}' -l mondoo_cr=mondoo-client -o json | jq -e '.items[0].spec.jobTemplate.spec.template.spec.containers[0].env[] | select(.name==\"HTTP_PROXY\")'" + + check "CronJob containers have HTTPS_PROXY env var" \ + bash -c "kubectl get cronjobs -n '${NAMESPACE}' -l mondoo_cr=mondoo-client -o json | jq -e '.items[0].spec.jobTemplate.spec.template.spec.containers[0].env[] | select(.name==\"HTTPS_PROXY\")'" + + # Check Squid access log for traffic (WARN, not FAIL — timing-dependent) + info "" + info "--- Squid Proxy Log Check ---" + SQUID_ZONE="${REGION:-europe-west3}-a" + SQUID_INSTANCE="${NAME_PREFIX:-mondoo-e2e}-squid-proxy" + + check_warn "Squid proxy shows access log traffic" \ + bash -c "gcloud compute ssh '${SQUID_INSTANCE}' --zone='${SQUID_ZONE}' --project='${PROJECT_ID}' --tunnel-through-iap --command='sudo tail -20 /var/log/squid/access.log' 2>/dev/null | grep -q ." + + info "Squid access log (last 20 lines):" + gcloud compute ssh "${SQUID_INSTANCE}" --zone="${SQUID_ZONE}" --project="${PROJECT_ID}" \ + --tunnel-through-iap --command="sudo tail -20 /var/log/squid/access.log" 2>/dev/null || warn "Could not retrieve Squid logs" +else + info "" + info "--- Proxy checks skipped (enable_proxy_test != true) ---" +fi + +info "" +info "=== Resource Details (Mirroring) ===" + +info "--- CronJob Image References ---" +kubectl get cronjobs -n "${NAMESPACE}" -l mondoo_cr=mondoo-client \ + -o jsonpath='{range .items[*]}{.metadata.name}: {.spec.jobTemplate.spec.template.spec.containers[0].image}{"\n"}{end}' 2>/dev/null || true + +info "" +info "--- MondooOperatorConfig ---" +kubectl get mondoooperatorconfigs.k8s.mondoo.com -n "${NAMESPACE}" -o yaml 2>/dev/null || true + +info "" +info "=== Results: ${PASS} passed, ${FAIL} failed, ${WARN_COUNT} warnings ===" + +if [[ ${FAIL} -gt 0 ]]; then + err "Some checks failed. Review the output above." + exit 1 +fi + +info "" +info "=== Manual Verification ===" +info "Check the Mondoo console for assets in the test space" +info " Space MRN: ${MONDOO_SPACE_MRN:-unknown}" +info " - Verify assets were scanned successfully through mirrored images" +info " - Container images should reference the mirror registry" +if [[ "${ENABLE_PROXY_TEST}" == "true" ]]; then + info " - Check Squid proxy logs for HTTPS CONNECT traffic" +fi +read -rp "Press Enter once verified (or Ctrl+C to abort)... " diff --git a/tests/e2e/terraform/main.tf b/tests/e2e/terraform/main.tf index 2641c319..3b5d68d1 100644 --- a/tests/e2e/terraform/main.tf +++ b/tests/e2e/terraform/main.tf @@ -57,6 +57,46 @@ resource "google_artifact_registry_repository" "e2e" { project = var.project_id } +################################################################################ +# Mirror Registry (optional, for registry mirroring tests) +# A second Artifact Registry repo used as the mirror target. +################################################################################ + +resource "google_artifact_registry_repository" "mirror" { + count = var.enable_mirror_test ? 1 : 0 + + location = var.region + repository_id = "${local.name_prefix}-mirror" + format = "DOCKER" + project = var.project_id +} + +# Service account with read-only access to the mirror repo. +# Used to create an imagePullSecret, testing the full imagePullSecrets path. +resource "google_service_account" "mirror_reader" { + count = var.enable_mirror_test ? 1 : 0 + + account_id = "${local.name_prefix}-mirror-sa" + display_name = "Mirror registry reader for e2e tests" + project = var.project_id +} + +resource "google_artifact_registry_repository_iam_member" "mirror_reader" { + count = var.enable_mirror_test ? 1 : 0 + + project = var.project_id + location = var.region + repository = google_artifact_registry_repository.mirror[0].repository_id + role = "roles/artifactregistry.reader" + member = "serviceAccount:${google_service_account.mirror_reader[0].email}" +} + +resource "google_service_account_key" "mirror_reader" { + count = var.enable_mirror_test ? 1 : 0 + + service_account_id = google_service_account.mirror_reader[0].name +} + ################################################################################ # Target Cluster (optional, for external cluster scanning tests) ################################################################################ diff --git a/tests/e2e/terraform/outputs.tf b/tests/e2e/terraform/outputs.tf index 04eada86..35df01db 100644 --- a/tests/e2e/terraform/outputs.tf +++ b/tests/e2e/terraform/outputs.tf @@ -49,3 +49,24 @@ output "target_cluster_name" { output "target_kubeconfig_path" { value = var.enable_target_cluster ? local_file.kubeconfig_target[0].filename : "" } + +output "enable_mirror_test" { + value = var.enable_mirror_test +} + +output "mirror_registry_repo" { + value = var.enable_mirror_test ? "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.mirror[0].repository_id}" : "" +} + +output "mirror_sa_key_b64" { + value = var.enable_mirror_test ? google_service_account_key.mirror_reader[0].private_key : "" + sensitive = true +} + +output "enable_proxy_test" { + value = var.enable_proxy_test +} + +output "squid_proxy_ip" { + value = var.enable_proxy_test ? google_compute_instance.squid_proxy[0].network_interface[0].network_ip : "" +} diff --git a/tests/e2e/terraform/proxy.tf b/tests/e2e/terraform/proxy.tf new file mode 100644 index 00000000..4eb9bd29 --- /dev/null +++ b/tests/e2e/terraform/proxy.tf @@ -0,0 +1,99 @@ +# Copyright Mondoo, Inc. 2026 +# SPDX-License-Identifier: BUSL-1.1 + +################################################################################ +# Squid Proxy VM (optional, for registry mirroring / proxy tests) +################################################################################ + +resource "google_compute_instance" "squid_proxy" { + count = var.enable_proxy_test ? 1 : 0 + + lifecycle { + precondition { + condition = var.enable_mirror_test + error_message = "enable_proxy_test requires enable_mirror_test to also be true." + } + } + + name = "${local.name_prefix}-squid-proxy" + project = var.project_id + zone = "${var.region}-a" + machine_type = "e2-micro" + + boot_disk { + initialize_params { + image = "debian-cloud/debian-12" + } + } + + network_interface { + network = "default" + # No access_config — use IAP tunneling for SSH instead of a public IP + } + + metadata_startup_script = <<-SCRIPT +#!/bin/bash +set -e +apt-get update +apt-get install -y squid + +cat > /etc/squid/squid.conf <<'EOF' +# Mondoo e2e test Squid proxy +acl localnet src 10.0.0.0/8 +acl localnet src 172.16.0.0/12 +acl localnet src 192.168.0.0/16 +acl SSL_ports port 443 +acl Safe_ports port 80 +acl Safe_ports port 443 +acl Safe_ports port 1025-65535 +acl CONNECT method CONNECT + +http_access allow localnet +http_access allow localhost +http_access deny all + +http_port 3128 + +access_log /var/log/squid/access.log squid +cache_log /var/log/squid/cache.log +EOF + +systemctl restart squid +systemctl enable squid + SCRIPT + + tags = ["squid-proxy"] +} + +resource "google_compute_firewall" "allow_squid" { + count = var.enable_proxy_test ? 1 : 0 + + name = "${local.name_prefix}-allow-squid" + project = var.project_id + network = "default" + + allow { + protocol = "tcp" + ports = ["3128"] + } + + source_ranges = ["10.0.0.0/8"] + target_tags = ["squid-proxy"] +} + +resource "google_compute_firewall" "allow_squid_iap_ssh" { + count = var.enable_proxy_test ? 1 : 0 + + name = "${local.name_prefix}-allow-squid-iap-ssh" + project = var.project_id + network = "default" + + allow { + protocol = "tcp" + ports = ["22"] + } + + # IAP's IP range for TCP forwarding + source_ranges = ["35.235.240.0/20"] + target_tags = ["squid-proxy"] +} diff --git a/tests/e2e/terraform/terraform.example.tfvars b/tests/e2e/terraform/terraform.example.tfvars index b2a0f531..06fb2fe4 100644 --- a/tests/e2e/terraform/terraform.example.tfvars +++ b/tests/e2e/terraform/terraform.example.tfvars @@ -3,4 +3,9 @@ project_id = "your-gcp-project-id" mondoo_org_id = "your-mondoo-organization-id" -autopilot = true \ No newline at end of file +autopilot = true + +# Set to true to provision a mirror AR repo for registry mirroring/imagePullSecrets tests +enable_mirror_test = false +# Set to true to also provision a Squid proxy VM for proxy tests (requires enable_mirror_test) +enable_proxy_test = false diff --git a/tests/e2e/terraform/variables.tf b/tests/e2e/terraform/variables.tf index 3b43665a..5f3130f1 100644 --- a/tests/e2e/terraform/variables.tf +++ b/tests/e2e/terraform/variables.tf @@ -28,3 +28,15 @@ variable "enable_target_cluster" { type = bool default = false } + +variable "enable_mirror_test" { + description = "Create a mirror Artifact Registry repo for registry mirroring and imagePullSecrets testing" + type = bool + default = false +} + +variable "enable_proxy_test" { + description = "Create a Squid proxy VM for proxy testing (requires enable_mirror_test=true)" + type = bool + default = false +}