Merge branch 'GoogleCloudPlatform:main' into main

Author: ravisiddhu

.ci/containers/README.md

@@ -14,16 +14,14 @@ The images are named according to their use. We have a small number of images th
 Before you begin, set up Docker (including configuring it to [authenticate with gcloud](https://cloud.google.com/container-registry/docs/advanced-authentication#gcloud-helper)).
 
 1. Make changes to the Dockerfile
-2. Build the image with the `testing` tag:
+2. Build & push the image with the `testing` tag:
    ```bash
-   sudo docker build . --tag gcr.io/graphite-docker-images/bash-plus:testing
+   gcloud builds submit . \
+    --tag gcr.io/graphite-docker-images/<target-image>:testing \
+    --project graphite-docker-images
    ```
-3. Push the image:
-   ```bash
-   sudo docker push gcr.io/graphite-docker-images/bash-plus:testing
-   ```
-4. Update cloudbuild yaml files to reference the image you just pushed by adding the `:testing` suffix
-5. Update files that will cause the cloudbuild yaml changes (and therefore your changes) to be exercised
+3. Update cloudbuild yaml files to reference the image you just pushed by adding the `:testing` suffix
+4. Update files that will cause the cloudbuild yaml changes (and therefore your changes) to be exercised
    - Tip: Modifying `mmv1/third_party/terraform/services/compute/metadata.go.tmpl` will trigger builds for TPG, TPGB, and TGC.
-6. Create a PR with these changes.
-7. Verify that the cloudbuild steps that should use your testing image _are_ using your testing image (in the Execution Details tab for the step.)
+5. Create a PR with these changes.
+6. Verify that the cloudbuild steps that should use your testing image _are_ using your testing image (in the Execution Details tab for the step.)

.ci/containers/build-environment/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.23-bullseye AS builder
+FROM golang:1.25-trixie AS builder
 
 # Set working directory
 WORKDIR /app
@@ -17,7 +17,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 1777 "$GOPATH"
 WORKDIR $GOPATH
 
 # terraform binary used by tfv/tgc
-COPY --from=hashicorp/terraform:1.11.0 /bin/terraform /bin/terraform
+COPY --from=hashicorp/terraform:1.14.3 /bin/terraform /bin/terraform
 
 SHELL ["/bin/bash", "-c"]
 

.ci/containers/go-plus/Dockerfile

@@ -1,5 +1,5 @@
 # Stage 1: Download go module cache for builds
-FROM golang:1.23-bullseye AS builder
+FROM golang:1.25-trixie AS builder
 ENV GOCACHE=/go/cache
 
 RUN apt-get update && apt-get install -y unzip
@@ -12,7 +12,7 @@ WORKDIR /app1/magic-modules-main/.ci/magician
 RUN go build -o /dev/null .
 
 # Stage 2: Creating the final image
-FROM golang:1.23-bullseye
+FROM golang:1.25-trixie
 SHELL ["/bin/bash", "-c"]
 ENV GOCACHE=/go/cache
 
@@ -21,15 +21,15 @@ COPY --from=builder /go/pkg/mod /go/pkg/mod
 COPY --from=builder /go/cache /go/cache
 
 RUN apt-get update && \
-    apt-get install -y git jq unzip zip parallel curl && \
-    echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
-    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - && \
+    apt-get install -y git jq unzip zip parallel apt-transport-https ca-certificates gnupg curl && \
+    echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && \
     apt-get update -y && \
-    apt-get install google-cloud-sdk -y && \
+    apt-get install google-cloud-cli -y && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN wget https://releases.hashicorp.com/terraform/1.11.0/terraform_1.11.0_linux_amd64.zip \
-    && unzip terraform_1.11.0_linux_amd64.zip \
-    && rm terraform_1.11.0_linux_amd64.zip \
-    && mv ./terraform /bin/terraform
+RUN wget https://releases.hashicorp.com/terraform/1.14.3/terraform_1.14.3_linux_amd64.zip \
+    && unzip terraform_1.14.3_linux_amd64.zip \
+    && rm terraform_1.14.3_linux_amd64.zip \
+    && mv ./terraform /bin/terraform

.ci/gcb-contributor-membership-checker.yml

@@ -0,0 +1,19 @@
+---
+steps:
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      id: collect-nightly-test-status
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      secretEnv: ["TEAMCITY_TOKEN"]
+      args:
+        - 'collect-nightly-test-status'
+        - $_CUSTOM_DATE
+
+timeout: 3600s
+options:
+    machineType: 'N1_HIGHCPU_32'
+
+logsBucket: 'gs://cloudbuild-ingest-test-data-logs'
+availableSecrets:
+  secretManager:
+    - versionName: projects/673497134629/secrets/teamcity-token/versions/latest
+      env: TEAMCITY_TOKEN

.ci/gcb-ingest-test-data.yaml

@@ -265,6 +265,7 @@ steps:
         - $BUILD_ID
         - $PROJECT_ID
         - "23"  # Build step
+        - "true"
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'

.ci/gcb-pr-downstream-generation-and-test.yml

@@ -1,21 +1,11 @@
 ---
 steps:
-    - name: 'gcr.io/graphite-docker-images/go-plus'
-      id: collect-nightly-test-status
-      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["TEAMCITY_TOKEN"]
-      args:
-        - 'collect-nightly-test-status'
-        - $_CUSTOM_DATE
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: create-test-failure-ticket
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["collect-nightly-test-status"]
       args:
         - 'create-test-failure-ticket'
-    - name: 'ubuntu'
-      args: ['sleep', '120']
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: manage-test-failure-ticket
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
@@ -31,7 +21,5 @@ options:
 logsBucket: 'gs://cloudbuild-test-failure-ticket-logs'
 availableSecrets:
   secretManager:
-    - versionName: projects/673497134629/secrets/teamcity-token/versions/latest
-      env: TEAMCITY_TOKEN
     - versionName: projects/673497134629/secrets/github-classic--repo-workflow/versions/latest
       env: GITHUB_TOKEN

.ci/gcb-test-failure-ticket.yaml

@@ -12,8 +12,8 @@ Prerequisites:
 - A BeyondCorp subscription on the organization
 
 After applying this configuration:
-- (Internal only) Enable stubbed calls for GKE MultiCloud resources
-- (Internal only) Verify ownership of `hashicorptest.com` for new service account
+- (Internal setup) Enable stubbed calls for GKE MultiCloud resources
+- (Internal setup) Verify ownership of `hashicorptest.com` for new service account
 - Enable Media CDN
 - Enable Access Boundary permissions
 - Enable BigQuery Table IAM conditions
@@ -48,9 +48,10 @@ After applying this configuration:
 - Enroll in Cloud Armor Managed Protection Plus tier
 - Add Cloud Identity Premium Plan to the Google Workspace domain
 - Perform the Privileged Access Manager set-up https://pantheon.corp.google.com/iam-admin/pam/setup
-- (Org only) Enroll the org in the Premium tier of Security Control Center
 - Upload a model with the name `tf-static-1` to the Vertex AI model registry
   - This should only be necessary until uploading new models is supported in the provider.
+- (Org only) Enroll the org in the Premium tier of Security Control Center
+- (Org only) Enable Compliance Manager https://cloud.google.com/security-command-center/docs/compliance-manager-enable
 
 Quotas that will need to be adjusted to support all tests:
 - Project quota for the new service account
@@ -74,3 +75,5 @@ Quotas that will need to be adjusted to support all tests:
 - compute.googleapis.com/n2_cpus (us-central1) to 36+
 - VMware Engine standard 72 vCPUs nodes per region - southamerica-east1 to 21
 - logging.googleapis.com/log_buckets_count to 200
+- "Reasoning engine write requests per minute" for us-central1 to 20+
+- aiplatform.googleapis.com/in_use_customer_managed_encryption_keys to 5+

.ci/infra/terraform/README.md

@@ -171,6 +171,18 @@ resource "google_organization_iam_member" "sa_principal_access_boundary_admin" {
   member = google_service_account.sa.member
 }
 
+resource "google_organization_iam_member" "dlp_admin" {
+  org_id = data.google_organization.org.org_id
+  role   = "roles/dlp.admin"
+  member = google_project_service_identity.dlp_sa.member
+}
+
+resource "google_organization_iam_member" "dlp_org_driver" {
+  org_id = data.google_organization.org.org_id
+  role   = "roles/dlp.orgdriver"
+  member = google_project_service_identity.dlp_sa.member
+}
+
 resource "google_billing_account_iam_member" "sa_master_billing_admin" {
   billing_account_id = data.google_billing_account.master_acct.id
   role               = "roles/billing.admin"
@@ -223,6 +235,7 @@ module "project-services" {
     "binaryauthorization.googleapis.com",
     "blockchainnodeengine.googleapis.com",
     "certificatemanager.googleapis.com",
+    "ces.googleapis.com",
     "chronicle.googleapis.com",
     "cloudaicompanion.googleapis.com",
     "cloudapis.googleapis.com",
@@ -236,6 +249,7 @@ module "project-services" {
     "cloudquotas.googleapis.com",
     "cloudresourcemanager.googleapis.com",
     "cloudscheduler.googleapis.com",
+    "cloudsecuritycompliance.googleapis.com",
     "cloudtasks.googleapis.com",
     "cloudtrace.googleapis.com",
     "composer.googleapis.com",
@@ -281,6 +295,7 @@ module "project-services" {
     "firebaseremoteconfig.googleapis.com",
     "firebaserules.googleapis.com",
     "firebasestorage.googleapis.com",
+    "firebasevertexai.googleapis.com",
     "firestore.googleapis.com",
     "firestorekeyvisualizer.googleapis.com",
     "gkebackup.googleapis.com",
@@ -306,6 +321,7 @@ module "project-services" {
     "migrationcenter.googleapis.com",
     "ml.googleapis.com",
     "mobilecrashreporting.googleapis.com",
+    "modelarmor.googleapis.com",
     "monitoring.googleapis.com",
     "multiclustermetering.googleapis.com",
     "netapp.googleapis.com",
@@ -314,6 +330,7 @@ module "project-services" {
     "networksecurity.googleapis.com",
     "networkservices.googleapis.com",
     "notebooks.googleapis.com",
+    "observability.googleapis.com",
     "orgpolicy.googleapis.com",
     "osconfig.googleapis.com",
     "oslogin.googleapis.com",
@@ -333,6 +350,7 @@ module "project-services" {
     "resourceviews.googleapis.com",
     "run.googleapis.com",
     "runtimeconfig.googleapis.com",
+    "saasservicemgmt.googleapis.com",
     "secretmanager.googleapis.com",
     "securesourcemanager.googleapis.com",
     "securetoken.googleapis.com",
@@ -418,6 +436,14 @@ resource "google_project_service_identity" "parametermanager_sa" {
   service = "parametermanager.googleapis.com"
 }
 
+resource "google_project_service_identity" "dlp_sa" {
+  provider = google-beta
+  depends_on = [module.project-services]
+
+  project = google_project.proj.project_id
+  service = "dlp.googleapis.com"
+}
+
 # TestAccComposerEnvironment_fixPyPiPackages
 # TestAccComposerEnvironmentComposer2_private
 # TestAccComposerEnvironment_withEncryptionConfigComposer1

.ci/infra/terraform/main.tf

@@ -30,7 +30,8 @@ func (cb *Client) ApproveDownstreamGenAndTest(prNumber, commitSha string) error
 	}
 
 	if buildId == "" {
-		return fmt.Errorf("Failed to find pending build for PR %s", prNumber)
+		fmt.Printf("WARNING: Failed to find pending build for PR %s\nThis build may have been approved already.\n", prNumber)
+		return nil
 	}
 
 	err = approveBuild(PROJECT_ID, buildId)