CTM-655 Fix CI issues with gcloud CLI

.github/workflows/integration_tests.yml

@@ -40,6 +40,9 @@ jobs:
           - build_type: centaurGcpBatchRestart
             build_mysql: 8.4
             friendly_name: Centaur GCP Batch (restart)
+          - build_type: centaurGcpBatchRestartCallCaching
+            build_mysql: 8.4
+            friendly_name: Centaur GCP Batch (restart, call caching)
           - build_type: dbms
             friendly_name: DBMS
           - build_type: centaurTes

centaur/src/main/resources/standardTestCases/dummy_backend/35k_scatter_success.test

@@ -14,4 +14,4 @@ metadata {
   "outputs.dummy_scatter.results_count": 35000
 }
 
-maximumTime = 12 minutes
+maximumTime = 15 minutes

docs/backends/GCPBatch.md

@@ -341,13 +341,14 @@ network labels, and then fall back to running on the default network.
 
 ### Custom Google Cloud SDK container
 
-Cromwell can't use Google's container registry if VPC Perimeter is used in project.
-Own repository can be used by adding `cloud-sdk-image-url` reference to used container:
+Cromwell uses `cloud-sdk:alpine` by default. If you require a static Cloud SDK version, set a custom value like `cloud-sdk:584.0.0-alpine` for `cloud-sdk-image-url`. Note that GCP [deletes tags after 1 year](https://github.com/GoogleCloudPlatform/cloud-sdk-docker#package-retention-policy), so any custom tag requires recurring updates.
+
+Cromwell can't use Google's container registry if VPC Perimeter is used in project. Work around by re-hosting within the project and setting `cloud-sdk-image-url`.
 
 ```
 google {
   ...
-  cloud-sdk-image-url = "eu.gcr.io/your-project-id/cloudsdktool/cloud-sdk:354.0.0-alpine"
+  cloud-sdk-image-url = "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine"
   cloud-sdk-image-size-gb = 1
 }
 ```

src/ci/bin/test.inc.sh

@@ -190,6 +190,9 @@ cromwell::private::create_build_variables() {
         centaurHoricromtalGcpBatch*)
             CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/gcp_batch_horicromtal_application.conf"
             ;;
+        centaurGcpBatchRestartCallCaching)
+            CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/gcp_batch_restart_application.conf"
+            ;;
         *)
             CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_BUILD_BACKEND_TYPE}_application.conf"
             ;;

src/ci/bin/testCentaurGcpBatchRestart.sh

@@ -12,12 +12,16 @@ cromwell::build::setup_common_environment
 
 cromwell::build::setup_centaur_environment
 
+# No DRS tests in this suite. Set a placeholder image so we do not spend time building the DRS localizer.
+export CROMWELL_BUILD_BATCH_DOCKER_IMAGE_DRS="mirror.gcr.io/almalinux:latest"
+
 cromwell::build::batch::setup_batch_centaur_environment
 
 cromwell::build::assemble_jars
 
 cromwell::build::run_centaur \
     -p 100 \
     -i restart \
+    -e call_cache_cha_cha_batch \
 
 cromwell::build::generate_code_coverage

src/ci/bin/testCentaurGcpBatchRestartCallCaching.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail
+export CROMWELL_BUILD_REQUIRES_SECURE=true
+# import in shellcheck / CI / IntelliJ compatible ways
+# shellcheck source=/dev/null
+source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh
+# shellcheck source=/dev/null
+source "${BASH_SOURCE%/*}/test_gcpbatch.inc.sh" || source test_gcpbatch.inc.sh
+
+cromwell::build::setup_common_environment
+
+cromwell::build::setup_centaur_environment
+
+# No DRS tests in this suite. Set a placeholder image so we do not spend time building the DRS localizer.
+export CROMWELL_BUILD_BATCH_DOCKER_IMAGE_DRS="mirror.gcr.io/almalinux:latest"
+
+cromwell::build::batch::setup_batch_centaur_environment
+
+cromwell::build::assemble_jars
+
+# Split out of testCentaurGcpBatchRestart.sh, which excludes this test. Restart tests are
+# serialized because they kill and restart the shared Cromwell, and this one runs about as
+# long as the other three combined.
+cromwell::build::run_centaur \
+    -p 100 \
+    -i call_cache_cha_cha_batch \
+
+cromwell::build::generate_code_coverage

supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/CheckpointingRunnable.scala

@@ -12,7 +12,7 @@ trait CheckpointingRunnable {
           createParameters.checkpointingConfiguration.checkpointingCommand(checkpointFilename,
                                                                            RunnableCommands.multiLineBinBashCommand
           )
-        val checkpointingEnvironment = Map.empty[String, String]
+        val checkpointingEnvironment = RunnableUtils.CloudSdkEnvironment
 
         // Initial sync from cloud:
         val initialCheckpointSyncRunnable = RunnableBuilder.cloudSdkShellRunnable(

supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/Delocalization.scala

@@ -35,6 +35,7 @@ trait Delocalization {
 
     RunnableBuilder
       .withImage(womOutputRuntimeExtractor.dockerImage.getOrElse(CloudSdkImage))
+      .withEnvironment(CloudSdkEnvironment)
       .withCommand(commands: _*)
       .withEntrypointCommand("/bin/bash")
       .withLabels(Map(Key.Tag -> Value.Delocalization))

supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableBuilder.scala

@@ -235,7 +235,8 @@ object RunnableBuilder extends BatchUtilityConversions {
       timeout = 30.minutes
     )
 
-  def cloudSdkRunnable: Runnable.Builder = Runnable.newBuilder.setContainer(cloudSdkContainerBuilder)
+  def cloudSdkRunnable: Runnable.Builder =
+    Runnable.newBuilder.setContainer(cloudSdkContainerBuilder).withEnvironment(CloudSdkEnvironment)
 
   // Set a default timeout of 24 hours for these runnables. They are typically used for running
   // localization, delocalization, small shell commands, etc. These processes occasionally hang and
@@ -247,6 +248,7 @@ object RunnableBuilder extends BatchUtilityConversions {
   ): Runnable.Builder =
     Runnable.newBuilder
       .setContainer(cloudSdkContainerBuilder)
+      .withEnvironment(CloudSdkEnvironment)
       .withVolumes(volumes)
       .withLabels(labels)
       .withEntrypointCommand(

supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableUtils.scala

@@ -16,7 +16,13 @@ object RunnableUtils {
     * http://gcr.io/google.com/cloudsdktool/cloud-sdk
     */
   val CloudSdkImage: String =
-    config.getOrElse("cloud-sdk-image-url", "gcr.io/google.com/cloudsdktool/cloud-sdk:461.0.0-alpine")
+    config.getOrElse("cloud-sdk-image-url", "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine")
+
+  /**
+   * Batch sets a `CLOUDSDK_PYTHON` value that is incorrect for some images.
+   * Erase it so that Cloud SDK self-discovers the right Python within its image.
+   */
+  val CloudSdkEnvironment: Map[String, String] = Map("CLOUDSDK_PYTHON" -> "")
 
   /** Quotes a string such that it's compatible as a string argument in the shell. */
   def shellEscaped(any: Any): String = {