diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 717218318be..e472df2cc87 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -40,6 +40,9 @@ jobs: - build_type: centaurGcpBatchRestart build_mysql: 8.4 friendly_name: Centaur GCP Batch (restart) + - build_type: centaurGcpBatchRestartCallCaching + build_mysql: 8.4 + friendly_name: Centaur GCP Batch (restart, call caching) - build_type: dbms friendly_name: DBMS - build_type: centaurTes diff --git a/centaur/src/main/resources/standardTestCases/dummy_backend/35k_scatter_success.test b/centaur/src/main/resources/standardTestCases/dummy_backend/35k_scatter_success.test index 482d380584d..1d29b88354d 100644 --- a/centaur/src/main/resources/standardTestCases/dummy_backend/35k_scatter_success.test +++ b/centaur/src/main/resources/standardTestCases/dummy_backend/35k_scatter_success.test @@ -14,4 +14,4 @@ metadata { "outputs.dummy_scatter.results_count": 35000 } -maximumTime = 12 minutes +maximumTime = 15 minutes diff --git a/docs/backends/GCPBatch.md b/docs/backends/GCPBatch.md index 52c98dec862..5c7594d97e7 100644 --- a/docs/backends/GCPBatch.md +++ b/docs/backends/GCPBatch.md @@ -341,13 +341,14 @@ network labels, and then fall back to running on the default network. ### Custom Google Cloud SDK container -Cromwell can't use Google's container registry if VPC Perimeter is used in project. -Own repository can be used by adding `cloud-sdk-image-url` reference to used container: +Cromwell uses `cloud-sdk:alpine` by default. If you require a static Cloud SDK version, set a custom value like `cloud-sdk:584.0.0-alpine` for `cloud-sdk-image-url`. Note that GCP [deletes tags after 1 year](https://github.com/GoogleCloudPlatform/cloud-sdk-docker#package-retention-policy), so any custom tag requires recurring updates. + +Cromwell can't use Google's container registry if VPC Perimeter is used in project. Work around by re-hosting within the project and setting `cloud-sdk-image-url`. ``` google { ... - cloud-sdk-image-url = "eu.gcr.io/your-project-id/cloudsdktool/cloud-sdk:354.0.0-alpine" + cloud-sdk-image-url = "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine" cloud-sdk-image-size-gb = 1 } ``` diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index c4686130d7f..b220e4409b9 100755 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -190,6 +190,9 @@ cromwell::private::create_build_variables() { centaurHoricromtalGcpBatch*) CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/gcp_batch_horicromtal_application.conf" ;; + centaurGcpBatchRestartCallCaching) + CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/gcp_batch_restart_application.conf" + ;; *) CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_BUILD_BACKEND_TYPE}_application.conf" ;; diff --git a/src/ci/bin/testCentaurGcpBatchRestart.sh b/src/ci/bin/testCentaurGcpBatchRestart.sh index 74d153be51b..72696b3dd2d 100755 --- a/src/ci/bin/testCentaurGcpBatchRestart.sh +++ b/src/ci/bin/testCentaurGcpBatchRestart.sh @@ -12,6 +12,9 @@ cromwell::build::setup_common_environment cromwell::build::setup_centaur_environment +# No DRS tests in this suite. Set a placeholder image so we do not spend time building the DRS localizer. +export CROMWELL_BUILD_BATCH_DOCKER_IMAGE_DRS="mirror.gcr.io/almalinux:latest" + cromwell::build::batch::setup_batch_centaur_environment cromwell::build::assemble_jars @@ -19,5 +22,6 @@ cromwell::build::assemble_jars cromwell::build::run_centaur \ -p 100 \ -i restart \ + -e call_cache_cha_cha_batch \ cromwell::build::generate_code_coverage diff --git a/src/ci/bin/testCentaurGcpBatchRestartCallCaching.sh b/src/ci/bin/testCentaurGcpBatchRestartCallCaching.sh new file mode 100755 index 00000000000..f9c47f33369 --- /dev/null +++ b/src/ci/bin/testCentaurGcpBatchRestartCallCaching.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail +export CROMWELL_BUILD_REQUIRES_SECURE=true +# import in shellcheck / CI / IntelliJ compatible ways +# shellcheck source=/dev/null +source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh +# shellcheck source=/dev/null +source "${BASH_SOURCE%/*}/test_gcpbatch.inc.sh" || source test_gcpbatch.inc.sh + +cromwell::build::setup_common_environment + +cromwell::build::setup_centaur_environment + +# No DRS tests in this suite. Set a placeholder image so we do not spend time building the DRS localizer. +export CROMWELL_BUILD_BATCH_DOCKER_IMAGE_DRS="mirror.gcr.io/almalinux:latest" + +cromwell::build::batch::setup_batch_centaur_environment + +cromwell::build::assemble_jars + +# Split out of testCentaurGcpBatchRestart.sh, which excludes this test. Restart tests are +# serialized because they kill and restart the shared Cromwell, and this one runs about as +# long as the other three combined. +cromwell::build::run_centaur \ + -p 100 \ + -i call_cache_cha_cha_batch \ + +cromwell::build::generate_code_coverage diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/CheckpointingRunnable.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/CheckpointingRunnable.scala index 081cbc1ee52..70a033515fd 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/CheckpointingRunnable.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/CheckpointingRunnable.scala @@ -12,7 +12,7 @@ trait CheckpointingRunnable { createParameters.checkpointingConfiguration.checkpointingCommand(checkpointFilename, RunnableCommands.multiLineBinBashCommand ) - val checkpointingEnvironment = Map.empty[String, String] + val checkpointingEnvironment = RunnableUtils.CloudSdkEnvironment // Initial sync from cloud: val initialCheckpointSyncRunnable = RunnableBuilder.cloudSdkShellRunnable( diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/Delocalization.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/Delocalization.scala index 4e1ab74104b..f2febc71a8a 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/Delocalization.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/Delocalization.scala @@ -35,6 +35,7 @@ trait Delocalization { RunnableBuilder .withImage(womOutputRuntimeExtractor.dockerImage.getOrElse(CloudSdkImage)) + .withEnvironment(CloudSdkEnvironment) .withCommand(commands: _*) .withEntrypointCommand("/bin/bash") .withLabels(Map(Key.Tag -> Value.Delocalization)) diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableBuilder.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableBuilder.scala index 4b6078dc3a7..ac750666832 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableBuilder.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableBuilder.scala @@ -235,7 +235,8 @@ object RunnableBuilder extends BatchUtilityConversions { timeout = 30.minutes ) - def cloudSdkRunnable: Runnable.Builder = Runnable.newBuilder.setContainer(cloudSdkContainerBuilder) + def cloudSdkRunnable: Runnable.Builder = + Runnable.newBuilder.setContainer(cloudSdkContainerBuilder).withEnvironment(CloudSdkEnvironment) // Set a default timeout of 24 hours for these runnables. They are typically used for running // localization, delocalization, small shell commands, etc. These processes occasionally hang and @@ -247,6 +248,7 @@ object RunnableBuilder extends BatchUtilityConversions { ): Runnable.Builder = Runnable.newBuilder .setContainer(cloudSdkContainerBuilder) + .withEnvironment(CloudSdkEnvironment) .withVolumes(volumes) .withLabels(labels) .withEntrypointCommand( diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableUtils.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableUtils.scala index 5af320ada8e..ebbe04409b0 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableUtils.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/runnable/RunnableUtils.scala @@ -16,7 +16,13 @@ object RunnableUtils { * http://gcr.io/google.com/cloudsdktool/cloud-sdk */ val CloudSdkImage: String = - config.getOrElse("cloud-sdk-image-url", "gcr.io/google.com/cloudsdktool/cloud-sdk:461.0.0-alpine") + config.getOrElse("cloud-sdk-image-url", "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine") + + /** + * Batch sets a `CLOUDSDK_PYTHON` value that is incorrect for some images. + * Erase it so that Cloud SDK self-discovers the right Python within its image. + */ + val CloudSdkEnvironment: Map[String, String] = Map("CLOUDSDK_PYTHON" -> "") /** Quotes a string such that it's compatible as a string argument in the shell. */ def shellEscaped(any: Any): String = {