feat: Allow overriding download behavior using GCLOUD_TF_DOWNLOAD environment variable. (#59)

Author: morgante

README.md

@@ -28,6 +28,16 @@ Functional examples are included in the
 
 The [jq](https://stedolan.github.io/jq/) binary is also included in this module so you can use it as well for either of your `create_cmd_entrypoint` or `destroy_cmd_entrypoint` values.
 
+## Downloading
+This module will automatically download gcloud on your behalf,
+allowing you to run it in environments without a gcloud binary available.
+
+You can override this behavior by setting the `skip_download` variable to `true` or `false`.
+
+You can also override the behavior by setting the `GCLOUD_TF_DOWNLOAD` environment variable.
+This environment variable will override all other settings.
+Setting it to `never` will *never* gcloud download and setting it to `always` will always download gcloud.
+
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
@@ -58,6 +68,7 @@ The [jq](https://stedolan.github.io/jq/) binary is also included in this module
 | bin\_dir | The full bin path of the modules executables |
 | create\_cmd\_bin | The full bin path & command used on create |
 | destroy\_cmd\_bin | The full bin path & command used on destroy |
+| downloaded | Whether gcloud was downloaded or not |
 | wait | An output to use when you want to depend on cmd finishing |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

build/int.cloudbuild.yaml

@@ -23,16 +23,36 @@ steps:
   - 'TF_VAR_billing_account=$_BILLING_ACCOUNT'
 - id: create
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create default']
 - id: converge
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge']
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge default']
 - id: verify
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify']
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify default']
 - id: destroy
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy']
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy default']
+- id: create-skipped
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create skipped-env']
+  env:
+  - 'GCLOUD_TF_DOWNLOAD=never'
+- id: converge-skipped
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge skipped-env']
+  env:
+    - 'GCLOUD_TF_DOWNLOAD=never'
+- id: verify-skipped
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify skipped-env']
+  env:
+    - 'GCLOUD_TF_DOWNLOAD=never'
+- id: destroy-skipped
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy skipped-env']
+  env:
+    - 'GCLOUD_TF_DOWNLOAD=never'
 tags:
 - 'ci'
 - 'integration'

examples/simple_example/README.md

@@ -9,6 +9,12 @@ This example illustrates how to use the `gcloud` module.
 |------|-------------|:----:|:-----:|:-----:|
 | project\_id | The ID of the project in which to provision resources. | string | n/a | yes |
 
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| downloaded | Whether gcloud was downloaded or not |
+
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
 To provision this example, run the following from within this directory:

examples/simple_example/outputs.tf

@@ -13,3 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+output "downloaded" {
+  description = "Whether gcloud was downloaded or not"
+  value       = module.cli.downloaded
+}

kitchen.yml

@@ -24,6 +24,7 @@ verifier:
 
 platforms:
   - name: default
+  - name: skipped_env
 
 suites:
   - name: simple_example
@@ -37,6 +38,21 @@ suites:
           backend: local
           controls:
             - gcloud
+    excludes:
+      - skipped_env
+  - name: override_example
+    driver:
+      command_timeout: 1800
+      root_module_directory: test/fixtures/override_example/
+    verifier:
+      color: false
+      systems:
+        - name: override_example local
+          backend: local
+          controls:
+            - gcloud
+    excludes:
+      - default
   - name: script_example
     driver:
       command_timeout: 1800
@@ -48,6 +64,8 @@ suites:
           backend: local
           controls:
             - gcloud
+    excludes:
+      - skipped_env
   - name: dependency_example
     driver:
       command_timeout: 1800
@@ -59,6 +77,8 @@ suites:
           backend: local
           controls:
             - file
+    excludes:
+      - skipped_env
   - name: kubectl_wrapper_example
     driver:
       command_timeout: 3600
@@ -70,3 +90,5 @@ suites:
           backend: local
           controls:
             - kubectl
+    excludes:
+      - skipped_env

main.tf

@@ -16,19 +16,22 @@
 
 locals {
   tmp_credentials_path = "${path.module}/terraform-google-credentials.json"
-  cache_path           = "${path.module}/cache/${random_id.cache.hex}"
+  cache_path           = local.skip_download ? "" : "${path.module}/cache/${random_id.cache[0].hex}"
   gcloud_tar_path      = "${local.cache_path}/google-cloud-sdk.tar.gz"
   gcloud_bin_path      = "${local.cache_path}/google-cloud-sdk/bin"
   gcloud_bin_abs_path  = abspath(local.gcloud_bin_path)
   components           = join(",", var.additional_components)
 
-  gcloud              = var.skip_download ? "gcloud" : "${local.gcloud_bin_path}/gcloud"
+  download_override = data.external.env_override.result.download
+  skip_download     = local.download_override == "always" ? false : (local.download_override == "never" ? true : var.skip_download)
+
+  gcloud              = local.skip_download ? "gcloud" : "${local.gcloud_bin_path}/gcloud"
   gcloud_download_url = var.gcloud_download_url != "" ? var.gcloud_download_url : "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${var.gcloud_sdk_version}-${var.platform}-x86_64.tar.gz"
   jq_platform         = var.platform == "darwin" ? "osx-amd" : var.platform
   jq_download_url     = var.jq_download_url != "" ? var.jq_download_url : "https://github.com/stedolan/jq/releases/download/jq-${var.jq_version}/jq-${local.jq_platform}64"
 
-  create_cmd_bin  = var.skip_download ? var.create_cmd_entrypoint : "${local.gcloud_bin_path}/${var.create_cmd_entrypoint}"
-  destroy_cmd_bin = var.skip_download ? var.destroy_cmd_entrypoint : "${local.gcloud_bin_path}/${var.destroy_cmd_entrypoint}"
+  create_cmd_bin  = local.skip_download ? var.create_cmd_entrypoint : "${local.gcloud_bin_path}/${var.create_cmd_entrypoint}"
+  destroy_cmd_bin = local.skip_download ? var.destroy_cmd_entrypoint : "${local.gcloud_bin_path}/${var.destroy_cmd_entrypoint}"
 
   wait = length(null_resource.additional_components.*.triggers) + length(
     null_resource.gcloud_auth_service_account_key_file.*.triggers,
@@ -41,7 +44,7 @@ locals {
   decompress_command                           = "tar -xzf ${local.gcloud_tar_path} -C ${local.cache_path} && cp ${local.cache_path}/jq ${local.cache_path}/google-cloud-sdk/bin/"
   decompress_wrapper                           = fileexists(local.gcloud_tar_path) ? local.decompress_command : "${local.prepare_cache_command} && ${local.download_gcloud_command} && ${local.download_jq_command} && ${local.decompress_command}"
   upgrade_command                              = "${local.gcloud} components update --quiet"
-  additional_components_command                = "${path.module}/check_components.sh ${local.gcloud} ${local.components}"
+  additional_components_command                = "${path.module}/scripts/check_components.sh ${local.gcloud} ${local.components}"
   gcloud_auth_service_account_key_file_command = "${local.gcloud} auth activate-service-account --key-file ${var.service_account_key_file}"
   gcloud_auth_google_credentials_command       = <<-EOT
     printf "%s" "$GOOGLE_CREDENTIALS" > ${local.tmp_credentials_path} &&
@@ -51,6 +54,8 @@ locals {
 }
 
 resource "random_id" "cache" {
+  count = (! local.skip_download) ? 1 : 0
+
   byte_length = 4
 }
 
@@ -62,8 +67,14 @@ resource "null_resource" "module_depends_on" {
   }
 }
 
+data "external" "env_override" {
+  program = ["${path.module}/scripts/check_env.sh"]
+
+  query = {}
+}
+
 resource "null_resource" "prepare_cache" {
-  count = (var.enabled && ! var.skip_download) ? 1 : 0
+  count = (var.enabled && ! local.skip_download) ? 1 : 0
 
   triggers = merge({
     md5                   = md5(var.create_cmd_entrypoint)
@@ -80,7 +91,7 @@ resource "null_resource" "prepare_cache" {
 }
 
 resource "null_resource" "download_gcloud" {
-  count = (var.enabled && ! var.skip_download) ? 1 : 0
+  count = (var.enabled && ! local.skip_download) ? 1 : 0
 
   triggers = merge({
     md5                     = md5(var.create_cmd_entrypoint)
@@ -97,7 +108,7 @@ resource "null_resource" "download_gcloud" {
 }
 
 resource "null_resource" "download_jq" {
-  count = (var.enabled && ! var.skip_download) ? 1 : 0
+  count = (var.enabled && ! local.skip_download) ? 1 : 0
 
   triggers = merge({
     md5                 = md5(var.create_cmd_entrypoint)
@@ -114,7 +125,7 @@ resource "null_resource" "download_jq" {
 }
 
 resource "null_resource" "decompress" {
-  count = (var.enabled && ! var.skip_download) ? 1 : 0
+  count = (var.enabled && ! local.skip_download) ? 1 : 0
 
   triggers = merge({
     md5                     = md5(var.create_cmd_entrypoint)
@@ -133,7 +144,7 @@ resource "null_resource" "decompress" {
 }
 
 resource "null_resource" "upgrade" {
-  count = (var.enabled && var.upgrade && ! var.skip_download) ? 1 : 0
+  count = (var.enabled && var.upgrade && ! local.skip_download) ? 1 : 0
 
   depends_on = [null_resource.decompress]
 
@@ -279,7 +290,7 @@ resource "null_resource" "additional_components_destroy" {
 }
 
 resource "null_resource" "upgrade_destroy" {
-  count = (var.enabled && var.upgrade && ! var.skip_download) ? 1 : 0
+  count = (var.enabled && var.upgrade && ! local.skip_download) ? 1 : 0
 
   depends_on = [
     null_resource.additional_components_destroy,
@@ -298,7 +309,7 @@ resource "null_resource" "upgrade_destroy" {
 }
 
 resource "null_resource" "decompress_destroy" {
-  count      = (var.enabled && ! var.skip_download) ? 1 : 0
+  count      = (var.enabled && ! local.skip_download) ? 1 : 0
   depends_on = [null_resource.upgrade_destroy]
 
   triggers = {

outputs.tf

@@ -33,3 +33,9 @@ output "wait" {
   description = "An output to use when you want to depend on cmd finishing"
   value       = local.wait
 }
+
+output "downloaded" {
+  description = "Whether gcloud was downloaded or not"
+  value       = ! local.skip_download
+  depends_on  = [local.wait]
+}

check_components.sh scripts/check_components.shcheck_components.sh renamed to scripts/check_components.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+SKIP=${GCLOUD_TF_DOWNLOAD}
+
+echo "{\"download\": \"${SKIP}\"}"

scripts/check_env.sh

@@ -0,0 +1,21 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "example" {
+  source = "../../../examples/simple_example"
+
+  project_id = var.project_id
+}