Start refactor to remove "generated-at-runtime" tf workspace files

Author: ceason

terraform/container.bzl

@@ -12,8 +12,6 @@ image_publisher = _image_publisher
 
 def _image_embedder_impl(ctx):
     providers = []
-    runfiles = []
-    transitive_runfiles = []
 
     out = ctx.actions.declare_file("%s.%s" % (ctx.attr.name, ctx.file.src.extension))
     providers.append(_embed_images(
@@ -23,21 +21,36 @@ def _image_embedder_impl(ctx):
         output_format = ctx.file.src.extension,
     ))
 
+    tar = ctx.actions.declare_file(ctx.attr.name + ".tar")
+    bundle_args = ctx.actions.args()
+    bundle_args.add("--output", tar)
+    bundle_args.add("--file", [out.basename, out])
+    ctx.actions.run(
+        inputs = [out],
+        outputs = [tar],
+        arguments = [bundle_args],
+        executable = ctx.executable._bundle_tool,
+    )
+
     return providers + [
-        _ModuleInfo(files = {out.basename: out}),
+        _ModuleInfo(
+            files = {out.basename: out},
+            tar = tar,
+        ),
         DefaultInfo(
             files = depset(direct = [out]),
-            runfiles = ctx.runfiles(
-                files = runfiles,
-                transitive_files = depset(transitive = transitive_runfiles),
-            ),
         ),
     ]
 
 _image_embedder = rule(
     implementation = _image_embedder_impl,
     attrs = _image_embedder_attrs + {
         "src": attr.label(allow_single_file = [".yaml", ".json", ".yml"]),
+        "_bundle_tool": attr.label(
+            default = Label("//terraform/internal:bundle"),
+            executable = True,
+            cfg = "host",
+        ),
     },
 )
 

terraform/dependencies.bzl

@@ -8,8 +8,8 @@ _EXTERNAL_BINARIES = {
         version = "1.11.1",
     ),
     "terraform-docs": dict(
-        url = "https://github.com/segmentio/terraform-docs/releases/download/v{version}/terraform-docs_{platform}_amd64",
-        version = "0.3.0",
+        url = "https://github.com/segmentio/terraform-docs/releases/download/{version}/terraform-docs-{version}-{platform}-amd64",
+        version = "v0.5.0",
     ),
     "terraform": dict(
         url = "https://releases.hashicorp.com/terraform/{version}/terraform_{version}_{platform}_amd64.zip",

terraform/internal/BUILD

@@ -10,6 +10,18 @@ py_binary(
     visibility = ["//visibility:public"],
 )
 
+py_binary(
+    name = "render_workspace",
+    srcs = ["render_workspace.py"],
+    visibility = ["//visibility:public"],
+)
+
+py_binary(
+    name = "bundle",
+    srcs = ["bundle.py"],
+    visibility = ["//visibility:public"],
+)
+
 py_binary(
     name = "stamper",
     srcs = ["stamper.py"],
@@ -32,3 +44,10 @@ py_binary(
     visibility = ["//visibility:public"],
     deps = ["@yaml"],
 )
+
+py_binary(
+    name = "k8s_manifest",
+    srcs = ["k8s_manifest.py"],
+    visibility = ["//visibility:public"],
+    deps = ["@yaml"],
+)

terraform/internal/bundle.py

@@ -0,0 +1,76 @@
+import argparse
+import collections
+import os
+import tarfile
+
+parser = argparse.ArgumentParser(
+    fromfile_prefix_chars='@',
+    description='Bundle terraform files into an archive')
+
+parser.add_argument(
+    '--file', action='append', metavar=('tgt_path', 'src'), nargs=2, default=[],
+    help="'src' file will be added to 'tgt_path'")
+
+parser.add_argument(
+    '--embed', action='append', metavar=('embed_path', 'src_tar'), nargs=2, default=[],
+    help="'src' archive will be embedded in 'embed_path'. If 'embed_path=.' then archive content will be merged into "
+         "the output root")
+
+parser.add_argument(
+    '--output', action='store', required=True,
+    help="Output path of bundled archive")
+
+BundleItem = collections.namedtuple('BundleItem', 'tarinfo file')
+
+
+class Bundle:
+
+    def __init__(self, output):
+        # map of paths to BundleItems
+        self._file_map = {}
+        self._output = tarfile.open(output, "w")
+
+    def add(self, src, arcname):
+        f = open(os.path.realpath(src), 'r')
+        tarinfo = self._output.gettarinfo(arcname=arcname, fileobj=f)
+        if self._file_map.has_key(tarinfo.name):
+            raise ValueError("File '%s' is already in archive" % tarinfo.name)
+        tarinfo.mtime = 0  # zero out modification time
+        self._file_map[tarinfo.name] = BundleItem(tarinfo, f)
+
+    def embed(self, archive, embed_path):
+        tar = tarfile.open(archive)
+        for tarinfo in tar.getmembers():
+            f = tar.extractfile(tarinfo)
+            if embed_path != ".":
+                tarinfo.name = embed_path + "/" + tarinfo.name
+            if self._file_map.has_key(tarinfo.name):
+                raise ValueError("File '%s' is already in archive" % tarinfo.name)
+            self._file_map[tarinfo.name] = BundleItem(tarinfo, f)
+
+    def finish(self):
+        for path in sorted(self._file_map.keys()):
+            tarinfo, f = self._file_map[path]
+            self._output.addfile(tarinfo, fileobj=f)
+
+
+def main(args):
+    """
+
+    :return:
+    """
+
+    # output = tarfile.open(args.output, "w")
+    bundle = Bundle(args.output)
+
+    # add each args.file
+    for tgt_path, src in args.file:
+        bundle.add(src, tgt_path)
+    # embed each args.embed
+    for embed_path, src_tar in args.embed:
+        bundle.embed(src_tar, embed_path)
+
+    bundle.finish()
+
+if __name__ == '__main__':
+    main(parser.parse_args())

terraform/internal/image_embedder.py

@@ -3,6 +3,7 @@
 import argparse
 import io
 import json
+import logging
 import os
 from collections import namedtuple
 
@@ -94,7 +95,6 @@ def publish(self, transport, threads=_THREADS):
             with v2_2_image.FromDisk(self._config, self._layers, legacy_base=self._tarball) as image:
                 # todo: output more friendly error message when this raises an exception
                 session.upload(image)
-                print("Successfully published image '%s@%s'" % (self._name_to_publish, image.digest()))
 
     def name_to_embed(self):
         with v2_2_image.FromDisk(self._config, self._layers, legacy_base=self._tarball) as image:
@@ -153,13 +153,13 @@ def walk(o):
         outputs.extend(map(walk, yaml.load_all(input_str)))
 
     if len(outputs) == 0:
-        print("ERROR: Nothing to resolve (Are you sure the input has valid json/yaml objects?)")
+        logging.fatal("Nothing to resolve (Are you sure the input has valid json/yaml objects?)")
         sys.exit(1)
 
     if len(unseen_strings) > 0:
-        print('ERROR: The following image references were not found:', file=sys.stderr)
-        for ref in unseen_strings:
-            print('    %s' % ref, file=sys.stderr)
+        msg = 'The following image references were not found:\n    '
+        msg = msg + "\n    ".join(unseen_strings)
+        logging.fatal(msg)
         sys.exit(1)
 
     output_content = io.BytesIO()
@@ -192,6 +192,7 @@ def publish(args):
 
 
 def main():
+    logging.basicConfig(stream=sys.stderr, level=logging.INFO)
     args = parser.parse_args()
 
     if args.command == "embed":

terraform/internal/image_embedder_lib.bzl

@@ -111,12 +111,13 @@ def create_image_publisher(ctx, output, aspect_targets):
     image_specs = []
 
     for t in aspect_targets:
-        targets = t[PublishableTargetsInfo].targets
-        if targets != None:
-            for target in targets.to_list():
-                info = target[ImagePublishInfo]
-                transitive_runfiles.append(info.runfiles)
-                image_specs.extend(info.image_specs)
+        if PublishableTargetsInfo in t:
+            targets = t[PublishableTargetsInfo].targets
+            if targets != None:
+                for target in targets.to_list():
+                    info = target[ImagePublishInfo]
+                    transitive_runfiles.append(info.runfiles)
+                    image_specs.extend(info.image_specs)
 
     # dedupe image specs
     image_specs = {k: None for k in image_specs}.keys()

terraform/internal/integration_test.bzl

@@ -0,0 +1,65 @@
+load("//terraform:providers.bzl", "WorkspaceInfo", "tf_workspace_files_prefix")
+load("//terraform/internal:image_embedder_lib.bzl", "create_image_publisher", "image_publisher_aspect", "image_publisher_attrs")
+
+def _integration_test_impl(ctx):
+    """
+    """
+
+    runfiles = []
+    transitive_runfiles = []
+
+    transitive_runfiles.append(ctx.attr._runner_template.data_runfiles.files)
+    transitive_runfiles.append(ctx.attr._stern.data_runfiles.files)
+    transitive_runfiles.append(ctx.attr.srctest.data_runfiles.files)
+    transitive_runfiles.append(ctx.attr.terraform_workspace.data_runfiles.files)
+    render_workspace = ctx.attr.terraform_workspace[WorkspaceInfo].render_workspace
+
+    ctx.actions.expand_template(
+        template = ctx.file._runner_template,
+        substitutions = {
+            "%{render_workspace}": render_workspace.short_path,
+            "%{srctest}": ctx.executable.srctest.short_path,
+            "%{stern}": ctx.executable._stern.short_path,
+        },
+        output = ctx.outputs.executable,
+        is_executable = True,
+    )
+
+    return [DefaultInfo(
+        runfiles = ctx.runfiles(
+            files = runfiles,
+            transitive_files = depset(transitive = transitive_runfiles),
+        ),
+    )]
+
+# Wraps the source test with infrastructure spinup and teardown
+terraform_integration_test = rule(
+    test = True,
+    implementation = _integration_test_impl,
+    attrs = image_publisher_attrs + {
+        "terraform_workspace": attr.label(
+            doc = "TF Workspace to spin up before testing & tear down after testing.",
+            mandatory = True,
+            executable = True,
+            cfg = "host",
+            providers = [WorkspaceInfo],
+            aspects = [image_publisher_aspect],
+        ),
+        "srctest": attr.label(
+            doc = "Label of source test to wrap",
+            mandatory = True,
+            executable = True,
+            cfg = "target",  # 'host' does not work for jvm source tests, because it launches with @embedded_jdk//:jar instead of @local_jdk//:jar
+            aspects = [image_publisher_aspect],
+        ),
+        "_runner_template": attr.label(
+            default = "//terraform/internal:integration_test_runner.sh.tpl",
+            allow_single_file = True,
+        ),
+        "_stern": attr.label(
+            executable = True,
+            cfg = "host",
+            default = "@tool_stern",
+        ),
+    },
+)

terraform/internal/integration_test_runner.sh.tpl

@@ -5,6 +5,9 @@ err_report() { echo "errexit on line $(caller)" >&2; }
 trap err_report ERR
 
 export RUNFILES=${RUNFILES_DIR}
+# guess the kubeconfig location if it isn't already set
+: ${KUBECONFIG:="/Users/$USER/.kube/config:/home/$USER/.kube/config"}
+export KUBECONFIG
 
 # register cleanup traps here, then execute them on EXIT!
 ITS_A_TRAP=()
@@ -23,43 +26,19 @@ cleanup(){
 }
 trap cleanup EXIT
 
-render_tf="%{render_tf}"
+render_workspace="%{render_workspace}"
 stern="$PWD/%{stern}"
 SRCTEST="%{srctest}"
-tf_workspace_files_prefix="%{tf_workspace_files_prefix}"
-PRETEST_PUBLISHERS=(%{pretest_publishers})
 
-# run pretest publishers (eg docker image publisher)
-for publisher in "${PRETEST_PUBLISHERS[@]}"; do
-	"$publisher"
-done
-
-mkdir -p "$tf_workspace_files_prefix"
-
-: ${TMPDIR:=/tmp}
-: ${TF_PLUGIN_CACHE_DIR:=$TMPDIR/rules_terraform/plugin-cache}
-export TF_PLUGIN_CACHE_DIR
-mkdir -p "$TF_PLUGIN_CACHE_DIR"
-
-# guess the kubeconfig location if it isn't already set
-: ${KUBECONFIG:="/Users/$USER/.kube/config:/home/$USER/.kube/config"}
-export KUBECONFIG
-
-# render the tf to a tempdir
-tfroot=$TEST_TMPDIR/tf/tfroot
-tfplan=$TEST_TMPDIR/tf/tfplan
-tfstate=$TEST_TMPDIR/tf/tfstate.json
-rm -rf "$TEST_TMPDIR/tf"
-mkdir -p "$TEST_TMPDIR/tf"
-chmod 700 $(dirname "$tfstate")
-"$render_tf" --output_dir "$tfroot" --plugin_dir "$tf_workspace_files_prefix/.terraform/plugins" --symlink_plugins
+# render the tf
+terraform=$("$render_workspace" --symlink_plugins "$PWD/.rules_terraform")
 
 # init and validate terraform
-pushd "$tf_workspace_files_prefix" > /dev/null
-timeout 20 terraform init -input=false "$tfroot"
-timeout 20 terraform validate "$tfroot"
+timeout 20 "$terraform" init -input=false
+timeout 20 "$terraform" validate
+
 # if the kubectl provider is used then create a namespace for the test
-if [ "$(find .terraform/plugins/ -type f \( -name 'terraform-provider-kubernetes_*' -o -name 'terraform-provider-kubectl_*' \)|wc -l)" -gt 0 ]; then
+if [ "$(find .rules_terraform/.terraform/plugins/ -type f \( -name 'terraform-provider-kubernetes_*' -o -name 'terraform-provider-kubectl_*' \)|wc -l)" -gt 0 ]; then
 	kubectl config view --merge --raw --flatten > "$TEST_TMPDIR/kubeconfig.yaml"
 	ITS_A_TRAP+=("rm -rf '$TEST_TMPDIR/kubeconfig.yaml'")
 	kube_context=$(kubectl config current-context)
@@ -71,14 +50,10 @@ if [ "$(find .terraform/plugins/ -type f \( -name 'terraform-provider-kubernetes
 	# tail stuff with stern in the background
 	"$stern" '.*' --tail 1 --color always &
 fi
-timeout 20 terraform plan -out="$tfplan" -input=false "$tfroot"
-popd > /dev/null
 
 # apply the terraform
-ITS_A_TRAP+=("cd '$PWD/$tf_workspace_files_prefix' && terraform destroy -state='$tfstate' -auto-approve -refresh=false")
-pushd "$tf_workspace_files_prefix" > /dev/null
-terraform apply -state-out="$tfstate" -auto-approve "$tfplan"
-popd > /dev/null
+ITS_A_TRAP+=("$terraform destroy -auto-approve -refresh=false")
+"$terraform" apply -input=false -auto-approve
 
 # run the test & await its completion
 "$SRCTEST" "$@"