diff --git a/examples/downstream/versions.tf b/examples/downstream/versions.tf index 98af46d..60809f7 100644 --- a/examples/downstream/versions.tf +++ b/examples/downstream/versions.tf @@ -1,10 +1,6 @@ terraform { required_version = ">= 1.5.0" required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.5" - } random = { source = "hashicorp/random" version = ">= 3.5.1" diff --git a/examples/downstream_splitrole/versions.tf b/examples/downstream_splitrole/versions.tf index 98af46d..60809f7 100644 --- a/examples/downstream_splitrole/versions.tf +++ b/examples/downstream_splitrole/versions.tf @@ -1,10 +1,6 @@ terraform { required_version = ">= 1.5.0" required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.5" - } random = { source = "hashicorp/random" version = ">= 3.5.1" diff --git a/examples/one/versions.tf b/examples/one/versions.tf index 98af46d..60809f7 100644 --- a/examples/one/versions.tf +++ b/examples/one/versions.tf @@ -1,10 +1,6 @@ terraform { required_version = ">= 1.5.0" required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.5" - } random = { source = "hashicorp/random" version = ">= 3.5.1" diff --git a/examples/prod/versions.tf b/examples/prod/versions.tf index 98af46d..60809f7 100644 --- a/examples/prod/versions.tf +++ b/examples/prod/versions.tf @@ -1,10 +1,6 @@ terraform { required_version = ">= 1.5.0" required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.5" - } random = { source = "hashicorp/random" version = ">= 3.5.1" diff --git a/examples/three/main.tf b/examples/three/main.tf index 28d470e..383a073 100644 --- a/examples/three/main.tf +++ b/examples/three/main.tf @@ -64,20 +64,20 @@ locals { indirect_access = true initial = true } - # "rancherB" = { - # type = "all-in-one" - # size = "xxl" - # os = local.os - # indirect_access = true - # initial = false - # } - # "rancherC" = { - # type = "all-in-one" - # size = "xxl" - # os = local.os - # indirect_access = true - # initial = false - # } + "rancherB" = { + type = "all-in-one" + size = "xxl" + os = local.os + indirect_access = true + initial = false + } + "rancherC" = { + type = "all-in-one" + size = "xxl" + os = local.os + indirect_access = true + initial = false + } } local_file_path = var.file_path runner_ip = chomp(data.http.myip.response_body) # "runner" is the server running Terraform diff --git a/examples/three/modules/tls/versions.tf b/examples/three/modules/tls/versions.tf index b3a9a80..28763e7 100644 --- a/examples/three/modules/tls/versions.tf +++ b/examples/three/modules/tls/versions.tf @@ -1,10 +1,6 @@ terraform { required_version = ">= 1.5.0" required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.5" - } tls = { source = "hashicorp/tls" version = ">= 4.0.5" diff --git a/flake.lock b/flake.lock index b340353..032e7aa 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1753151930, - "narHash": "sha256-XSQy6wRKHhRe//iVY5lS/ZpI/Jn6crWI8fQzl647wCg=", + "lastModified": 1759322190, + "narHash": "sha256-s+0wBPx9FAphKv8BYRN7OLCiZkiK1Dc8ebbCzczI9S8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "83e677f31c84212343f4cc553bab85c2efcad60a", + "rev": "ee4cd98afba682ca27242f2d7fdbd8583f6ef51a", "type": "github" }, "original": { diff --git a/modules/cluster/main.tf b/modules/cluster/main.tf index 9fc9a10..22d9bdf 100644 --- a/modules/cluster/main.tf +++ b/modules/cluster/main.tf @@ -136,10 +136,39 @@ module "deploy_initial_node" { depends_on = [ data.aws_availability_zones.available, ] - for_each = local.initial_node - deploy_path = each.value.deploy_path - data_path = each.value.deploy_path - template_path = "${path.module}/node_template" + for_each = local.initial_node + deploy_path = each.value.deploy_path + data_path = each.value.deploy_path + # if any of this changes, update/redeploy + deploy_trigger = md5(join("-", [ + each.key, + md5(base64encode(jsonencode(each.value))), + local.identifier, + local.owner, + local.acme_server_url, + local.project_name, + local.ip_family, + md5(base64encode(jsonencode(data.aws_availability_zones.available.names))), + md5(base64encode(jsonencode(local.project_subnet_names))), + md5(base64encode(jsonencode(local.project_load_balancer_access_cidrs))), + local.domain, + local.zone, + local.skip_cert, + data.aws_availability_zones.available.names[0], + md5(base64encode(jsonencode(values(local.target_groups)))), + md5(base64encode(jsonencode(local.server_access_addresses))), + local.username, + local.ssh_key, + local.install_method, + local.download, + local.rke2_version, + ])) + template_files = [ + join("/", [path.module, "node_template", "main.tf"]), + join("/", [path.module, "node_template", "outputs.tf"]), + join("/", [path.module, "node_template", "variables.tf"]), + join("/", [path.module, "node_template", "versions.tf"]), + ] inputs = <<-EOT identifier = "${local.identifier}" owner = "${local.owner}" @@ -229,10 +258,39 @@ module "deploy_additional_nodes" { data.aws_availability_zones.available, module.deploy_initial_node, ] - for_each = local.additional_nodes - deploy_path = each.value.deploy_path - data_path = each.value.deploy_path - template_path = "${path.module}/node_template" + for_each = local.additional_nodes + deploy_path = each.value.deploy_path + data_path = each.value.deploy_path + # if any of this changes, update/redeploy + deploy_trigger = md5(join("-", [ + each.key, + md5(base64encode(jsonencode(each.value))), + local.identifier, + local.owner, + local.acme_server_url, + local.project_name, + local.ip_family, + md5(base64encode(jsonencode(data.aws_availability_zones.available.names))), + md5(base64encode(jsonencode(local.project_subnet_names))), + md5(base64encode(jsonencode(local.project_load_balancer_access_cidrs))), + local.domain, + local.zone, + local.skip_cert, + data.aws_availability_zones.available.names[0], + md5(base64encode(jsonencode(values(local.target_groups)))), + md5(base64encode(jsonencode(local.server_access_addresses))), + local.username, + local.ssh_key, + local.install_method, + local.download, + local.rke2_version, + ])) + template_files = [ + join("/", [path.module, "node_template", "main.tf"]), + join("/", [path.module, "node_template", "outputs.tf"]), + join("/", [path.module, "node_template", "variables.tf"]), + join("/", [path.module, "node_template", "versions.tf"]), + ] inputs = <<-EOT identifier = "${local.identifier}" owner = "${local.owner}" @@ -311,11 +369,12 @@ strcontains(each.value.type, "database") ? local.database_config : EOT } -resource "local_sensitive_file" "kubeconfig" { +resource "file_local" "kubeconfig" { depends_on = [ module.deploy_initial_node, module.deploy_additional_nodes, ] - content = local.ino.output.kubeconfig - filename = "${local.local_file_path}/kubeconfig" + name = "kubeconfig" + directory = local.local_file_path + contents = local.ino.output.kubeconfig } diff --git a/modules/cluster/node_template/versions.tf b/modules/cluster/node_template/versions.tf index b5ed9df..9f38064 100644 --- a/modules/cluster/node_template/versions.tf +++ b/modules/cluster/node_template/versions.tf @@ -1,10 +1,6 @@ terraform { required_version = ">= 1.5.0" required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.5" - } random = { source = "hashicorp/random" version = ">= 3.5.1" diff --git a/modules/cluster/versions.tf b/modules/cluster/versions.tf index e065e62..46fd573 100644 --- a/modules/cluster/versions.tf +++ b/modules/cluster/versions.tf @@ -1,9 +1,9 @@ terraform { required_version = ">= 1.5.0" required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.5" + file = { + source = "rancher/file" + version = ">= 1.1" } random = { source = "hashicorp/random" diff --git a/modules/deploy/create.sh.tpl b/modules/deploy/create.sh.tpl index a68ff92..fc61795 100644 --- a/modules/deploy/create.sh.tpl +++ b/modules/deploy/create.sh.tpl @@ -1,6 +1,5 @@ -${export_contents} cd ${deploy_path} -export TF_DATA_DIR="${tf_data_dir}" +source envrc TF_CLI_ARGS_init="" TF_CLI_ARGS_apply="" diff --git a/modules/deploy/destroy.sh.tpl b/modules/deploy/destroy.sh.tpl index 66f912c..f47f5d9 100644 --- a/modules/deploy/destroy.sh.tpl +++ b/modules/deploy/destroy.sh.tpl @@ -1,8 +1,7 @@ -${export_contents} cd ${deploy_path} +source envrc TF_CLI_ARGS_init="" TF_CLI_ARGS_apply="" -export TF_DATA_DIR="${tf_data_dir}" if [ -z "${skip_destroy}" ]; then timeout -k 1m ${timeout} terraform init -upgrade -reconfigure timeout -k 1m ${timeout} terraform destroy -var-file="${deploy_path}/inputs.tfvars" -auto-approve -state="${deploy_path}/tfstate" || true diff --git a/modules/deploy/main.tf b/modules/deploy/main.tf index caf3371..a3be460 100644 --- a/modules/deploy/main.tf +++ b/modules/deploy/main.tf @@ -3,134 +3,270 @@ # I felt this was the best way to accomplish the goal without incurring additional dependencies locals { - inputs = var.inputs - inputs_hash = md5(local.inputs) - template_path = var.template_path - template_files = var.template_files - # tflint-ignore: terraform_unused_declarations - fail_no_template = ((local.template_path == null && length(local.template_files) == 0) ? one([local.template_path, "missing_template"]) : false) - # tflint-ignore: terraform_unused_declarations - fail_too_much_template = ((local.template_path != null && length(local.template_files) > 0) ? one([local.template_path, "template_path_or_template_files"]) : false) - template_file_list = ( - local.template_path != null ? - [ - for i in range(length(fileset(local.template_path, "**"))) : - join("/", [local.template_path, tolist(fileset(local.template_path, "**"))[i]]) - ] - : local.template_files - ) - template_file_map = { for file in local.template_file_list : basename(file) => file } - template_files_hash = md5(join("-", local.template_file_list)) - deploy_path = chomp(var.deploy_path) + template_files = var.template_files + template_file_map = { for file in local.template_files : basename(file) => file } - environment_variables = var.environment_variables + # template_file_map = { for i in range(length(local.template_files)) : tostring(i) => local.template_files[i] } + # need to figure out how to copy the files sent, the for_each loop won't work due to the dynamic read of the directory + inputs = var.inputs + environment_variables = merge(var.environment_variables, { "TF_DATA_DIR" = local.tf_data_dir }) export_contents = ( local.environment_variables != null ? join(";", [for k, v in local.environment_variables : "export ${k}=${v}"]) : "" ) - export_hash = md5(local.export_contents) - attempts = var.attempts - interval = var.interval - timeout = var.timeout - init = var.init - init_script = (local.init ? "terraform init -reconfigure -upgrade" : "") - tf_data_dir = var.data_path != null ? var.data_path : path.root - skip_destroy = (var.skip_destroy ? "true" : "") + + deploy_trigger = var.deploy_trigger + deploy_path = chomp(var.deploy_path) + attempts = var.attempts + interval = var.interval + timeout = var.timeout + init = var.init + init_script = (local.init ? "terraform init -reconfigure -upgrade" : "") + tf_data_dir = (var.data_path != null ? var.data_path : path.root) + skip_destroy = (var.skip_destroy ? "true" : "") +} + +resource "file_local_directory" "deploy_path" { + path = local.deploy_path + permissions = "0755" } -module "persist_template" { - source = "../persist_file" +resource "file_local_directory" "tf_data_dir" { + count = (local.tf_data_dir != local.deploy_path ? 1 : 0) + path = local.tf_data_dir + permissions = "0755" +} + +### Template Files ### +data "file_local" "template_files" { + for_each = local.template_file_map + directory = dirname(each.value) + name = each.key +} +resource "file_local_snapshot" "persist_tpl_file" { depends_on = [ + file_local_directory.deploy_path, + file_local_directory.tf_data_dir, + data.file_local.template_files, ] - for_each = local.template_file_map - path = "${local.deploy_path}/${each.key}" - contents = file(each.value) - recreate = filemd5(each.value) + for_each = local.template_file_map + directory = dirname(each.value) + name = each.key + update_trigger = local.deploy_trigger +} +resource "file_local" "instantiate_tpl_snapshot" { + depends_on = [ + file_local_directory.deploy_path, + file_local_directory.tf_data_dir, + data.file_local.template_files, + file_local_snapshot.persist_tpl_file, + ] + for_each = local.template_file_map + directory = local.deploy_path + name = each.key + permissions = data.file_local.template_files[each.key].permissions + contents = base64decode(file_local_snapshot.persist_tpl_file[each.key].snapshot) } -module "persist_inputs" { - source = "../persist_file" +### Inputs ### +resource "terraform_data" "write_tmp_inputs" { + depends_on = [ + file_local_directory.deploy_path, + file_local_directory.tf_data_dir, + ] + triggers_replace = { + trigger = local.deploy_trigger + } + provisioner "local-exec" { + command = <<-EOT + cat <<'EOF'> "${local.tf_data_dir}/inputs" + ${local.inputs} + EOF + EOT + } +} +resource "file_local_snapshot" "persist_inputs" { depends_on = [ + file_local_directory.deploy_path, + file_local_directory.tf_data_dir, + terraform_data.write_tmp_inputs, ] - path = "${local.deploy_path}/inputs.tfvars" - contents = local.inputs - recreate = md5(local.inputs) + directory = local.tf_data_dir + name = "inputs" + update_trigger = local.deploy_trigger +} +resource "terraform_data" "remove_tmp_inputs" { + depends_on = [ + file_local_directory.deploy_path, + file_local_directory.tf_data_dir, + terraform_data.write_tmp_inputs, + file_local_snapshot.persist_inputs, + ] + triggers_replace = { + trigger = local.deploy_trigger + } + provisioner "local-exec" { + command = <<-EOT + rm -f "${local.tf_data_dir}/inputs" + EOT + } +} +resource "file_local" "instantiate_inputs_snapshot" { + depends_on = [ + file_local_directory.deploy_path, + file_local_directory.tf_data_dir, + terraform_data.write_tmp_inputs, + file_local_snapshot.persist_inputs, + terraform_data.remove_tmp_inputs, + ] + directory = local.deploy_path + name = "inputs.tfvars" + contents = base64decode(file_local_snapshot.persist_inputs.snapshot) } +### Environment Variables ### +resource "terraform_data" "write_tmp_envrc" { + depends_on = [ + file_local_directory.deploy_path, + file_local_directory.tf_data_dir, + ] + triggers_replace = { + trigger = local.deploy_trigger + } + provisioner "local-exec" { + command = <<-EOT + cat <<'EOF'> "${local.tf_data_dir}/envrc" + ${local.export_contents} + EOF + EOT + } +} +resource "file_local_snapshot" "persist_envrc" { + depends_on = [ + file_local_directory.deploy_path, + file_local_directory.tf_data_dir, + terraform_data.write_tmp_envrc, + ] + directory = local.tf_data_dir + name = "envrc" + update_trigger = local.deploy_trigger +} +resource "terraform_data" "remove_tmp_envrc" { + depends_on = [ + file_local_directory.deploy_path, + file_local_directory.tf_data_dir, + terraform_data.write_tmp_envrc, + file_local_snapshot.persist_envrc, + ] + triggers_replace = { + trigger = local.deploy_trigger + } + provisioner "local-exec" { + command = <<-EOT + rm -f "${local.tf_data_dir}/envrc" + EOT + } +} +resource "file_local" "instantiate_envrc_snapshot" { + depends_on = [ + file_local_directory.deploy_path, + file_local_directory.tf_data_dir, + terraform_data.write_tmp_envrc, + file_local_snapshot.persist_envrc, + terraform_data.remove_tmp_envrc, + ] + directory = local.deploy_path + name = "envrc" + contents = base64decode(file_local_snapshot.persist_envrc.snapshot) +} + + resource "terraform_data" "destroy" { depends_on = [ - module.persist_template, - module.persist_inputs, + file_local.instantiate_envrc_snapshot, + file_local.instantiate_inputs_snapshot, + file_local.instantiate_tpl_snapshot, ] triggers_replace = { - inputs = local.inputs_hash - files = local.template_files_hash - env = local.export_hash - ec = local.export_contents - dp = local.deploy_path - to = local.timeout - dd = local.tf_data_dir - sd = local.skip_destroy + trigger = local.deploy_trigger + dp = local.deploy_path + sd = local.skip_destroy + to = local.timeout } provisioner "local-exec" { when = destroy command = templatefile("${path.module}/destroy.sh.tpl", { - export_contents = self.triggers_replace.ec - tf_data_dir = self.triggers_replace.dd - deploy_path = self.triggers_replace.dp - skip_destroy = self.triggers_replace.sd - timeout = self.triggers_replace.to + deploy_path = self.triggers_replace.dp + skip_destroy = self.triggers_replace.sd + timeout = self.triggers_replace.to }) } } resource "terraform_data" "create" { depends_on = [ - module.persist_template, - module.persist_inputs, + file_local.instantiate_envrc_snapshot, + file_local.instantiate_inputs_snapshot, + file_local.instantiate_tpl_snapshot, terraform_data.destroy, ] triggers_replace = { - files = local.template_files_hash + never = <<-EOT + This resource is only meant to run once, + on the initial deploy, + the secondary create manages updates. + EOT } provisioner "local-exec" { command = templatefile("${path.module}/create.sh.tpl", { - export_contents = local.export_contents - deploy_path = local.deploy_path - tf_data_dir = local.tf_data_dir - init_script = local.init_script - attempts = local.attempts - timeout = local.timeout - interval = local.interval + deploy_path = local.deploy_path + init_script = local.init_script + attempts = local.attempts + timeout = local.timeout + interval = local.interval }) } } -module "persist_state" { +resource "file_local_snapshot" "persist_state" { + depends_on = [ + terraform_data.destroy, + terraform_data.create, + ] + directory = local.deploy_path + name = "tfstate" + update_trigger = terraform_data.create.id +} +resource "file_local" "instantiate_state" { depends_on = [ - module.persist_template, - module.persist_inputs, terraform_data.destroy, terraform_data.create, + file_local_snapshot.persist_state, ] - source = "../persist_file" - path = "${local.deploy_path}/tfstate" - sourcefile = "${local.deploy_path}/tfstate" - recreate = terraform_data.create.id + directory = local.deploy_path + name = "tfstate" + contents = base64decode(file_local_snapshot.persist_state.snapshot) } -module "persist_outputs" { +resource "file_local_snapshot" "persist_outputs" { + depends_on = [ + terraform_data.destroy, + terraform_data.create, + ] + directory = local.deploy_path + name = "outputs.json" + update_trigger = terraform_data.create.id +} +resource "file_local" "instantiate_outputs" { depends_on = [ - module.persist_template, - module.persist_inputs, terraform_data.destroy, terraform_data.create, + file_local_snapshot.persist_outputs, ] - source = "../persist_file" - path = "${local.deploy_path}/outputs.json" - sourcefile = "${local.deploy_path}/outputs.json" - recreate = terraform_data.create.id + directory = local.deploy_path + name = "outputs.json" + contents = base64decode(file_local_snapshot.persist_outputs.snapshot) } # during initial create this should be an extra apply that has no effect @@ -138,59 +274,50 @@ module "persist_outputs" { # to rebuild the template before running the create script resource "terraform_data" "create_after_persist" { depends_on = [ - module.persist_template, - module.persist_inputs, + file_local.instantiate_envrc_snapshot, + file_local.instantiate_inputs_snapshot, + file_local.instantiate_tpl_snapshot, terraform_data.destroy, terraform_data.create, - module.persist_state, - module.persist_outputs, + file_local.instantiate_state, + file_local.instantiate_outputs, ] triggers_replace = { - inputs = local.inputs_hash - files = local.template_files_hash - env = local.export_hash + trigger = local.deploy_trigger } provisioner "local-exec" { command = templatefile("${path.module}/create.sh.tpl", { - export_contents = local.export_contents - deploy_path = local.deploy_path - tf_data_dir = local.tf_data_dir - init_script = local.init_script - attempts = local.attempts - timeout = local.timeout - interval = local.interval + deploy_path = local.deploy_path + init_script = local.init_script + attempts = local.attempts + timeout = local.timeout + interval = local.interval }) } } resource "terraform_data" "destroy_end" { depends_on = [ - module.persist_template, - module.persist_inputs, + file_local.instantiate_envrc_snapshot, + file_local.instantiate_inputs_snapshot, + file_local.instantiate_tpl_snapshot, terraform_data.destroy, terraform_data.create, - module.persist_state, - module.persist_outputs, + file_local.instantiate_state, + file_local.instantiate_outputs, terraform_data.create_after_persist, ] triggers_replace = { - inputs = local.inputs_hash - files = local.template_files_hash - env = local.export_hash - ec = local.export_contents - dp = local.deploy_path - to = local.timeout - dd = local.tf_data_dir - sd = local.skip_destroy + dp = local.deploy_path + sd = local.skip_destroy + to = local.timeout } provisioner "local-exec" { when = destroy command = templatefile("${path.module}/destroy.sh.tpl", { - export_contents = self.triggers_replace.ec - tf_data_dir = self.triggers_replace.dd - deploy_path = self.triggers_replace.dp - skip_destroy = self.triggers_replace.sd - timeout = self.triggers_replace.to + deploy_path = self.triggers_replace.dp + skip_destroy = self.triggers_replace.sd + timeout = self.triggers_replace.to }) } } diff --git a/modules/deploy/outputs.tf b/modules/deploy/outputs.tf index faad33b..673fb8e 100644 --- a/modules/deploy/outputs.tf +++ b/modules/deploy/outputs.tf @@ -1,5 +1,5 @@ output "output" { - value = { for k, v in jsondecode(module.persist_outputs.contents) : k => v.value } + value = { for k, v in jsondecode(base64decode(file_local_snapshot.persist_outputs.snapshot)) : k => v.value } } # output "raw_output" { diff --git a/modules/deploy/variables.tf b/modules/deploy/variables.tf index bf2bf2b..90aa9c3 100644 --- a/modules/deploy/variables.tf +++ b/modules/deploy/variables.tf @@ -5,16 +5,6 @@ variable "inputs" { EOT default = "" } -variable "template_path" { - type = string - description = <<-EOT - Path to the module to deploy. - These files will be copied to the deploy path, not used directly. - This is optional, but one of template_path or template_files must be specified. - Only one of template_path or template_files can be specified. - EOT - default = null -} variable "template_files" { type = list(any) description = <<-EOT @@ -92,3 +82,12 @@ variable "skip_destroy" { EOT default = false } +variable "deploy_trigger" { + type = string + description = <<-EOT + An arbitrary string which describes the deployment itself (not what it is deploying). + When this string changes the module will update the deployment files from the other inputs given. + This means that arbitrary changes to this module's inputs don't cause the deployment to trigger, + the deployment will only trigger when this string changes. + EOT +} diff --git a/modules/deploy/versions.tf b/modules/deploy/versions.tf index 824ea62..2e06979 100644 --- a/modules/deploy/versions.tf +++ b/modules/deploy/versions.tf @@ -1,9 +1,9 @@ terraform { required_version = ">= 1.5.0" required_providers { - filesystem = { - source = "sethvargo/filesystem" - version = "1.0.0" + file = { + source = "rancher/file" + version = ">= 1.1.0" } } } diff --git a/modules/install_cert_manager/main.tf b/modules/install_cert_manager/main.tf index 1d1ec1c..1375a3c 100644 --- a/modules/install_cert_manager/main.tf +++ b/modules/install_cert_manager/main.tf @@ -21,10 +21,27 @@ module "deploy_cert_manager" { source = "../deploy" depends_on = [ ] - deploy_path = local.deploy_path - data_path = local.deploy_path - template_path = local.cert_manager_path - skip_destroy = true # this is a one way operation, uninstall is not supported + deploy_path = local.deploy_path + data_path = local.deploy_path + template_files = [ + join("/", [local.cert_manager_path, "main.tf"]), + join("/", [local.cert_manager_path, "variables.tf"]), + join("/", [local.cert_manager_path, "versions.tf"]), + ] + skip_destroy = true # this is a one way operation, uninstall is not supported + # if any of these change, redeploy/update + deploy_trigger = md5( + join("-", [ + local.rancher_domain, + local.zone_id, + local.project_cert_key_id, + local.path, + local.cert_manager_version, + local.cert_manager_path, + md5(jsonencode(local.cert_manager_config)), + local.deploy_path, + ]) + ) environment_variables = { KUBE_CONFIG_PATH = "${abspath(local.path)}/kubeconfig" KUBECONFIG = "${abspath(local.path)}/kubeconfig" diff --git a/modules/install_cert_manager/versions.tf b/modules/install_cert_manager/versions.tf index 859ef02..8eb53dc 100644 --- a/modules/install_cert_manager/versions.tf +++ b/modules/install_cert_manager/versions.tf @@ -9,10 +9,6 @@ terraform { source = "hashicorp/kubernetes" version = ">= 2.31.0" } - local = { - source = "hashicorp/local" - version = ">= 2.5" - } time = { source = "hashicorp/time" version = ">= 0.12.0" @@ -21,5 +17,9 @@ terraform { source = "hashicorp/aws" version = ">= 5.11" } + file = { + source = "rancher/file" + version = ">= 2.0.0" + } } } diff --git a/modules/persist_file/main.tf b/modules/persist_file/main.tf deleted file mode 100644 index 5eda740..0000000 --- a/modules/persist_file/main.tf +++ /dev/null @@ -1,53 +0,0 @@ -locals { - full_path = abspath(var.path) # where to place the file - contents = var.contents # the contents to persist - sourcefile = var.sourcefile # the sourcefile to persist - recreate = var.recreate # when this changes update the persisted data to match contents - - # tflint-ignore: terraform_unused_declarations - fail_no_source = ((local.contents == "" && local.sourcefile == "") ? one([local.contents, "missing_something_to_persist"]) : false) - - data = (local.contents != "" ? local.contents : data.external.read_file.result.data) -} - -resource "terraform_data" "recreate" { - input = local.recreate -} - -data "external" "read_file" { - depends_on = [ - terraform_data.recreate, - ] - program = ["bash", "${path.module}/read_file.sh"] - query = { - filepath = local.sourcefile - } -} - -resource "terraform_data" "snapshot" { - depends_on = [ - data.external.read_file, - terraform_data.recreate, - ] - input = local.data - # we want this data to persist even if the input data changes - # the point of this is so that we control when the data is updated ie. when the snapshot is saved/updated - lifecycle { - ignore_changes = [ - input, - ] - } - triggers_replace = [ - terraform_data.recreate.output, - ] -} - -resource "local_sensitive_file" "file" { - depends_on = [ - data.external.read_file, - terraform_data.recreate, - terraform_data.snapshot, - ] - filename = local.full_path - content = terraform_data.snapshot.output -} diff --git a/modules/persist_file/outputs.tf b/modules/persist_file/outputs.tf deleted file mode 100644 index e5f5bcf..0000000 --- a/modules/persist_file/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "contents" { - value = local_sensitive_file.file.content - sensitive = true -} diff --git a/modules/persist_file/read_file.sh b/modules/persist_file/read_file.sh deleted file mode 100755 index 5ed788f..0000000 --- a/modules/persist_file/read_file.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -JSON_INPUT="$(jq -r '.')" -FILEPATH="$(jq -r '.filepath' <<<"$JSON_INPUT")" - -jq -n --rawfile data "$FILEPATH" '{"data": $data}' 2>/dev/null || jq -n '{"data":"error"}' diff --git a/modules/persist_file/variables.tf b/modules/persist_file/variables.tf deleted file mode 100644 index d9914e1..0000000 --- a/modules/persist_file/variables.tf +++ /dev/null @@ -1,28 +0,0 @@ -variable "path" { - type = string - description = <<-EOT - The path to save the contents to. - EOT -} -variable "recreate" { - type = string - description = <<-EOT - When this string changes, update the file snapshot. - EOT -} -variable "contents" { - type = string - description = <<-EOT - The contents to persist, one of "contents" or "sourcefile" must be given. - EOT - default = "" - sensitive = true -} -variable "sourcefile" { - type = string - description = <<-EOT - A file to persist, one of "contents" or "sourcefile" must be given. - EOT - default = "" - sensitive = true -} diff --git a/modules/persist_file/versions.tf b/modules/persist_file/versions.tf deleted file mode 100644 index e57796b..0000000 --- a/modules/persist_file/versions.tf +++ /dev/null @@ -1,13 +0,0 @@ -terraform { - required_version = ">= 1.5.0" - required_providers { - external = { - source = "hashicorp/external" - version = ">= 2.3" - } - local = { - source = "hashicorp/local" - version = ">= 2.5.3" - } - } -} diff --git a/modules/rancher_bootstrap/main.tf b/modules/rancher_bootstrap/main.tf index 00957e0..8b9ecee 100644 --- a/modules/rancher_bootstrap/main.tf +++ b/modules/rancher_bootstrap/main.tf @@ -24,13 +24,40 @@ locals { } module "deploy_rancher" { - source = "../deploy" - deploy_path = local.deploy_path - data_path = local.deploy_path - template_path = local.rancher_path - attempts = 5 - interval = 60 - skip_destroy = true # this is a one way operation, uninstall not supported + source = "../deploy" + deploy_path = local.deploy_path + data_path = local.deploy_path + template_files = [ + join("/", [local.rancher_path, "main.tf"]), + join("/", [local.rancher_path, "outputs.tf"]), + join("/", [local.rancher_path, "variables.tf"]), + join("/", [local.rancher_path, "versions.tf"]), + join("/", [local.rancher_path, "runningPods.sh"]), + join("/", [local.rancher_path, "runningDeployments.sh"]), + ] + attempts = 5 + interval = 60 + skip_destroy = true # this is a one way operation, uninstall not supported + # if any of these change, redeploy/update + deploy_trigger = md5(join("-", [ + local.project_domain, + local.zone_id, + local.region, + local.email, + local.acme_server_url, + local.rancher_version, + local.rancher_helm_repo, + local.rancher_helm_channel, + local.cert_manager_version, + local.path, + local.externalTLS, + local.rancher_path, + local.deploy_path, + md5(jsonencode(local.rancher_helm_chart_values)), + local.cert_public, + local.cert_private, + local.cert_chain, + ])) environment_variables = { KUBECONFIG = "${local.path}/kubeconfig" KUBE_CONFIG_PATH = "${local.path}/kubeconfig" diff --git a/modules/rancher_bootstrap/rancher_externalTLS/main.tf b/modules/rancher_bootstrap/rancher_externalTLS/main.tf index 4fe2447..0f931f8 100644 --- a/modules/rancher_bootstrap/rancher_externalTLS/main.tf +++ b/modules/rancher_bootstrap/rancher_externalTLS/main.tf @@ -33,21 +33,21 @@ locals { ) # WARNING! helm_chart_use_strategy is required and must be "default", "merge", or "provide", if the strategy isn't found, the coalesce will fail } -resource "local_file" "hcv" { - filename = "helm_chart_values.txt" - content = jsonencode(local.rancher_helm_chart_values) +resource "file_local" "hcv" { + name = "helm_chart_values.txt" + contents = jsonencode(local.rancher_helm_chart_values) } -resource "local_file" "pc" { - filename = "public.cert" - content = local.public_cert +resource "file_local" "pc" { + name = "public.cert" + contents = local.public_cert } -resource "local_file" "ca" { - filename = "ca.cert" - content = local.ca_certs +resource "file_local" "ca" { + name = "ca.cert" + contents = local.ca_certs } -resource "local_file" "key" { - filename = "private.key" - content = local.private_key +resource "file_local" "key" { + name = "private.key" + contents = local.private_key } resource "time_sleep" "settle_before_rancher" { diff --git a/modules/rancher_bootstrap/rancher_externalTLS/runningDeployments.sh b/modules/rancher_bootstrap/rancher_externalTLS/runningDeployments.sh new file mode 100755 index 0000000..c732f81 --- /dev/null +++ b/modules/rancher_bootstrap/rancher_externalTLS/runningDeployments.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -x + +JSONPATH="'{range .items[*]} + {.metadata.name}{\"\\t\"} \ + {.metadata.namespace}{\"\\t\"} \ + {.status.readyReplicas}{\"\\t\"} \ + {.status.replicas}{\"\\n\"} \ +{end}'" + +notReady() { + NOT_READY="" + ITEMS="$(kubectl get deployments -A -o jsonpath="$JSONPATH")" + while IFS= read -r item; do + ready="$(echo "$item" | awk '{print $3}')" + total="$(echo "$item" | awk '{print $4}')" + if [ "$ready" != "$total" ]; then + NOT_READY=1 + fi + done <<< "$ITEMS" + # shellcheck disable=SC2060,SC2140 + if [ -z "$NOT_READY" ]; then + # All items are ready + return 1 + else + # Some items aren't ready + return 0 + fi +} + +readyWait() { + TIMEOUT=10 # 10 minutes + TIMEOUT_MINUTES=$((TIMEOUT * 60)) + INTERVAL=30 # 30 seconds + MAX=$((TIMEOUT_MINUTES / INTERVAL)) + ATTEMPTS=0 + + while notReady; do + if [ "$ATTEMPTS" -lt "$MAX" ]; then + ATTEMPTS=$((ATTEMPTS + 1)) + sleep "$INTERVAL"; + else + return 1 + fi + done + return 0 +} + +SUCCESSES=0 +SUCCESSES_NEEDED=2 + +while readyWait && [ "$SUCCESSES" -lt "$SUCCESSES_NEEDED" ]; do + SUCCESSES=$((SUCCESSES + 1)) + echo "succeeeded $SUCCESSES times..." + sleep 30 +done + +if [ "$SUCCESSES" -eq "$SUCCESSES_NEEDED" ]; then + echo "$SUCCESSES_NEEDED successes reached, passed..." + EXITCODE=0 +else + echo "$SUCCESSES_NEEDED successes not reached, failed..." + EXITCODE=1 +fi + +echo "nodes..." +kubectl get nodes || true + +echo "all..." +kubectl get all -A || true + +echo "deployments..." +kubectl get deployments -A || true + +exit $EXITCODE diff --git a/modules/rancher_bootstrap/rancher_externalTLS/versions.tf b/modules/rancher_bootstrap/rancher_externalTLS/versions.tf index bfc7e72..6aabb11 100644 --- a/modules/rancher_bootstrap/rancher_externalTLS/versions.tf +++ b/modules/rancher_bootstrap/rancher_externalTLS/versions.tf @@ -5,10 +5,6 @@ terraform { source = "hashicorp/helm" version = "2.14" } - local = { - source = "hashicorp/local" - version = ">= 2.5" - } rancher2 = { source = "rancher/rancher2" version = ">= 5.0.0" @@ -29,5 +25,9 @@ terraform { source = "hashicorp/aws" version = ">= 5.11" } + file = { + source = "rancher/file" + version = ">= 2.2.0" + } } } diff --git a/modules/rancher_bootstrap/versions.tf b/modules/rancher_bootstrap/versions.tf index 23a517a..a44fb2f 100644 --- a/modules/rancher_bootstrap/versions.tf +++ b/modules/rancher_bootstrap/versions.tf @@ -1,9 +1,9 @@ terraform { required_version = ">= 1.5.0" required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.5" + file = { + source = "rancher/file" + version = ">= 2.0.0" } helm = { source = "hashicorp/helm" diff --git a/test/tests/three/state_test.go b/test/tests/three/state_test.go new file mode 100644 index 0000000..b389bf9 --- /dev/null +++ b/test/tests/three/state_test.go @@ -0,0 +1,164 @@ +package one + +import ( + "os" + "path/filepath" + "strings" + "testing" + + aws "github.com/gruntwork-io/terratest/modules/aws" + g "github.com/gruntwork-io/terratest/modules/git" + "github.com/gruntwork-io/terratest/modules/ssh" + "github.com/gruntwork-io/terratest/modules/terraform" + util "github.com/rancher/terraform-rancher2-aws/test/tests" +) + +// This test is the same as basic but it also tests that the state is correctly stored in S3 and can be used to re-create the cluster +func TestThreeState(t *testing.T) { + t.Parallel() + util.SetAcmeServer() + + id := util.GetId() + region := util.GetRegion() + directory := "three" + owner := "terraform-ci@suse.com" + repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) + if err != nil { + t.Fatalf("Error getting git root directory: %v", err) + } + exampleDir := repoRoot + "/examples/" + directory + testDir := repoRoot + "/test/tests/data/" + id + + err = util.CreateTestDirectories(t, id) + if err != nil { + os.RemoveAll(testDir) + t.Fatalf("Error creating test data directories: %s", err) + } + keyPair, err := util.CreateKeypair(t, region, owner, id) + if err != nil { + os.RemoveAll(testDir) + t.Fatalf("Error creating test key pair: %s", err) + } + err = os.WriteFile(testDir+"/id_rsa", []byte(keyPair.KeyPair.PrivateKey), 0600) + if err != nil { + err = aws.DeleteEC2KeyPairE(t, keyPair) + if err != nil { + t.Logf("Failed to destroy key pair: %v", err) + } + os.RemoveAll(testDir) + t.Fatalf("Error creating test key pair: %s", err) + } + sshAgent := ssh.SshAgentWithKeyPair(t, keyPair.KeyPair) + t.Logf("Key %s created and added to agent", keyPair.Name) + + backendTerraformOptions, err := util.CreateObjectStorageBackend(t, testDir, id, owner, region) + tfOptions := []*terraform.Options{backendTerraformOptions} + if err != nil { + t.Log("Test failed, tearing down...") + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) + t.Fatalf("Error creating cluster: %s", err) + } + + // use oldest RKE2, remember it releases much more than Rancher + _, _, rke2Version, err := util.GetRke2Releases() + if err != nil { + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) + t.Fatalf("Error getting Rke2 release version: %s", err) + } + + rancherVersion := os.Getenv("RANCHER_VERSION") + if rancherVersion == "" { + // use stable version if not specified + // using stable prevents problems where the Rancher provider hasn't released to fit the latest Rancher + _, rancherVersion, _, err = util.GetRancherReleases() + } + if err != nil { + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) + t.Fatalf("Error getting Rancher release version: %s", err) + } + + terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ + TerraformDir: exampleDir, + // Variables to pass to our Terraform code using -var options + Vars: map[string]interface{}{ + "identifier": id, + "owner": owner, + "key_name": keyPair.Name, + "key": keyPair.KeyPair.PublicKey, + "zone": os.Getenv("ZONE"), + "rke2_version": rke2Version, + "rancher_version": rancherVersion, + "file_path": testDir, + }, + // Environment variables to set when running Terraform + EnvVars: map[string]string{ + "AWS_DEFAULT_REGION": region, + "AWS_REGION": region, + "TF_DATA_DIR": testDir, + "TF_IN_AUTOMATION": "1", + "TF_CLI_ARGS_init": "-backend-config=\"bucket=" + strings.ToLower(id) + "\"", + }, + RetryableTerraformErrors: util.GetRetryableTerraformErrors(), + NoColor: true, + SshAgent: sshAgent, + Reconfigure: true, + Upgrade: true, + }) + // we need to prepend the main options because we need to destroy it before the backend + newTfOptions := []*terraform.Options{terraformOptions, backendTerraformOptions} + _, err = terraform.InitAndApplyE(t, terraformOptions) + if err != nil { + t.Log("Test failed, tearing down...") + util.GetErrorLogs(t, testDir+"/kubeconfig") + util.Teardown(t, testDir, exampleDir, newTfOptions, keyPair, sshAgent) + t.Fatalf("Error creating cluster: %s", err) + } + util.CheckReady(t, testDir+"/kubeconfig") + util.CheckRunning(t, testDir+"/kubeconfig") + + os.RemoveAll(testDir) + err = util.CreateTestDirectories(t, id) + if err != nil { + t.Log("Test failed, tearing down...") + util.GetErrorLogs(t, testDir+"/kubeconfig") + util.Teardown(t, testDir, exampleDir, newTfOptions, keyPair, sshAgent) + t.Fatalf("Error creating cluster: %s", err) + } + + // Running the apply again should re-create everything from state in S3 + // This should only recreate the files, the resources should be untouched + err = os.WriteFile(testDir+"/id_rsa", []byte(keyPair.KeyPair.PrivateKey), 0600) + if err != nil { + t.Log("Test failed, tearing down...") + util.GetErrorLogs(t, testDir+"/kubeconfig") + util.Teardown(t, testDir, exampleDir, newTfOptions, keyPair, sshAgent) + t.Fatalf("Error creating cluster: %s", err) + } + _, err = terraform.InitAndApplyE(t, terraformOptions) + if err != nil { + t.Log("Test failed, tearing down...") + util.GetErrorLogs(t, testDir+"/kubeconfig") + util.Teardown(t, testDir, exampleDir, newTfOptions, keyPair, sshAgent) + t.Fatalf("Error creating cluster: %s", err) + } + util.CheckReady(t, testDir+"/kubeconfig") + util.CheckRunning(t, testDir+"/kubeconfig") + + // Running the apply again should not change anything + _, err = terraform.InitAndApplyE(t, terraformOptions) + if err != nil { + t.Log("Test failed, tearing down...") + util.GetErrorLogs(t, testDir+"/kubeconfig") + util.Teardown(t, testDir, exampleDir, newTfOptions, keyPair, sshAgent) + t.Fatalf("Error creating cluster: %s", err) + } + util.CheckReady(t, testDir+"/kubeconfig") + util.CheckRunning(t, testDir+"/kubeconfig") + + if t.Failed() { + t.Log("Test failed...") + } else { + t.Log("Test passed...") + } + util.Teardown(t, testDir, exampleDir, newTfOptions, keyPair, sshAgent) +}