From 2888feaa247f245b9ddb3c56fc18f8a61f62a209 Mon Sep 17 00:00:00 2001 From: matttrach Date: Wed, 12 Nov 2025 17:11:57 -0600 Subject: [PATCH 1/3] fix: separate install from bootstrap Signed-off-by: matttrach --- cleanup.sh | 116 ++++++++++++++++ examples/dev/README.md | 5 + examples/dev/main.tf | 122 +++++++++++++++++ examples/dev/modules/tls/main.tf | 57 ++++++++ examples/dev/modules/tls/outputs.tf | 12 ++ examples/dev/modules/tls/variables.tf | 6 + examples/dev/modules/tls/versions.tf | 9 ++ examples/dev/outputs.tf | 22 +++ examples/dev/variables.tf | 54 ++++++++ examples/dev/versions.tf | 53 +++++++ examples/downstream/main.tf | 7 +- examples/downstream/variables.tf | 8 ++ examples/downstream_splitrole/main.tf | 33 ++--- examples/downstream_splitrole/variables.tf | 8 ++ examples/one/main.tf | 19 +-- examples/one/variables.tf | 8 ++ examples/prod/main.tf | 21 +-- examples/prod/variables.tf | 8 ++ examples/three/main.tf | 19 +-- examples/three/variables.tf | 8 ++ flake.lock | 6 +- main.tf | 21 ++- modules/bootstrap_rancher/bootstrap/main.tf | 19 +++ .../bootstrap}/outputs.tf | 12 +- .../bootstrap_rancher/bootstrap/variables.tf | 15 ++ .../bootstrap_rancher/bootstrap/versions.tf | 13 ++ modules/bootstrap_rancher/main.tf | 39 ++++++ modules/bootstrap_rancher/outputs.tf | 9 ++ modules/bootstrap_rancher/variables.tf | 16 +++ modules/bootstrap_rancher/versions.tf | 9 ++ modules/deploy/create.sh.tpl | 10 +- modules/deploy/destroy.sh.tpl | 6 +- modules/install_cert_manager/main.tf | 9 +- .../main.tf | 5 +- modules/install_rancher/outputs.tf | 9 ++ .../rancher/main.tf | 45 ++---- modules/install_rancher/rancher/outputs.tf | 9 ++ .../rancher/runningDeployments.sh | 0 .../rancher/runningPods.sh | 0 .../rancher/variables.tf | 1 + .../rancher/versions.tf | 4 - .../rancher_externalTLS/main.tf | 49 ++----- .../rancher_externalTLS/outputs.tf | 9 ++ .../rancher_externalTLS/runningDeployments.sh | 0 .../rancher_externalTLS/runningPods.sh | 0 .../rancher_externalTLS/variables.tf | 1 + .../rancher_externalTLS}/versions.tf | 12 +- .../variables.tf | 1 + .../versions.tf | 12 +- modules/rancher_bootstrap/outputs.tf | 14 -- modules/rancher_bootstrap/rancher/outputs.tf | 14 -- outputs.tf | 6 +- run_tests.sh | 81 ++--------- test/tests/dev/basic_test.go | 129 ++++++++++++++++++ test/tests/downstream/basic_test.go | 3 +- test/tests/downstream/splitrole_test.go | 3 +- test/tests/one/basic_test.go | 3 +- test/tests/prod/basic_test.go | 3 +- test/tests/three/basic_test.go | 3 +- test/tests/three/state_test.go | 3 +- test/tests/util.go | 2 +- variables.tf | 12 +- versions.tf | 4 - 63 files changed, 936 insertions(+), 280 deletions(-) create mode 100755 cleanup.sh create mode 100644 examples/dev/README.md create mode 100644 examples/dev/main.tf create mode 100644 examples/dev/modules/tls/main.tf create mode 100644 examples/dev/modules/tls/outputs.tf create mode 100644 examples/dev/modules/tls/variables.tf create mode 100644 examples/dev/modules/tls/versions.tf create mode 100644 examples/dev/outputs.tf create mode 100644 examples/dev/variables.tf create mode 100644 examples/dev/versions.tf create mode 100644 modules/bootstrap_rancher/bootstrap/main.tf rename modules/{rancher_bootstrap/rancher_externalTLS => bootstrap_rancher/bootstrap}/outputs.tf (53%) create mode 100644 modules/bootstrap_rancher/bootstrap/variables.tf create mode 100644 modules/bootstrap_rancher/bootstrap/versions.tf create mode 100644 modules/bootstrap_rancher/main.tf create mode 100644 modules/bootstrap_rancher/outputs.tf create mode 100644 modules/bootstrap_rancher/variables.tf create mode 100644 modules/bootstrap_rancher/versions.tf rename modules/{rancher_bootstrap => install_rancher}/main.tf (95%) create mode 100644 modules/install_rancher/outputs.tf rename modules/{rancher_bootstrap => install_rancher}/rancher/main.tf (90%) create mode 100644 modules/install_rancher/rancher/outputs.tf rename modules/{rancher_bootstrap => install_rancher}/rancher/runningDeployments.sh (100%) rename modules/{rancher_bootstrap => install_rancher}/rancher/runningPods.sh (100%) rename modules/{rancher_bootstrap => install_rancher}/rancher/variables.tf (99%) rename modules/{rancher_bootstrap => install_rancher}/rancher/versions.tf (84%) rename modules/{rancher_bootstrap => install_rancher}/rancher_externalTLS/main.tf (86%) create mode 100644 modules/install_rancher/rancher_externalTLS/outputs.tf rename modules/{rancher_bootstrap => install_rancher}/rancher_externalTLS/runningDeployments.sh (100%) rename modules/{rancher_bootstrap => install_rancher}/rancher_externalTLS/runningPods.sh (100%) rename modules/{rancher_bootstrap => install_rancher}/rancher_externalTLS/variables.tf (99%) rename modules/{rancher_bootstrap => install_rancher/rancher_externalTLS}/versions.tf (86%) rename modules/{rancher_bootstrap => install_rancher}/variables.tf (99%) rename modules/{rancher_bootstrap/rancher_externalTLS => install_rancher}/versions.tf (86%) delete mode 100644 modules/rancher_bootstrap/outputs.tf delete mode 100644 modules/rancher_bootstrap/rancher/outputs.tf create mode 100644 test/tests/dev/basic_test.go diff --git a/cleanup.sh b/cleanup.sh new file mode 100755 index 0000000..bfb6a84 --- /dev/null +++ b/cleanup.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +# This script is run by the run_tests.sh script to clean up AWS resources created during testing. +# It can also be run independently to clean up resources by providing a cleanup ID. +cleanup_id="$1" +if [ -z "$cleanup_id" ]; then + echo "No cleanup Id provided. Exiting." + exit 1 +fi +echo "Starting cleanup for Id: $cleanup_id" +IDENTIFIER="$cleanup_id" +AWS_REGION="${AWS_REGION:-us-west-2}" + +echo "Clearing leftovers with Id $IDENTIFIER in $AWS_REGION..." + +max_attempts=3 + +attempts=0 +resources_to_clear="$(leftovers -d --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" | grep -v 'AccessDenied')" +while [ -n "$resources_to_clear" ] && [ $attempts -lt $max_attempts ]; do + echo -e "found these resources to clear:\n $resources_to_clear\n" + leftovers --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" --no-confirm | grep -v 'AccessDenied' || true + sleep 10 + resources_to_clear="$(leftovers -d --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" | grep -v 'AccessDenied')" + if [ -n "$resources_to_clear" ]; then + echo "Some resources failed to clear, retrying in $((attempts * 10)) seconds..." + fi + sleep $((attempts * 10)) + attempts=$((attempts + 1)) +done + +if [ $attempts -eq $max_attempts ]; then + echo "Warning: Failed to clear all resources after $max_attempts attempts." +fi + +# remove secrets +attempts=0 +while [ $attempts -lt $max_attempts ]; do + while read -r arn; do + if [ -z "$arn" ]; then + continue + fi + echo "removing secret $arn..." + aws secretsmanager delete-secret --secret-id "$arn" --force-delete-without-recovery + done <<<"$(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "secretsmanager:secret" --tag-filters "Key=Id,Values=$IDENTIFIER" | jq -r '.ResourceTagMappingList[]?.ResourceARN')" + sleep $((attempts * 10)) + attempts=$((attempts + 1)) +done + +# remove s3 storage +attempts=0 +while [ $attempts -lt $max_attempts ]; do + while read -r id; do + if [ -z "$id" ]; then + continue + fi + echo "removing s3 bucket $id..." + aws s3 rb "s3://$id" --force + done <<<"$(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "s3:bucket" --tag-filters "Key=Id,Values=$IDENTIFIER" | jq -r '.ResourceTagMappingList[]?.ResourceARN' | awk -F'arn:aws:s3:::' '{print $2}')" + sleep $((attempts * 10)) + attempts=$((attempts + 1)) +done + +# remove key pairs +attempts=0 +while [ $attempts -lt $max_attempts ]; do + while read -r id; do + if [ -z "$id" ]; then + continue + fi + echo "removing ec2 key pair $id..." + aws ec2 delete-key-pair --key-pair-id "$id" + done <<<"$(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "ec2:key-pair" --tag-filters "Key=Id,Values=$IDENTIFIER" | jq -r '.ResourceTagMappingList[]?.ResourceARN' | awk -F'/' '{print $2}')" + sleep $((attempts * 10)) + attempts=$((attempts + 1)) +done + +# remove server certificates +# unfortunately get-resources doesn't support iam server certificates +attempts=0 +while [ $attempts -lt $max_attempts ]; do + while read -r name; do + if [ -z "$name" ]; then + continue + fi + if aws iam list-server-certificate-tags --server-certificate-name "$name" | jq -e --arg ID "$IDENTIFIER" '.Tags[] | select(.Key=="Id" and .Value==$ID)' > /dev/null; then + echo "removing iam server certificate $name..." + aws iam delete-server-certificate --server-certificate-name "$name" + fi + done <<<"$(aws iam list-server-certificates | jq -r '.ServerCertificateMetadataList[].ServerCertificateName')" + sleep $((attempts * 10)) + attempts=$((attempts + 1)) +done + +# remove load balancer target groups +attempts=0 +while [ $attempts -lt $max_attempts ]; do + while read -r arn; do + if [ -z "$arn" ]; then + continue + fi + echo "removing load balancer target group $arn..." + aws elbv2 delete-target-group --target-group-arn "$arn"; + done <<<"$(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "elasticloadbalancing:targetgroup" --tag-filters "Key=Id,Values=$IDENTIFIER" | jq -r '.ResourceTagMappingList[]?.ResourceARN')" + sleep $((attempts * 10)) + attempts=$((attempts + 1)) +done + +echo "Cleanup completed." + +# These examples find Ids that need to be cleaned up by looking for resources owned by CI and extracting their Id tags. +# This is useful if you happen to come across leftover resources and want to clean up anything that might have been missed with their specific Id. +# For example, if you hit a quota limit and notice there a bunch of leftover secrets or target groups, you can run these commands to clean up all resources with the same Id as the leftover resources. +# for id in $(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "elasticloadbalancing:targetgroup" --tag-filters "Key=Owner,Values=terraform-ci@suse.com" | jq -r '.ResourceTagMappingList[]?.Tags[] | select(.Key=="Id") | .Value'); do ./cleanup.sh "$id"; done +# for id in $(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "secretsmanager:secret" --tag-filters "Key=Owner,Values=terraform-ci@suse.com" | jq -r '.ResourceTagMappingList[]?.Tags[] | select(.Key=="Id") | .Value'); do ./cleanup.sh "$id"; done +# for id in $(for name in $(aws iam list-server-certificates | jq -r '.ServerCertificateMetadataList[].ServerCertificateName'); do echo "$(aws iam list-server-certificate-tags --server-certificate-name "$name" | jq -r '.Tags[] | select(.Key=="Id").Value')"; done); do echo "$id"; done diff --git a/examples/dev/README.md b/examples/dev/README.md new file mode 100644 index 0000000..62f93da --- /dev/null +++ b/examples/dev/README.md @@ -0,0 +1,5 @@ +# Development Example + +This example is specifically designed to help test the Rancher provider. +The idea is to have an example that builds up everything you need to get started with the provider without actually implementing it. +This example stops after installing Rancher with the Helm chart. diff --git a/examples/dev/main.tf b/examples/dev/main.tf new file mode 100644 index 0000000..629ee8f --- /dev/null +++ b/examples/dev/main.tf @@ -0,0 +1,122 @@ +provider "aws" { + default_tags { + tags = { + Id = local.identifier + Owner = local.owner + } + } +} + +provider "github" {} +provider "kubernetes" {} # make sure you set the env variable KUBE_CONFIG_PATH to local_file_path (file_path variable) +provider "helm" {} # make sure you set the env variable KUBE_CONFIG_PATH to local_file_path (file_path variable) + + +terraform { + backend "s3" { + # This needs to be set in the backend configs on the command line or somewhere that your identifier can be set. + # terraform init -reconfigure -backend-config="bucket=" + # https://developer.hashicorp.com/terraform/language/backend/s3 + # https://developer.hashicorp.com/terraform/language/backend#partial-configuration + key = "tfstate" + } +} + +locals { + identifier = var.identifier + example = "dev" + project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" + username = local.project_name + domain = local.project_name + zone = var.zone + key_name = var.key_name + key = var.key + owner = var.owner + rke2_version = var.rke2_version + rancher_helm_repo = "https://releases.rancher.com/server-charts" + rancher_helm_channel = "stable" + helm_chart_strategy = "provide" + # These options use the Let's Encrypt cert that the module generates for you when you deploy the VPC and Domain. + # WARNING! "hostname" must be an fqdn + helm_chart_values = { + "hostname" = "${local.domain}.${local.zone}" + "replicas" = "1" + "bootstrapPassword" = random_password.admin_password.result + "tls" = "ingress" + "ingress.enabled" = "true" + "ingress.tls.source" = "secret" + "ingress.tls.secretName" = "tls-rancher-ingress" + "certmanager.version" = local.cert_manager_version + "agentTLSMode" = "strict" + "privateCA" = "true" + "additionalTrustedCAs" = "true" + } + node_configuration = { + "rancher" = { + type = "all-in-one" + size = "xxl" + os = local.os + indirect_access = true + initial = true + } + } + local_file_path = var.file_path + runner_ip = chomp(data.http.myip.response_body) # "runner" is the server running Terraform + rancher_version = var.rancher_version + cert_manager_version = "1.18.1" + os = "sle-micro-61" +} + +resource "random_password" "admin_password" { + length = 16 + special = true + override_special = "!#$%&-_=+" +} + + +data "http" "myip" { + url = "https://ipinfo.io/ip" +} + +# you shouldn't do this in production, I am trying to show/prove self-signed certificates working with the Rancher configuration +# this could easily be replaced by some secret resource from Vault or if you are using Terraform 1.11+ you should use the ephemeral resources +module "tls" { + source = "./modules/tls" + domain = "${local.domain}.${local.zone}" +} + +module "rancher" { + depends_on = [ + module.tls, + ] + source = "../../" + # project + identifier = local.identifier + owner = local.owner + project_name = local.project_name + domain = local.domain + zone = local.zone + # access + key_name = local.key_name + key = local.key + username = local.username + admin_ip = local.runner_ip + # rke2 + rke2_version = local.rke2_version + local_file_path = local.local_file_path + install_method = "rpm" # rpm only for now, need to figure out local helm chart installs otherwise + cni = "canal" + node_configuration = local.node_configuration + # rancher + cert_manager_version = local.cert_manager_version + cert_use_strategy = "supply" + tls_public_cert = module.tls.tls_public_certificate # just the cert, not any CA + tls_private_key = module.tls.tls_private_key + tls_public_chain = module.tls.certificate_chain # just the chain, it should not include the cert itself + rancher_version = local.rancher_version + rancher_helm_repo = local.rancher_helm_repo + rancher_helm_channel = local.rancher_helm_channel + rancher_helm_chart_use_strategy = local.helm_chart_strategy + rancher_helm_chart_values = local.helm_chart_values + bootstrap_rancher = false +} diff --git a/examples/dev/modules/tls/main.tf b/examples/dev/modules/tls/main.tf new file mode 100644 index 0000000..14669c1 --- /dev/null +++ b/examples/dev/modules/tls/main.tf @@ -0,0 +1,57 @@ + +locals { + domain = var.domain +} + +resource "tls_private_key" "ca_key" { + algorithm = "RSA" + rsa_bits = 2048 +} + +resource "tls_self_signed_cert" "ca_cert" { + private_key_pem = tls_private_key.ca_key.private_key_pem + + subject { + common_name = "Example CA" + organization = "Example" + } + + validity_period_hours = 8760 + is_ca_certificate = true + + allowed_uses = [ + "cert_signing", + "crl_signing", + ] +} + +// TLS Certificate +resource "tls_private_key" "tls_key" { + algorithm = "RSA" + rsa_bits = 2048 +} + +resource "tls_cert_request" "tls_csr" { + private_key_pem = tls_private_key.tls_key.private_key_pem + + subject { + common_name = local.domain + organization = "Example" + } + + dns_names = [local.domain] +} + +resource "tls_locally_signed_cert" "tls_cert" { + cert_request_pem = tls_cert_request.tls_csr.cert_request_pem + ca_private_key_pem = tls_private_key.ca_key.private_key_pem + ca_cert_pem = tls_self_signed_cert.ca_cert.cert_pem + + validity_period_hours = 8760 + + allowed_uses = [ + "key_encipherment", + "digital_signature", + "server_auth", + ] +} diff --git a/examples/dev/modules/tls/outputs.tf b/examples/dev/modules/tls/outputs.tf new file mode 100644 index 0000000..7f0832c --- /dev/null +++ b/examples/dev/modules/tls/outputs.tf @@ -0,0 +1,12 @@ +output "tls_public_certificate" { + value = tls_locally_signed_cert.tls_cert.cert_pem +} + +output "tls_private_key" { + value = tls_private_key.tls_key.private_key_pem + sensitive = true +} + +output "certificate_chain" { + value = tls_self_signed_cert.ca_cert.cert_pem +} diff --git a/examples/dev/modules/tls/variables.tf b/examples/dev/modules/tls/variables.tf new file mode 100644 index 0000000..9bbec36 --- /dev/null +++ b/examples/dev/modules/tls/variables.tf @@ -0,0 +1,6 @@ +variable "domain" { + type = string + description = <<-EOT + The domain name to set as common name and dns sans. + EOT +} diff --git a/examples/dev/modules/tls/versions.tf b/examples/dev/modules/tls/versions.tf new file mode 100644 index 0000000..28763e7 --- /dev/null +++ b/examples/dev/modules/tls/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.5.0" + required_providers { + tls = { + source = "hashicorp/tls" + version = ">= 4.0.5" + } + } +} diff --git a/examples/dev/outputs.tf b/examples/dev/outputs.tf new file mode 100644 index 0000000..41bccff --- /dev/null +++ b/examples/dev/outputs.tf @@ -0,0 +1,22 @@ +output "kubeconfig" { + value = module.rancher.kubeconfig + description = <<-EOT + The kubeconfig for the server. + EOT + sensitive = true +} +output "address" { + value = module.rancher.address +} +output "admin_token" { + value = module.rancher.admin_token + sensitive = true +} +output "admin_password" { + value = module.rancher.admin_password + sensitive = true +} +# output "cluster_data" { +# value = jsonencode(data.rancher2_cluster.local) +# sensitive = true +# } diff --git a/examples/dev/variables.tf b/examples/dev/variables.tf new file mode 100644 index 0000000..3d7bc84 --- /dev/null +++ b/examples/dev/variables.tf @@ -0,0 +1,54 @@ +variable "key_name" { + type = string + description = <<-EOT + The name of an AWS key pair to use for SSH access to the instance. + This key should already be added to your ssh agent for server authentication. + EOT +} +variable "key" { + type = string + description = <<-EOT + The contents of an AWS key pair to use for SSH access to the instance. + This is necessary for installing rke2 on the nodes and will be removed after installation. + EOT +} +variable "identifier" { + type = string + description = <<-EOT + A unique identifier for the project, this helps when generating names for infrastructure items." + EOT +} +variable "owner" { + type = string + description = <<-EOT + The owner of the project, this helps when generating names for infrastructure items." + EOT +} +variable "zone" { + type = string + description = <<-EOT + The Route53 DNS zone to deploy the cluster into. + This is used to generate the DNS name for the cluster. + The zone must already exist. + EOT +} +variable "rke2_version" { + type = string + description = <<-EOT + The version of rke2 to install on the nodes. + EOT +} +variable "rancher_version" { + type = string + description = <<-EOT + The version of rancher to install on the rke2 cluster. + EOT + default = "2.9.1" +} +variable "file_path" { + type = string + description = <<-EOT + The path to the file containing the rke2 install script. + EOT + default = "./rke2" +} diff --git a/examples/dev/versions.tf b/examples/dev/versions.tf new file mode 100644 index 0000000..ce73c4d --- /dev/null +++ b/examples/dev/versions.tf @@ -0,0 +1,53 @@ +terraform { + required_version = ">= 1.5.0" + required_providers { + file = { + source = "rancher/file" + version = ">= 2.2" + } + random = { + source = "hashicorp/random" + version = ">= 3.5.1" + } + github = { + source = "integrations/github" + version = ">= 5.44" + } + aws = { + source = "hashicorp/aws" + version = ">= 5.11" + } + http = { + source = "hashicorp/http" + version = ">= 3.4" + } + null = { + source = "hashicorp/null" + version = ">= 3" + } + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } + acme = { + source = "vancluever/acme" + version = ">= 2.0" + } + cloudinit = { + source = "hashicorp/cloudinit" + version = ">= 2.3.3" + } + helm = { + source = "hashicorp/helm" + version = "2.14" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.31.0" + } + time = { + source = "hashicorp/time" + version = ">= 0.12.0" + } + } +} diff --git a/examples/downstream/main.tf b/examples/downstream/main.tf index b469855..c6246de 100644 --- a/examples/downstream/main.tf +++ b/examples/downstream/main.tf @@ -40,9 +40,10 @@ locals { # tflint-ignore: terraform_unused_declarations aws_instance_type = "m5.large" # tflint-ignore: terraform_unused_declarations - node_count = 1 - email = (var.email != "" ? var.email : "${local.identifier}@${local.zone}") - acme_server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" #"https://acme-v02.api.letsencrypt.org/directory" + node_count = 1 + email = (var.email != "" ? var.email : "${local.identifier}@${local.zone}") + # "https://acme-staging-v02.api.letsencrypt.org/directory" #"https://acme-v02.api.letsencrypt.org/directory" + acme_server_url = var.acme_server_url helm_chart_values = { "hostname" = "${local.domain}.${local.zone}" "replicas" = "1" diff --git a/examples/downstream/variables.tf b/examples/downstream/variables.tf index c9fb55b..b2e86be 100644 --- a/examples/downstream/variables.tf +++ b/examples/downstream/variables.tf @@ -91,3 +91,11 @@ variable "email" { EOT default = "" } +variable "acme_server_url" { + type = string + description = <<-EOT + The ACME server URL to use for cert-manager. + This is useful for using the Let's Encrypt staging server for testing. + EOT + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} diff --git a/examples/downstream_splitrole/main.tf b/examples/downstream_splitrole/main.tf index 51aaf53..401bcc6 100644 --- a/examples/downstream_splitrole/main.tf +++ b/examples/downstream_splitrole/main.tf @@ -18,22 +18,23 @@ provider "helm" {} # make sure you set the env variable KUBE_CONFIG_PATH t locals { - identifier = var.identifier - example = "downstream_splitrole" - project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" - username = local.project_name - domain = local.project_name - zone = var.zone - key_name = var.key_name - key = var.key - owner = var.owner - rke2_version = var.rke2_version - local_file_path = var.file_path - runner_ip = chomp(data.http.myip.response_body) # "runner" is the server running Terraform - rancher_version = var.rancher_version - cert_manager_version = "1.18.1" - os = "sle-micro-61" - acme_server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" #"https://acme-v02.api.letsencrypt.org/directory" + identifier = var.identifier + example = "downstream_splitrole" + project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" + username = local.project_name + domain = local.project_name + zone = var.zone + key_name = var.key_name + key = var.key + owner = var.owner + rke2_version = var.rke2_version + local_file_path = var.file_path + runner_ip = chomp(data.http.myip.response_body) # "runner" is the server running Terraform + rancher_version = var.rancher_version + cert_manager_version = "1.18.1" + os = "sle-micro-61" + # "https://acme-staging-v02.api.letsencrypt.org/directory" #"https://acme-v02.api.letsencrypt.org/directory" + acme_server_url = var.acme_server_url aws_access_key_id = var.aws_access_key_id aws_secret_access_key = var.aws_secret_access_key aws_region = var.aws_region diff --git a/examples/downstream_splitrole/variables.tf b/examples/downstream_splitrole/variables.tf index 612abf4..e844eb3 100644 --- a/examples/downstream_splitrole/variables.tf +++ b/examples/downstream_splitrole/variables.tf @@ -91,3 +91,11 @@ variable "email" { EOT default = "" } +variable "acme_server_url" { + type = string + description = <<-EOT + The ACME server URL to use for cert-manager. + This is useful for using the Let's Encrypt staging server for testing. + EOT + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} diff --git a/examples/one/main.tf b/examples/one/main.tf index d4c1daf..5c05e66 100644 --- a/examples/one/main.tf +++ b/examples/one/main.tf @@ -17,15 +17,16 @@ provider "helm" {} # make sure you set the env variable KUBE_CONFIG_PATH t locals { - identifier = var.identifier - example = "basic" - project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" - username = local.project_name - domain = local.project_name - zone = var.zone - key_name = var.key_name - key = var.key - acme_server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" # "https://acme-v02.api.letsencrypt.org/directory" + identifier = var.identifier + example = "basic" + project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" + username = local.project_name + domain = local.project_name + zone = var.zone + key_name = var.key_name + key = var.key + # "https://acme-staging-v02.api.letsencrypt.org/directory" or "https://acme-v02.api.letsencrypt.org/directory" + acme_server_url = var.acme_server_url owner = var.owner rke2_version = var.rke2_version local_file_path = var.file_path diff --git a/examples/one/variables.tf b/examples/one/variables.tf index 3d7bc84..64d307f 100644 --- a/examples/one/variables.tf +++ b/examples/one/variables.tf @@ -52,3 +52,11 @@ variable "file_path" { EOT default = "./rke2" } +variable "acme_server_url" { + type = string + description = <<-EOT + The ACME server URL to use for cert-manager. + This is useful for using the Let's Encrypt staging server for testing. + EOT + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} diff --git a/examples/prod/main.tf b/examples/prod/main.tf index cbf6ae5..8896916 100644 --- a/examples/prod/main.tf +++ b/examples/prod/main.tf @@ -17,16 +17,17 @@ provider "helm" {} # make sure you set the env variable KUBE_CONFIG_PATH t locals { - identifier = var.identifier - example = "prod" - project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" - username = local.project_name - domain = local.project_name - email = var.email - zone = var.zone - key_name = var.key_name - key = var.key - acme_server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" # "https://acme-v02.api.letsencrypt.org/directory" + identifier = var.identifier + example = "prod" + project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" + username = local.project_name + domain = local.project_name + email = var.email + zone = var.zone + key_name = var.key_name + key = var.key + # "https://acme-staging-v02.api.letsencrypt.org/directory" # "https://acme-v02.api.letsencrypt.org/directory" + acme_server_url = var.acme_server_url owner = var.owner rke2_version = var.rke2_version local_file_path = var.file_path diff --git a/examples/prod/variables.tf b/examples/prod/variables.tf index 4dcf01f..421cdf8 100644 --- a/examples/prod/variables.tf +++ b/examples/prod/variables.tf @@ -90,3 +90,11 @@ variable "email" { EOT default = "" } +variable "acme_server_url" { + type = string + description = <<-EOT + The ACME server URL to use for cert-manager. + This is useful for using the Let's Encrypt staging server for testing. + EOT + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} diff --git a/examples/three/main.tf b/examples/three/main.tf index 383a073..eb369c5 100644 --- a/examples/three/main.tf +++ b/examples/three/main.tf @@ -27,15 +27,16 @@ terraform { } locals { - identifier = var.identifier - example = "basic" - project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" - username = local.project_name - domain = local.project_name - zone = var.zone - key_name = var.key_name - key = var.key - acme_server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" + identifier = var.identifier + example = "basic" + project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" + username = local.project_name + domain = local.project_name + zone = var.zone + key_name = var.key_name + key = var.key + # "https://acme-staging-v02.api.letsencrypt.org/directory" # "https://acme-v02.api.letsencrypt.org/directory" + acme_server_url = var.acme_server_url owner = var.owner rke2_version = var.rke2_version rancher_helm_repo = "https://releases.rancher.com/server-charts" diff --git a/examples/three/variables.tf b/examples/three/variables.tf index 3d7bc84..64d307f 100644 --- a/examples/three/variables.tf +++ b/examples/three/variables.tf @@ -52,3 +52,11 @@ variable "file_path" { EOT default = "./rke2" } +variable "acme_server_url" { + type = string + description = <<-EOT + The ACME server URL to use for cert-manager. + This is useful for using the Let's Encrypt staging server for testing. + EOT + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} diff --git a/flake.lock b/flake.lock index 032e7aa..fa7acf0 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1759322190, - "narHash": "sha256-s+0wBPx9FAphKv8BYRN7OLCiZkiK1Dc8ebbCzczI9S8=", + "lastModified": 1762604901, + "narHash": "sha256-Pr2jpryIaQr9Yx8p6QssS03wqB6UifnnLr3HJw9veDw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ee4cd98afba682ca27242f2d7fdbd8583f6ef51a", + "rev": "f6b44b2401525650256b977063dbcf830f762369", "type": "github" }, "original": { diff --git a/main.tf b/main.tf index 0b21791..0e1f7a3 100644 --- a/main.tf +++ b/main.tf @@ -73,6 +73,7 @@ locals { ip_family = "ipv4" rancher_helm_chart_values = var.rancher_helm_chart_values rancher_helm_chart_use_strategy = var.rancher_helm_chart_use_strategy + install_rancher = var.install_rancher bootstrap_rancher = var.bootstrap_rancher acme_server_url = var.acme_server_url } @@ -117,13 +118,13 @@ module "install_cert_manager" { cert_manager_configuration = local.cert_manager_config } -module "rancher_bootstrap" { +module "install_rancher" { depends_on = [ module.cluster, module.install_cert_manager, ] - count = (local.bootstrap_rancher ? 1 : 0) - source = "./modules/rancher_bootstrap" + count = (local.install_rancher ? 1 : 0) + source = "./modules/install_rancher" path = local.local_file_path project_domain = local.fqdn zone_id = data.aws_route53_zone.zone.zone_id @@ -141,3 +142,17 @@ module "rancher_bootstrap" { rancher_helm_chart_values = local.rancher_helm_chart_values rancher_helm_chart_use_strategy = local.rancher_helm_chart_use_strategy } + +module "bootstrap_rancher" { + depends_on = [ + module.cluster, + module.install_cert_manager, + module.install_rancher, + ] + count = (local.bootstrap_rancher ? 1 : 0) + source = "./modules/bootstrap_rancher" + path = local.local_file_path + rancher_domain = local.fqdn + ca_certs = module.install_rancher[0].ca_certs + admin_password = module.install_rancher[0].rancher_admin_password +} diff --git a/modules/bootstrap_rancher/bootstrap/main.tf b/modules/bootstrap_rancher/bootstrap/main.tf new file mode 100644 index 0000000..64e875e --- /dev/null +++ b/modules/bootstrap_rancher/bootstrap/main.tf @@ -0,0 +1,19 @@ +locals { + rancher_domain = var.rancher_domain + ca_certs = base64decode(var.ca_certs) + admin_password = var.admin_password +} + +provider "rancher2" { + api_url = "https://${local.rancher_domain}" + bootstrap = true + ca_certs = local.ca_certs + timeout = "300s" +} + +resource "rancher2_bootstrap" "admin" { + initial_password = local.admin_password + password = local.admin_password + token_update = true + token_ttl = 7200 # 2 hours +} diff --git a/modules/rancher_bootstrap/rancher_externalTLS/outputs.tf b/modules/bootstrap_rancher/bootstrap/outputs.tf similarity index 53% rename from modules/rancher_bootstrap/rancher_externalTLS/outputs.tf rename to modules/bootstrap_rancher/bootstrap/outputs.tf index 73c2095..aa3ee58 100644 --- a/modules/rancher_bootstrap/rancher_externalTLS/outputs.tf +++ b/modules/bootstrap_rancher/bootstrap/outputs.tf @@ -1,14 +1,8 @@ -output "admin_token" { - value = rancher2_bootstrap.admin.token - sensitive = true -} - output "admin_password" { - value = random_password.password.result + value = rancher2_bootstrap.admin.password sensitive = true } - -output "ca_certs" { - value = local.ca_certs +output "admin_token" { + value = rancher2_bootstrap.admin.token sensitive = true } diff --git a/modules/bootstrap_rancher/bootstrap/variables.tf b/modules/bootstrap_rancher/bootstrap/variables.tf new file mode 100644 index 0000000..7abeb5b --- /dev/null +++ b/modules/bootstrap_rancher/bootstrap/variables.tf @@ -0,0 +1,15 @@ +variable "rancher_domain" { + description = "The domain name for the Rancher server" + type = string +} + +variable "ca_certs" { + description = "Base64 encoded CA certificate chain to trust when connecting to the Rancher server" + type = string + default = "" +} + +variable "admin_password" { + description = "The initial admin password for the Rancher server" + type = string +} diff --git a/modules/bootstrap_rancher/bootstrap/versions.tf b/modules/bootstrap_rancher/bootstrap/versions.tf new file mode 100644 index 0000000..1fc55ee --- /dev/null +++ b/modules/bootstrap_rancher/bootstrap/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.5.0" + required_providers { + rancher2 = { + source = "rancher/rancher2" + version = ">= 5.0.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.6.2" + } + } +} diff --git a/modules/bootstrap_rancher/main.tf b/modules/bootstrap_rancher/main.tf new file mode 100644 index 0000000..cb99360 --- /dev/null +++ b/modules/bootstrap_rancher/main.tf @@ -0,0 +1,39 @@ +locals { + rancher_domain = var.rancher_domain + ca_certs = var.ca_certs + path = var.path + deploy_path = "${local.path}/bootstrap_rancher" + data_path = local.deploy_path + bootstrap_path = "${path.module}/bootstrap" + kubeconfig_path = "../kubeconfig" # relative to deploy path + admin_password = var.admin_password +} + +module "bootstrap" { + source = "../deploy" + deploy_path = local.deploy_path + data_path = local.data_path + template_files = [ + join("/", [local.bootstrap_path, "main.tf"]), + join("/", [local.bootstrap_path, "outputs.tf"]), + join("/", [local.bootstrap_path, "variables.tf"]), + join("/", [local.bootstrap_path, "versions.tf"]), + ] + attempts = 5 + interval = 60 + skip_destroy = true # this is a one way operation, un-bootstrap not supported + # if any of these change, redeploy/update + deploy_trigger = md5(join("-", [ + local.rancher_domain, + local.ca_certs, + ])) + environment_variables = { + KUBECONFIG = local.kubeconfig_path + KUBE_CONFIG_PATH = local.kubeconfig_path + } + inputs = <<-EOT + rancher_domain = "${local.rancher_domain}" + ca_certs = "${base64encode(local.ca_certs)}" + admin_password = "${local.admin_password}" + EOT +} diff --git a/modules/bootstrap_rancher/outputs.tf b/modules/bootstrap_rancher/outputs.tf new file mode 100644 index 0000000..ae168c6 --- /dev/null +++ b/modules/bootstrap_rancher/outputs.tf @@ -0,0 +1,9 @@ +output "admin_token" { + value = module.bootstrap.output.admin_token + sensitive = true +} + +output "admin_password" { + value = module.bootstrap.output.admin_password + sensitive = true +} diff --git a/modules/bootstrap_rancher/variables.tf b/modules/bootstrap_rancher/variables.tf new file mode 100644 index 0000000..707e589 --- /dev/null +++ b/modules/bootstrap_rancher/variables.tf @@ -0,0 +1,16 @@ +variable "path" { + description = "The root path where files will be deployed from" + type = string +} +variable "rancher_domain" { + description = "The domain name to use for Rancher" + type = string +} +variable "ca_certs" { + description = "The CA certificates to trust when accessing Rancher" + type = string +} +variable "admin_password" { + description = "The initial admin password for Rancher" + type = string +} diff --git a/modules/bootstrap_rancher/versions.tf b/modules/bootstrap_rancher/versions.tf new file mode 100644 index 0000000..e44cd41 --- /dev/null +++ b/modules/bootstrap_rancher/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.5.0" + required_providers { + file = { + source = "rancher/file" + version = ">= 2.2" + } + } +} diff --git a/modules/deploy/create.sh.tpl b/modules/deploy/create.sh.tpl index ebb3a82..9adade9 100644 --- a/modules/deploy/create.sh.tpl +++ b/modules/deploy/create.sh.tpl @@ -4,7 +4,9 @@ cd ${deploy_path} pwd ls -lah whoami -. ${deploy_path}/envrc +. envrc +terraform version + TF_CLI_ARGS_init="" TF_CLI_ARGS_apply="" @@ -18,7 +20,7 @@ E1=0 while [ $EXITCODE -gt 0 ] && [ $ATTEMPTS -lt $MAX ]; do A=0 while [ $E -gt 0 ] && [ $A -lt $MAX ]; do - timeout -k 1m ${timeout} terraform apply -var-file="${deploy_path}/inputs.tfvars" -no-color -auto-approve -state="${deploy_path}/tfstate" + timeout -k 1m ${timeout} terraform apply -var-file="inputs.tfvars" -no-color -auto-approve -state="tfstate" E=$? if [ $E -eq 124 ]; then echo "Apply timed out after ${timeout}"; fi A=$((A+1)) @@ -27,7 +29,7 @@ while [ $EXITCODE -gt 0 ] && [ $ATTEMPTS -lt $MAX ]; do if [ $E -gt 0 ] && [ $ATTEMPTS != $((MAX-1)) ]; then A1=0 while [ $E1 -gt 0 ] && [ $A1 -lt $MAX ]; do - timeout -k 1m ${timeout} terraform destroy -var-file="${deploy_path}/inputs.tfvars" -no-color -auto-approve -state="${deploy_path}/tfstate" + timeout -k 1m ${timeout} terraform destroy -var-file="inputs.tfvars" -no-color -auto-approve -state="tfstate" E1=$? if [ $E1 -eq 124 ]; then echo "Apply timed out after ${timeout}"; fi A1=$((A1+1)) @@ -54,7 +56,7 @@ if [ $ATTEMPTS -eq $MAX ]; then echo "max attempts reached..."; fi if [ $EXITCODE -ne 0 ]; then echo "failure, exit code $EXITCODE..."; fi if [ $EXITCODE -eq 0 ]; then echo "success..."; - terraform output -json -state="${deploy_path}/tfstate" > ${deploy_path}/outputs.json + terraform output -json -state="tfstate" > outputs.json fi cd $DIR exit $EXITCODE diff --git a/modules/deploy/destroy.sh.tpl b/modules/deploy/destroy.sh.tpl index c3ff2fd..e6ac08a 100644 --- a/modules/deploy/destroy.sh.tpl +++ b/modules/deploy/destroy.sh.tpl @@ -4,12 +4,14 @@ cd ${deploy_path} pwd ls -lah whoami -. ${deploy_path}/envrc +. envrc +terraform version + TF_CLI_ARGS_init="" TF_CLI_ARGS_apply="" if [ -z "${skip_destroy}" ]; then timeout -k 1m ${timeout} terraform init -upgrade -reconfigure -no-color - timeout -k 1m ${timeout} terraform destroy -var-file="${deploy_path}/inputs.tfvars" -no-color -auto-approve -state="${deploy_path}/tfstate" || true + timeout -k 1m ${timeout} terraform destroy -var-file="inputs.tfvars" -no-color -auto-approve -state="tfstate" || true else echo "Not destroying deployed module, it will no longer be managed here." fi diff --git a/modules/install_cert_manager/main.tf b/modules/install_cert_manager/main.tf index 1375a3c..c069bf9 100644 --- a/modules/install_cert_manager/main.tf +++ b/modules/install_cert_manager/main.tf @@ -15,12 +15,11 @@ locals { cert_manager_path = "${path.module}/${local.cert_manager_configured}" cert_manager_config = var.cert_manager_configuration deploy_path = "${local.path}/install_cert_manager" + kubeconfig_path = "../kubeconfig" # relative to deploy path } module "deploy_cert_manager" { - source = "../deploy" - depends_on = [ - ] + source = "../deploy" deploy_path = local.deploy_path data_path = local.deploy_path template_files = [ @@ -43,8 +42,8 @@ module "deploy_cert_manager" { ]) ) environment_variables = { - KUBE_CONFIG_PATH = "${abspath(local.path)}/kubeconfig" - KUBECONFIG = "${abspath(local.path)}/kubeconfig" + KUBE_CONFIG_PATH = local.kubeconfig_path + KUBECONFIG = local.kubeconfig_path } inputs = <<-EOT cert_manager_version = "${local.cert_manager_version}" diff --git a/modules/rancher_bootstrap/main.tf b/modules/install_rancher/main.tf similarity index 95% rename from modules/rancher_bootstrap/main.tf rename to modules/install_rancher/main.tf index 8b9ecee..4fbaa17 100644 --- a/modules/rancher_bootstrap/main.tf +++ b/modules/install_rancher/main.tf @@ -16,6 +16,7 @@ locals { externalTLS = var.externalTLS rancher_path = (local.externalTLS ? "${path.module}/rancher_externalTLS" : "${path.module}/rancher") deploy_path = "${local.path}/rancher_bootstrap" + kubeconfig_path = "../kubeconfig" # relative to deploy path rancher_helm_chart_values = var.rancher_helm_chart_values rancher_helm_chart_use_strategy = var.rancher_helm_chart_use_strategy cert_public = var.cert_public @@ -59,8 +60,8 @@ module "deploy_rancher" { local.cert_chain, ])) environment_variables = { - KUBECONFIG = "${local.path}/kubeconfig" - KUBE_CONFIG_PATH = "${local.path}/kubeconfig" + KUBECONFIG = local.kubeconfig_path + KUBE_CONFIG_PATH = local.kubeconfig_path } inputs = <<-EOT project_domain = "${local.project_domain}" diff --git a/modules/install_rancher/outputs.tf b/modules/install_rancher/outputs.tf new file mode 100644 index 0000000..aff93ea --- /dev/null +++ b/modules/install_rancher/outputs.tf @@ -0,0 +1,9 @@ +output "ca_certs" { + value = module.deploy_rancher.output.ca_certs + sensitive = true +} + +output "rancher_admin_password" { + value = module.deploy_rancher.output.rancher_admin_password + sensitive = true +} diff --git a/modules/rancher_bootstrap/rancher/main.tf b/modules/install_rancher/rancher/main.tf similarity index 90% rename from modules/rancher_bootstrap/rancher/main.tf rename to modules/install_rancher/rancher/main.tf index e960b53..afabe0e 100644 --- a/modules/rancher_bootstrap/rancher/main.tf +++ b/modules/install_rancher/rancher/main.tf @@ -9,7 +9,7 @@ locals { default_hc_values = { "hostname" = local.rancher_domain "replicas" = "3" - "bootstrapPassword" = "admin" + "bootstrapPassword" = random_password.admin_password.result "ingress.enabled" = "true" "ingress.tls.source" = "letsEncrypt" "tls" = "ingress" @@ -22,11 +22,11 @@ locals { "additionalTrustedCAs" = "true" "ingress.extraAnnotations.cert-manager\\.io\\/issuer" = "rancher" } - helm_chart_values = coalesce( # using coalesce like this essentially gives us a switch function + helm_chart_values = tomap(coalesce( # using coalesce like this essentially gives us a switch function (local.helm_chart_use_strategy == "merge" ? merge(local.default_hc_values, local.rancher_helm_chart_values) : null), (local.helm_chart_use_strategy == "default" ? local.default_hc_values : null), (local.helm_chart_use_strategy == "provide" ? local.rancher_helm_chart_values : null), - ) # WARNING! Some config is necessary, if the result is an empty string the coalesce will fail + )) # WARNING! Some config is necessary, if the result is an empty string the coalesce will fail zone_id = var.zone_id region = var.region email = var.email @@ -34,6 +34,12 @@ locals { acme_server = var.acme_server_url } +resource "random_password" "admin_password" { + length = 16 + special = true + override_special = "!#$%&-_=+" +} + resource "time_sleep" "settle_before_rancher" { create_duration = "30s" } @@ -154,7 +160,8 @@ resource "helm_release" "rancher" { timeout = 1800 # 30m dynamic "set" { - for_each = local.helm_chart_values + # Terraform won't iterate over sensitive values, so we have to wrap it in nonsensitive() + for_each = nonsensitive(local.helm_chart_values) content { name = set.key type = "string" @@ -300,33 +307,3 @@ resource "terraform_data" "get_public_cert_info" { EOT } } - -provider "rancher2" { - api_url = "https://${local.rancher_domain}" - bootstrap = true - ca_certs = data.kubernetes_secret_v1.certificate.data["tls.crt"] - alias = "bootstrap" -} - -resource "random_password" "password" { - length = 16 - special = true - override_special = "!-_=+" -} - -resource "rancher2_bootstrap" "admin" { - depends_on = [ - time_sleep.settle_before_rancher, - terraform_data.wait_for_nginx, - terraform_data.cattle-system, - kubernetes_manifest.issuer, - helm_release.rancher, - terraform_data.wait_for_rancher, - terraform_data.get_public_cert_info, - data.kubernetes_secret_v1.certificate, - kubernetes_secret.rancher_tls_ca, - kubernetes_secret.rancher_tls_ca_additional, - ] - provider = rancher2.bootstrap - password = random_password.password.result -} diff --git a/modules/install_rancher/rancher/outputs.tf b/modules/install_rancher/rancher/outputs.tf new file mode 100644 index 0000000..e311533 --- /dev/null +++ b/modules/install_rancher/rancher/outputs.tf @@ -0,0 +1,9 @@ +output "ca_certs" { + value = data.kubernetes_secret_v1.certificate.data["tls.crt"] + sensitive = true +} + +output "rancher_admin_password" { + value = random_password.admin_password.result + sensitive = true +} diff --git a/modules/rancher_bootstrap/rancher/runningDeployments.sh b/modules/install_rancher/rancher/runningDeployments.sh similarity index 100% rename from modules/rancher_bootstrap/rancher/runningDeployments.sh rename to modules/install_rancher/rancher/runningDeployments.sh diff --git a/modules/rancher_bootstrap/rancher/runningPods.sh b/modules/install_rancher/rancher/runningPods.sh similarity index 100% rename from modules/rancher_bootstrap/rancher/runningPods.sh rename to modules/install_rancher/rancher/runningPods.sh diff --git a/modules/rancher_bootstrap/rancher/variables.tf b/modules/install_rancher/rancher/variables.tf similarity index 99% rename from modules/rancher_bootstrap/rancher/variables.tf rename to modules/install_rancher/rancher/variables.tf index fecedce..b28f716 100644 --- a/modules/rancher_bootstrap/rancher/variables.tf +++ b/modules/install_rancher/rancher/variables.tf @@ -67,6 +67,7 @@ variable "rancher_helm_chart_values" { } EOT default = "{}" + sensitive = true } variable "zone_id" { type = string diff --git a/modules/rancher_bootstrap/rancher/versions.tf b/modules/install_rancher/rancher/versions.tf similarity index 84% rename from modules/rancher_bootstrap/rancher/versions.tf rename to modules/install_rancher/rancher/versions.tf index a598965..de75423 100644 --- a/modules/rancher_bootstrap/rancher/versions.tf +++ b/modules/install_rancher/rancher/versions.tf @@ -5,10 +5,6 @@ terraform { source = "hashicorp/helm" version = "2.14" } - rancher2 = { - source = "rancher/rancher2" - version = ">= 5.0.0" - } kubernetes = { source = "hashicorp/kubernetes" version = ">= 2.31.0" diff --git a/modules/rancher_bootstrap/rancher_externalTLS/main.tf b/modules/install_rancher/rancher_externalTLS/main.tf similarity index 86% rename from modules/rancher_bootstrap/rancher_externalTLS/main.tf rename to modules/install_rancher/rancher_externalTLS/main.tf index 0f931f8..0adc7cb 100644 --- a/modules/rancher_bootstrap/rancher_externalTLS/main.tf +++ b/modules/install_rancher/rancher_externalTLS/main.tf @@ -15,7 +15,7 @@ locals { default_hc_values = { "hostname" = local.rancher_domain # must be an fqdn "replicas" = "3" - "bootstrapPassword" = "admin" + "bootstrapPassword" = random_password.admin_password.result "ingress.enabled" = "true" "ingress.tls.source" = "secret" "ingress.tls.secretName" = "tls-rancher-ingress" @@ -24,15 +24,18 @@ locals { "additionalTrustedCAs" = "true" } helm_chart_values = coalesce( # using coalesce like this essentially gives us a switch function - (local.helm_chart_use_strategy == "default" ? - local.default_hc_values : null), - (local.helm_chart_use_strategy == "merge" ? - merge(local.default_hc_values, local.rancher_helm_chart_values) : null), - (local.helm_chart_use_strategy == "provide" ? - local.rancher_helm_chart_values : null), + (local.helm_chart_use_strategy == "default" ? local.default_hc_values : null), + (local.helm_chart_use_strategy == "merge" ? merge(local.default_hc_values, local.rancher_helm_chart_values) : null), + (local.helm_chart_use_strategy == "provide" ? local.rancher_helm_chart_values : null), ) # WARNING! helm_chart_use_strategy is required and must be "default", "merge", or "provide", if the strategy isn't found, the coalesce will fail } +resource "random_password" "admin_password" { + length = 16 + special = true + override_special = "!#$%&-_=+" +} + resource "file_local" "hcv" { name = "helm_chart_values.txt" contents = jsonencode(local.rancher_helm_chart_values) @@ -191,7 +194,8 @@ resource "helm_release" "rancher" { timeout = 1800 # 30m dynamic "set" { - for_each = local.helm_chart_values + # Terraform won't iterate over sensitive values, so we have to wrap it in nonsensitive() + for_each = nonsensitive(local.helm_chart_values) content { name = set.key type = "string" @@ -220,12 +224,6 @@ resource "terraform_data" "wait_for_rancher" { } } -resource "random_password" "password" { - length = 16 - special = true - override_special = "!$%&-_=+" -} - resource "terraform_data" "get_public_cert_info" { depends_on = [ time_sleep.settle_before_rancher, @@ -252,26 +250,3 @@ resource "terraform_data" "get_public_cert_info" { EOT } } - -provider "rancher2" { - api_url = "https://${local.rancher_domain}" - bootstrap = true - ca_certs = local.ca_certs - alias = "bootstrap" -} - -resource "rancher2_bootstrap" "admin" { - depends_on = [ - time_sleep.settle_before_rancher, - terraform_data.wait_for_nginx, - terraform_data.cattle-system, - kubernetes_secret.tls_rancher_ingress, - kubernetes_secret.rancher_tls_ca, - kubernetes_secret.rancher_tls_ca_additional, - helm_release.rancher, - terraform_data.wait_for_rancher, - terraform_data.get_public_cert_info, - ] - provider = rancher2.bootstrap - password = random_password.password.result -} diff --git a/modules/install_rancher/rancher_externalTLS/outputs.tf b/modules/install_rancher/rancher_externalTLS/outputs.tf new file mode 100644 index 0000000..59b1427 --- /dev/null +++ b/modules/install_rancher/rancher_externalTLS/outputs.tf @@ -0,0 +1,9 @@ +output "ca_certs" { + value = local.ca_certs + sensitive = true +} + +output "rancher_admin_password" { + value = random_password.admin_password.result + sensitive = true +} diff --git a/modules/rancher_bootstrap/rancher_externalTLS/runningDeployments.sh b/modules/install_rancher/rancher_externalTLS/runningDeployments.sh similarity index 100% rename from modules/rancher_bootstrap/rancher_externalTLS/runningDeployments.sh rename to modules/install_rancher/rancher_externalTLS/runningDeployments.sh diff --git a/modules/rancher_bootstrap/rancher_externalTLS/runningPods.sh b/modules/install_rancher/rancher_externalTLS/runningPods.sh similarity index 100% rename from modules/rancher_bootstrap/rancher_externalTLS/runningPods.sh rename to modules/install_rancher/rancher_externalTLS/runningPods.sh diff --git a/modules/rancher_bootstrap/rancher_externalTLS/variables.tf b/modules/install_rancher/rancher_externalTLS/variables.tf similarity index 99% rename from modules/rancher_bootstrap/rancher_externalTLS/variables.tf rename to modules/install_rancher/rancher_externalTLS/variables.tf index c36931e..8028338 100644 --- a/modules/rancher_bootstrap/rancher_externalTLS/variables.tf +++ b/modules/install_rancher/rancher_externalTLS/variables.tf @@ -62,6 +62,7 @@ variable "rancher_helm_chart_values" { } EOT default = "{}" + sensitive = true } variable "ca_certs" { type = string diff --git a/modules/rancher_bootstrap/versions.tf b/modules/install_rancher/rancher_externalTLS/versions.tf similarity index 86% rename from modules/rancher_bootstrap/versions.tf rename to modules/install_rancher/rancher_externalTLS/versions.tf index 113008a..99c0af6 100644 --- a/modules/rancher_bootstrap/versions.tf +++ b/modules/install_rancher/rancher_externalTLS/versions.tf @@ -1,18 +1,10 @@ terraform { required_version = ">= 1.5.0" required_providers { - file = { - source = "rancher/file" - version = ">= 2.2.0" - } helm = { source = "hashicorp/helm" version = "2.14" } - rancher2 = { - source = "rancher/rancher2" - version = ">= 5.0.0" - } random = { source = "hashicorp/random" version = ">= 3.6.2" @@ -29,5 +21,9 @@ terraform { source = "hashicorp/aws" version = ">= 5.11" } + file = { + source = "rancher/file" + version = ">= 2.2.0" + } } } diff --git a/modules/rancher_bootstrap/variables.tf b/modules/install_rancher/variables.tf similarity index 99% rename from modules/rancher_bootstrap/variables.tf rename to modules/install_rancher/variables.tf index 03700e5..c6ab3b1 100644 --- a/modules/rancher_bootstrap/variables.tf +++ b/modules/install_rancher/variables.tf @@ -114,6 +114,7 @@ variable "rancher_helm_chart_values" { } EOT default = {} + sensitive = true } variable "cert_public" { type = string diff --git a/modules/rancher_bootstrap/rancher_externalTLS/versions.tf b/modules/install_rancher/versions.tf similarity index 86% rename from modules/rancher_bootstrap/rancher_externalTLS/versions.tf rename to modules/install_rancher/versions.tf index 6aabb11..cdf82a1 100644 --- a/modules/rancher_bootstrap/rancher_externalTLS/versions.tf +++ b/modules/install_rancher/versions.tf @@ -1,14 +1,14 @@ terraform { required_version = ">= 1.5.0" required_providers { + file = { + source = "rancher/file" + version = ">= 2.2.0" + } helm = { source = "hashicorp/helm" version = "2.14" } - rancher2 = { - source = "rancher/rancher2" - version = ">= 5.0.0" - } random = { source = "hashicorp/random" version = ">= 3.6.2" @@ -25,9 +25,5 @@ terraform { source = "hashicorp/aws" version = ">= 5.11" } - file = { - source = "rancher/file" - version = ">= 2.2.0" - } } } diff --git a/modules/rancher_bootstrap/outputs.tf b/modules/rancher_bootstrap/outputs.tf deleted file mode 100644 index 67f458d..0000000 --- a/modules/rancher_bootstrap/outputs.tf +++ /dev/null @@ -1,14 +0,0 @@ -output "admin_token" { - value = module.deploy_rancher.output.admin_token - sensitive = true -} - -output "admin_password" { - value = module.deploy_rancher.output.admin_password - sensitive = true -} - -output "ca_certs" { - value = module.deploy_rancher.output.ca_certs - sensitive = true -} diff --git a/modules/rancher_bootstrap/rancher/outputs.tf b/modules/rancher_bootstrap/rancher/outputs.tf deleted file mode 100644 index 336cabc..0000000 --- a/modules/rancher_bootstrap/rancher/outputs.tf +++ /dev/null @@ -1,14 +0,0 @@ -output "admin_token" { - value = rancher2_bootstrap.admin.token - sensitive = true -} - -output "admin_password" { - value = random_password.password.result - sensitive = true -} - -output "ca_certs" { - value = data.kubernetes_secret_v1.certificate.data["tls.crt"] - sensitive = true -} diff --git a/outputs.tf b/outputs.tf index 9e4ccf8..e1bcb9d 100644 --- a/outputs.tf +++ b/outputs.tf @@ -11,12 +11,12 @@ output "address" { } output "admin_token" { - value = module.rancher_bootstrap[0].admin_token + value = try(module.bootstrap_rancher[0].admin_token, "") sensitive = true } output "admin_password" { - value = module.rancher_bootstrap[0].admin_password + value = try(module.install_rancher[0].rancher_admin_password, "") sensitive = true } @@ -39,5 +39,5 @@ output "domain_object" { value = module.cluster.project_domain_object } output "tls_certificate_chain" { - value = module.rancher_bootstrap[0].ca_certs + value = try(module.install_rancher[0].ca_certs, "") } diff --git a/run_tests.sh b/run_tests.sh index 4cde16d..6abe87c 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -218,78 +218,17 @@ if [ -z "$cleanup_id" ]; then fi fi -echo "Clearing leftovers with Id $IDENTIFIER in $AWS_REGION..." - -# shellcheck disable=SC2143 -if [ -n "$IDENTIFIER" ]; then - attempts=0 - # shellcheck disable=SC2143 - while [ -n "$(leftovers -d --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" | grep -v 'AccessDenied')" ] && [ $attempts -lt 3 ]; do - leftovers --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" --no-confirm | grep -v 'AccessDenied' || true - sleep 10 - attempts=$((attempts + 1)) - done - - if [ $attempts -eq 3 ]; then - echo "Warning: Failed to clear all resources after 3 attempts." - fi - - # remove key pairs - attempts=0 - # shellcheck disable=SC2143 - while [ -n "$(leftovers -d --iaas=aws --aws-region="$AWS_REGION" --type="ec2-key-pair" --filter="terraform-ci-$IDENTIFIER" | grep -v 'AccessDenied')" ] && [ $attempts -lt 3 ]; do - leftovers --iaas=aws --aws-region="$AWS_REGION" --type="ec2-key-pair" --filter="terraform-ci-$IDENTIFIER" --no-confirm | grep -v 'AccessDenied' || true - sleep 10 - attempts=$((attempts + 1)) - done - - if [ $attempts -eq 3 ]; then - echo "Warning: Failed to clear all EC2 key pairs after 3 attempts." - fi - - # remove s3 storage - attempts=0 - ID="$(aws s3 ls | grep -i "$IDENTIFIER" | awk '{print $3}')" - # shellcheck disable=SC2143 - while [ -n "$(aws s3 ls | grep -i "$IDENTIFIER")" ] && [ $attempts -lt 3 ]; do - echo "found s3 bucket $ID, removing..." - while read -r v; do - if [ -z "$v" ]; then continue; fi; - aws s3api delete-object --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" --key "tfstate" --version-id="$v" - done <<<"$( - aws s3api list-object-versions --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" | jq -r '.Versions[]?.VersionId' - )" - - while read -r v; do - if [ -z "$v" ]; then continue; fi; - aws s3api delete-object --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" --key "tfstate" --version-id="$v"; - done <<<"$( - aws s3api list-object-versions --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" | jq -r '.DeleteMarkers[]?.VersionId' - )" - - aws s3api delete-bucket --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" - - sleep 10 - attempts=$((attempts + 1)) - done +echo "Starting cleanup..." +sh "$REPO_ROOT/cleanup.sh" "$IDENTIFIER" +C=$? +if [ $C -ne 0 ]; then + echo "Cleanup failed with exit code $C" + exit $C +fi +echo "Cleanup completed successfully." - # remove load balancer target groups - attempts=0 - # shellcheck disable=SC2143 - while [ $attempts -lt 3 ]; do - while read -r line; do - if [ -z "$line" ]; then continue; fi - echo "removing load balancer target group, $line..." - aws elbv2 delete-target-group --target-group-arn "$line"; - done <<<"$( - while read -r line; do - if [ -z "$line" ]; then continue; fi - aws elbv2 describe-tags --resource-arns "$line" | jq -r --arg id "$IDENTIFIER" '.TagDescriptions[] | select(any(.Tags[]; .Key == "Id" and .Value == $id)) | .ResourceArn // ""'; - done <<<"$(aws elbv2 describe-target-groups | jq -r '.TargetGroups[]?.TargetGroupArn')" - )" - sleep 10 - attempts=$((attempts + 1)) - done +if [ -n "$cleanup_id" ]; then + exit 0 fi if [ -f "/tmp/${IDENTIFIER}_failed_tests.txt" ]; then diff --git a/test/tests/dev/basic_test.go b/test/tests/dev/basic_test.go new file mode 100644 index 0000000..62ac24d --- /dev/null +++ b/test/tests/dev/basic_test.go @@ -0,0 +1,129 @@ +package three + +import ( + "os" + "path/filepath" + "strings" + "testing" + + aws "github.com/gruntwork-io/terratest/modules/aws" + g "github.com/gruntwork-io/terratest/modules/git" + "github.com/gruntwork-io/terratest/modules/ssh" + "github.com/gruntwork-io/terratest/modules/terraform" + util "github.com/rancher/terraform-rancher2-aws/test/tests" +) + +func TestDevBasic(t *testing.T) { + t.Parallel() + id := util.GetId() + region := util.GetRegion() + directory := "dev" + owner := "terraform-ci@suse.com" + acme_server_url := util.SetAcmeServer() + + repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) + if err != nil { + t.Fatalf("Error getting git root directory: %v", err) + } + + exampleDir := repoRoot + "/examples/" + directory + testDir := repoRoot + "/test/tests/data/" + id + + err = util.CreateTestDirectories(t, id) + if err != nil { + os.RemoveAll(testDir) + t.Fatalf("Error creating test data directories: %s", err) + } + keyPair, err := util.CreateKeypair(t, region, owner, id) + if err != nil { + os.RemoveAll(testDir) + t.Fatalf("Error creating test key pair: %s", err) + } + err = os.WriteFile(testDir+"/id_rsa", []byte(keyPair.KeyPair.PrivateKey), 0600) + if err != nil { + err = aws.DeleteEC2KeyPairE(t, keyPair) + if err != nil { + t.Logf("Failed to destroy key pair: %v", err) + } + os.RemoveAll(testDir) + t.Fatalf("Error creating test key pair: %s", err) + } + sshAgent := ssh.SshAgentWithKeyPair(t, keyPair.KeyPair) + t.Logf("Key %s created and added to agent", keyPair.Name) + + backendTerraformOptions, err := util.CreateObjectStorageBackend(t, testDir, id, owner, region) + tfOptions := []*terraform.Options{backendTerraformOptions} + if err != nil { + t.Log("Test failed, tearing down...") + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) + t.Fatalf("Error creating cluster: %s", err) + } + + // use oldest RKE2, remember it releases much more than Rancher + _, _, rke2Version, err := util.GetRke2Releases() + if err != nil { + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) + t.Fatalf("Error getting Rke2 release version: %s", err) + } + + rancherVersion := os.Getenv("RANCHER_VERSION") + if rancherVersion == "" { + // use stable version if not specified + // using stable prevents problems where the Rancher provider hasn't released to fit the latest Rancher + _, rancherVersion, _, err = util.GetRancherReleases() + } + if err != nil { + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) + t.Fatalf("Error getting Rancher release version: %s", err) + } + + terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ + TerraformDir: exampleDir, + // Variables to pass to our Terraform code using -var options + Vars: map[string]interface{}{ + "identifier": id, + "owner": owner, + "key_name": keyPair.Name, + "key": keyPair.KeyPair.PublicKey, + "zone": os.Getenv("ZONE"), + "rke2_version": rke2Version, + "rancher_version": rancherVersion, + "file_path": testDir, + "acme_server_url": acme_server_url, + }, + // Environment variables to set when running Terraform + EnvVars: map[string]string{ + "AWS_DEFAULT_REGION": region, + "AWS_REGION": region, + "TF_DATA_DIR": testDir, + "TF_IN_AUTOMATION": "1", + "TF_CLI_ARGS_init": "-backend-config=\"bucket=" + strings.ToLower(id) + "\"", + "TF_CLI_ARGS_plan": "-no-color", // using remote state from storage backend + "TF_CLI_ARGS_apply": "-no-color -parallelism=5", + "TF_CLI_ARGS_destroy": "-no-color", + "TF_CLI_ARGS_output": "-no-color", + }, + RetryableTerraformErrors: util.GetRetryableTerraformErrors(), + NoColor: true, + SshAgent: sshAgent, + Reconfigure: true, + Upgrade: true, + }) + // we need to prepend the main options because we need to destroy it before the backend + newTfOptions := []*terraform.Options{terraformOptions, backendTerraformOptions} + _, err = terraform.InitAndApplyE(t, terraformOptions) + if err != nil { + t.Log("Test failed, tearing down...") + util.GetErrorLogs(t, testDir+"/kubeconfig") + util.Teardown(t, testDir, exampleDir, newTfOptions, keyPair, sshAgent) + t.Fatalf("Error creating cluster: %s", err) + } + util.CheckReady(t, testDir+"/kubeconfig") + util.CheckRunning(t, testDir+"/kubeconfig") + if t.Failed() { + t.Log("Test failed...") + } else { + t.Log("Test passed...") + } + util.Teardown(t, testDir, exampleDir, newTfOptions, keyPair, sshAgent) +} diff --git a/test/tests/downstream/basic_test.go b/test/tests/downstream/basic_test.go index 3ce9f0d..99228be 100644 --- a/test/tests/downstream/basic_test.go +++ b/test/tests/downstream/basic_test.go @@ -21,7 +21,7 @@ func TestDownstreamBasic(t *testing.T) { sessionToken := util.GetAwsSessionToken() directory := "downstream" owner := "terraform-ci@suse.com" - util.SetAcmeServer() + acme_server_url := util.SetAcmeServer() repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) if err != nil { @@ -87,6 +87,7 @@ func TestDownstreamBasic(t *testing.T) { "aws_secret_access_key": secretKey, "aws_session_token": sessionToken, "aws_region": region, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/downstream/splitrole_test.go b/test/tests/downstream/splitrole_test.go index 1562c36..507a7be 100644 --- a/test/tests/downstream/splitrole_test.go +++ b/test/tests/downstream/splitrole_test.go @@ -21,7 +21,7 @@ func TestDownstreamSplitrole(t *testing.T) { sessionToken := util.GetAwsSessionToken() directory := "downstream_splitrole" owner := "terraform-ci@suse.com" - util.SetAcmeServer() + acme_server_url := util.SetAcmeServer() repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) if err != nil { @@ -89,6 +89,7 @@ func TestDownstreamSplitrole(t *testing.T) { "aws_secret_access_key": secretKey, "aws_session_token": sessionToken, "aws_region": region, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/one/basic_test.go b/test/tests/one/basic_test.go index ec6169f..9b6a2ba 100644 --- a/test/tests/one/basic_test.go +++ b/test/tests/one/basic_test.go @@ -18,7 +18,7 @@ func TestOneBasic(t *testing.T) { region := util.GetRegion() directory := "one" owner := "terraform-ci@suse.com" - util.SetAcmeServer() + acme_server_url := util.SetAcmeServer() repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) if err != nil { @@ -81,6 +81,7 @@ func TestOneBasic(t *testing.T) { "rke2_version": rke2Version, "rancher_version": rancherVersion, "file_path": testDir, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/prod/basic_test.go b/test/tests/prod/basic_test.go index 78861e7..7fbbf5b 100644 --- a/test/tests/prod/basic_test.go +++ b/test/tests/prod/basic_test.go @@ -21,7 +21,7 @@ func TestProdBasic(t *testing.T) { sessionToken := util.GetAwsSessionToken() directory := "prod" owner := "terraform-ci@suse.com" - util.SetAcmeServer() + acme_server_url := util.SetAcmeServer() repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) if err != nil { @@ -89,6 +89,7 @@ func TestProdBasic(t *testing.T) { "aws_secret_access_key": secretKey, "aws_session_token": sessionToken, "aws_region": region, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/three/basic_test.go b/test/tests/three/basic_test.go index 44a20ea..93f07c6 100644 --- a/test/tests/three/basic_test.go +++ b/test/tests/three/basic_test.go @@ -19,7 +19,7 @@ func TestThreeBasic(t *testing.T) { region := util.GetRegion() directory := "three" owner := "terraform-ci@suse.com" - util.SetAcmeServer() + acme_server_url := util.SetAcmeServer() repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) if err != nil { @@ -89,6 +89,7 @@ func TestThreeBasic(t *testing.T) { "rke2_version": rke2Version, "rancher_version": rancherVersion, "file_path": testDir, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/three/state_test.go b/test/tests/three/state_test.go index 1efe42e..6a0edea 100644 --- a/test/tests/three/state_test.go +++ b/test/tests/three/state_test.go @@ -16,7 +16,7 @@ import ( // This test is the same as basic but it also tests that the state is correctly stored in S3 and can be used to re-create the cluster func TestThreeState(t *testing.T) { t.Parallel() - util.SetAcmeServer() + acme_server_url := til.SetAcmeServer() id := util.GetId() region := util.GetRegion() @@ -89,6 +89,7 @@ func TestThreeState(t *testing.T) { "rke2_version": rke2Version, "rancher_version": rancherVersion, "file_path": testDir, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/util.go b/test/tests/util.go index dfd5faa..e7e6b4c 100644 --- a/test/tests/util.go +++ b/test/tests/util.go @@ -249,7 +249,7 @@ func removeZeroPadding(v *[]string) { func CreateKeypair(t *testing.T, region string, owner string, id string) (*aws.Ec2Keypair, error) { t.Log("Creating keypair...") // Create an EC2 KeyPair that we can use for SSH access - keyPairName := fmt.Sprintf("terraform-ci-%s", id) + keyPairName := fmt.Sprintf("%s", id) keyPair := aws.CreateAndImportEC2KeyPair(t, region, keyPairName) // tag the key pair so we can find in the access module diff --git a/variables.tf b/variables.tf index a8d7d02..e506a4f 100644 --- a/variables.tf +++ b/variables.tf @@ -209,7 +209,7 @@ variable "rancher_helm_channel" { EOT default = "stable" } -variable "bootstrap_rancher" { +variable "install_rancher" { type = bool description = <<-EOT Whether or not to install Rancher, defaults to true. @@ -218,6 +218,15 @@ variable "bootstrap_rancher" { EOT default = true } +variable "bootstrap_rancher" { + type = bool + description = <<-EOT + Whether or not to bootstrap Rancher, defaults to true. + When this is false the module will install Rancher using helm, but it won't instantiate the rancher provider. + This mostly exists to provide a convenient way to test provider changes. + EOT + default = true +} variable "cert_manager_configuration" { type = object({ aws_access_key_id = string @@ -269,6 +278,7 @@ variable "rancher_helm_chart_values" { } EOT default = {} + sensitive = true } variable "acme_server_url" { type = string diff --git a/versions.tf b/versions.tf index 9434f99..c269279 100644 --- a/versions.tf +++ b/versions.tf @@ -37,10 +37,6 @@ terraform { source = "hashicorp/helm" version = "2.14" } - rancher2 = { - source = "rancher/rancher2" - version = ">= 5.0.0" - } kubernetes = { source = "hashicorp/kubernetes" version = ">= 2.31.0" From 640b42057605a11a2b561fd55bd4f66f1e6d5d08 Mon Sep 17 00:00:00 2001 From: matttrach Date: Wed, 12 Nov 2025 17:25:29 -0600 Subject: [PATCH 2/3] fix: remove cluster data from development output Signed-off-by: matttrach --- examples/dev/README.md | 2 ++ examples/dev/outputs.tf | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/dev/README.md b/examples/dev/README.md index 62f93da..ca65779 100644 --- a/examples/dev/README.md +++ b/examples/dev/README.md @@ -3,3 +3,5 @@ This example is specifically designed to help test the Rancher provider. The idea is to have an example that builds up everything you need to get started with the provider without actually implementing it. This example stops after installing Rancher with the Helm chart. + +The Rancher provider isn't instantiated in this example. diff --git a/examples/dev/outputs.tf b/examples/dev/outputs.tf index 41bccff..d9eca66 100644 --- a/examples/dev/outputs.tf +++ b/examples/dev/outputs.tf @@ -16,7 +16,3 @@ output "admin_password" { value = module.rancher.admin_password sensitive = true } -# output "cluster_data" { -# value = jsonencode(data.rancher2_cluster.local) -# sensitive = true -# } From 37fc86ebeeab4195fa498e893184a9cb2a1c6890 Mon Sep 17 00:00:00 2001 From: matttrach Date: Wed, 12 Nov 2025 17:32:53 -0600 Subject: [PATCH 3/3] fix: format tests Signed-off-by: matttrach --- test/tests/dev/basic_test.go | 2 +- test/tests/downstream/basic_test.go | 2 +- test/tests/downstream/splitrole_test.go | 2 +- test/tests/one/basic_test.go | 2 +- test/tests/prod/basic_test.go | 2 +- test/tests/three/basic_test.go | 2 +- test/tests/three/state_test.go | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/tests/dev/basic_test.go b/test/tests/dev/basic_test.go index 62ac24d..642b249 100644 --- a/test/tests/dev/basic_test.go +++ b/test/tests/dev/basic_test.go @@ -89,7 +89,7 @@ func TestDevBasic(t *testing.T) { "rke2_version": rke2Version, "rancher_version": rancherVersion, "file_path": testDir, - "acme_server_url": acme_server_url, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/downstream/basic_test.go b/test/tests/downstream/basic_test.go index 99228be..9817753 100644 --- a/test/tests/downstream/basic_test.go +++ b/test/tests/downstream/basic_test.go @@ -87,7 +87,7 @@ func TestDownstreamBasic(t *testing.T) { "aws_secret_access_key": secretKey, "aws_session_token": sessionToken, "aws_region": region, - "acme_server_url": acme_server_url, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/downstream/splitrole_test.go b/test/tests/downstream/splitrole_test.go index 507a7be..8e42bf6 100644 --- a/test/tests/downstream/splitrole_test.go +++ b/test/tests/downstream/splitrole_test.go @@ -89,7 +89,7 @@ func TestDownstreamSplitrole(t *testing.T) { "aws_secret_access_key": secretKey, "aws_session_token": sessionToken, "aws_region": region, - "acme_server_url": acme_server_url, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/one/basic_test.go b/test/tests/one/basic_test.go index 9b6a2ba..8f4c35c 100644 --- a/test/tests/one/basic_test.go +++ b/test/tests/one/basic_test.go @@ -81,7 +81,7 @@ func TestOneBasic(t *testing.T) { "rke2_version": rke2Version, "rancher_version": rancherVersion, "file_path": testDir, - "acme_server_url": acme_server_url, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/prod/basic_test.go b/test/tests/prod/basic_test.go index 7fbbf5b..73baffb 100644 --- a/test/tests/prod/basic_test.go +++ b/test/tests/prod/basic_test.go @@ -89,7 +89,7 @@ func TestProdBasic(t *testing.T) { "aws_secret_access_key": secretKey, "aws_session_token": sessionToken, "aws_region": region, - "acme_server_url": acme_server_url, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/three/basic_test.go b/test/tests/three/basic_test.go index 93f07c6..cef0735 100644 --- a/test/tests/three/basic_test.go +++ b/test/tests/three/basic_test.go @@ -89,7 +89,7 @@ func TestThreeBasic(t *testing.T) { "rke2_version": rke2Version, "rancher_version": rancherVersion, "file_path": testDir, - "acme_server_url": acme_server_url, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/three/state_test.go b/test/tests/three/state_test.go index 6a0edea..23585bd 100644 --- a/test/tests/three/state_test.go +++ b/test/tests/three/state_test.go @@ -89,7 +89,7 @@ func TestThreeState(t *testing.T) { "rke2_version": rke2Version, "rancher_version": rancherVersion, "file_path": testDir, - "acme_server_url": acme_server_url, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{