diff --git a/cleanup.sh b/cleanup.sh new file mode 100755 index 0000000..bfb6a84 --- /dev/null +++ b/cleanup.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +# This script is run by the run_tests.sh script to clean up AWS resources created during testing. +# It can also be run independently to clean up resources by providing a cleanup ID. +cleanup_id="$1" +if [ -z "$cleanup_id" ]; then + echo "No cleanup Id provided. Exiting." + exit 1 +fi +echo "Starting cleanup for Id: $cleanup_id" +IDENTIFIER="$cleanup_id" +AWS_REGION="${AWS_REGION:-us-west-2}" + +echo "Clearing leftovers with Id $IDENTIFIER in $AWS_REGION..." + +max_attempts=3 + +attempts=0 +resources_to_clear="$(leftovers -d --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" | grep -v 'AccessDenied')" +while [ -n "$resources_to_clear" ] && [ $attempts -lt $max_attempts ]; do + echo -e "found these resources to clear:\n $resources_to_clear\n" + leftovers --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" --no-confirm | grep -v 'AccessDenied' || true + sleep 10 + resources_to_clear="$(leftovers -d --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" | grep -v 'AccessDenied')" + if [ -n "$resources_to_clear" ]; then + echo "Some resources failed to clear, retrying in $((attempts * 10)) seconds..." + fi + sleep $((attempts * 10)) + attempts=$((attempts + 1)) +done + +if [ $attempts -eq $max_attempts ]; then + echo "Warning: Failed to clear all resources after $max_attempts attempts." +fi + +# remove secrets +attempts=0 +while [ $attempts -lt $max_attempts ]; do + while read -r arn; do + if [ -z "$arn" ]; then + continue + fi + echo "removing secret $arn..." + aws secretsmanager delete-secret --secret-id "$arn" --force-delete-without-recovery + done <<<"$(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "secretsmanager:secret" --tag-filters "Key=Id,Values=$IDENTIFIER" | jq -r '.ResourceTagMappingList[]?.ResourceARN')" + sleep $((attempts * 10)) + attempts=$((attempts + 1)) +done + +# remove s3 storage +attempts=0 +while [ $attempts -lt $max_attempts ]; do + while read -r id; do + if [ -z "$id" ]; then + continue + fi + echo "removing s3 bucket $id..." + aws s3 rb "s3://$id" --force + done <<<"$(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "s3:bucket" --tag-filters "Key=Id,Values=$IDENTIFIER" | jq -r '.ResourceTagMappingList[]?.ResourceARN' | awk -F'arn:aws:s3:::' '{print $2}')" + sleep $((attempts * 10)) + attempts=$((attempts + 1)) +done + +# remove key pairs +attempts=0 +while [ $attempts -lt $max_attempts ]; do + while read -r id; do + if [ -z "$id" ]; then + continue + fi + echo "removing ec2 key pair $id..." + aws ec2 delete-key-pair --key-pair-id "$id" + done <<<"$(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "ec2:key-pair" --tag-filters "Key=Id,Values=$IDENTIFIER" | jq -r '.ResourceTagMappingList[]?.ResourceARN' | awk -F'/' '{print $2}')" + sleep $((attempts * 10)) + attempts=$((attempts + 1)) +done + +# remove server certificates +# unfortunately get-resources doesn't support iam server certificates +attempts=0 +while [ $attempts -lt $max_attempts ]; do + while read -r name; do + if [ -z "$name" ]; then + continue + fi + if aws iam list-server-certificate-tags --server-certificate-name "$name" | jq -e --arg ID "$IDENTIFIER" '.Tags[] | select(.Key=="Id" and .Value==$ID)' > /dev/null; then + echo "removing iam server certificate $name..." + aws iam delete-server-certificate --server-certificate-name "$name" + fi + done <<<"$(aws iam list-server-certificates | jq -r '.ServerCertificateMetadataList[].ServerCertificateName')" + sleep $((attempts * 10)) + attempts=$((attempts + 1)) +done + +# remove load balancer target groups +attempts=0 +while [ $attempts -lt $max_attempts ]; do + while read -r arn; do + if [ -z "$arn" ]; then + continue + fi + echo "removing load balancer target group $arn..." + aws elbv2 delete-target-group --target-group-arn "$arn"; + done <<<"$(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "elasticloadbalancing:targetgroup" --tag-filters "Key=Id,Values=$IDENTIFIER" | jq -r '.ResourceTagMappingList[]?.ResourceARN')" + sleep $((attempts * 10)) + attempts=$((attempts + 1)) +done + +echo "Cleanup completed." + +# These examples find Ids that need to be cleaned up by looking for resources owned by CI and extracting their Id tags. +# This is useful if you happen to come across leftover resources and want to clean up anything that might have been missed with their specific Id. +# For example, if you hit a quota limit and notice there a bunch of leftover secrets or target groups, you can run these commands to clean up all resources with the same Id as the leftover resources. +# for id in $(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "elasticloadbalancing:targetgroup" --tag-filters "Key=Owner,Values=terraform-ci@suse.com" | jq -r '.ResourceTagMappingList[]?.Tags[] | select(.Key=="Id") | .Value'); do ./cleanup.sh "$id"; done +# for id in $(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "secretsmanager:secret" --tag-filters "Key=Owner,Values=terraform-ci@suse.com" | jq -r '.ResourceTagMappingList[]?.Tags[] | select(.Key=="Id") | .Value'); do ./cleanup.sh "$id"; done +# for id in $(for name in $(aws iam list-server-certificates | jq -r '.ServerCertificateMetadataList[].ServerCertificateName'); do echo "$(aws iam list-server-certificate-tags --server-certificate-name "$name" | jq -r '.Tags[] | select(.Key=="Id").Value')"; done); do echo "$id"; done diff --git a/examples/dev/README.md b/examples/dev/README.md new file mode 100644 index 0000000..ca65779 --- /dev/null +++ b/examples/dev/README.md @@ -0,0 +1,7 @@ +# Development Example + +This example is specifically designed to help test the Rancher provider. +The idea is to have an example that builds up everything you need to get started with the provider without actually implementing it. +This example stops after installing Rancher with the Helm chart. + +The Rancher provider isn't instantiated in this example. diff --git a/examples/dev/main.tf b/examples/dev/main.tf new file mode 100644 index 0000000..629ee8f --- /dev/null +++ b/examples/dev/main.tf @@ -0,0 +1,122 @@ +provider "aws" { + default_tags { + tags = { + Id = local.identifier + Owner = local.owner + } + } +} + +provider "github" {} +provider "kubernetes" {} # make sure you set the env variable KUBE_CONFIG_PATH to local_file_path (file_path variable) +provider "helm" {} # make sure you set the env variable KUBE_CONFIG_PATH to local_file_path (file_path variable) + + +terraform { + backend "s3" { + # This needs to be set in the backend configs on the command line or somewhere that your identifier can be set. + # terraform init -reconfigure -backend-config="bucket=" + # https://developer.hashicorp.com/terraform/language/backend/s3 + # https://developer.hashicorp.com/terraform/language/backend#partial-configuration + key = "tfstate" + } +} + +locals { + identifier = var.identifier + example = "dev" + project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" + username = local.project_name + domain = local.project_name + zone = var.zone + key_name = var.key_name + key = var.key + owner = var.owner + rke2_version = var.rke2_version + rancher_helm_repo = "https://releases.rancher.com/server-charts" + rancher_helm_channel = "stable" + helm_chart_strategy = "provide" + # These options use the Let's Encrypt cert that the module generates for you when you deploy the VPC and Domain. + # WARNING! "hostname" must be an fqdn + helm_chart_values = { + "hostname" = "${local.domain}.${local.zone}" + "replicas" = "1" + "bootstrapPassword" = random_password.admin_password.result + "tls" = "ingress" + "ingress.enabled" = "true" + "ingress.tls.source" = "secret" + "ingress.tls.secretName" = "tls-rancher-ingress" + "certmanager.version" = local.cert_manager_version + "agentTLSMode" = "strict" + "privateCA" = "true" + "additionalTrustedCAs" = "true" + } + node_configuration = { + "rancher" = { + type = "all-in-one" + size = "xxl" + os = local.os + indirect_access = true + initial = true + } + } + local_file_path = var.file_path + runner_ip = chomp(data.http.myip.response_body) # "runner" is the server running Terraform + rancher_version = var.rancher_version + cert_manager_version = "1.18.1" + os = "sle-micro-61" +} + +resource "random_password" "admin_password" { + length = 16 + special = true + override_special = "!#$%&-_=+" +} + + +data "http" "myip" { + url = "https://ipinfo.io/ip" +} + +# you shouldn't do this in production, I am trying to show/prove self-signed certificates working with the Rancher configuration +# this could easily be replaced by some secret resource from Vault or if you are using Terraform 1.11+ you should use the ephemeral resources +module "tls" { + source = "./modules/tls" + domain = "${local.domain}.${local.zone}" +} + +module "rancher" { + depends_on = [ + module.tls, + ] + source = "../../" + # project + identifier = local.identifier + owner = local.owner + project_name = local.project_name + domain = local.domain + zone = local.zone + # access + key_name = local.key_name + key = local.key + username = local.username + admin_ip = local.runner_ip + # rke2 + rke2_version = local.rke2_version + local_file_path = local.local_file_path + install_method = "rpm" # rpm only for now, need to figure out local helm chart installs otherwise + cni = "canal" + node_configuration = local.node_configuration + # rancher + cert_manager_version = local.cert_manager_version + cert_use_strategy = "supply" + tls_public_cert = module.tls.tls_public_certificate # just the cert, not any CA + tls_private_key = module.tls.tls_private_key + tls_public_chain = module.tls.certificate_chain # just the chain, it should not include the cert itself + rancher_version = local.rancher_version + rancher_helm_repo = local.rancher_helm_repo + rancher_helm_channel = local.rancher_helm_channel + rancher_helm_chart_use_strategy = local.helm_chart_strategy + rancher_helm_chart_values = local.helm_chart_values + bootstrap_rancher = false +} diff --git a/examples/dev/modules/tls/main.tf b/examples/dev/modules/tls/main.tf new file mode 100644 index 0000000..14669c1 --- /dev/null +++ b/examples/dev/modules/tls/main.tf @@ -0,0 +1,57 @@ + +locals { + domain = var.domain +} + +resource "tls_private_key" "ca_key" { + algorithm = "RSA" + rsa_bits = 2048 +} + +resource "tls_self_signed_cert" "ca_cert" { + private_key_pem = tls_private_key.ca_key.private_key_pem + + subject { + common_name = "Example CA" + organization = "Example" + } + + validity_period_hours = 8760 + is_ca_certificate = true + + allowed_uses = [ + "cert_signing", + "crl_signing", + ] +} + +// TLS Certificate +resource "tls_private_key" "tls_key" { + algorithm = "RSA" + rsa_bits = 2048 +} + +resource "tls_cert_request" "tls_csr" { + private_key_pem = tls_private_key.tls_key.private_key_pem + + subject { + common_name = local.domain + organization = "Example" + } + + dns_names = [local.domain] +} + +resource "tls_locally_signed_cert" "tls_cert" { + cert_request_pem = tls_cert_request.tls_csr.cert_request_pem + ca_private_key_pem = tls_private_key.ca_key.private_key_pem + ca_cert_pem = tls_self_signed_cert.ca_cert.cert_pem + + validity_period_hours = 8760 + + allowed_uses = [ + "key_encipherment", + "digital_signature", + "server_auth", + ] +} diff --git a/examples/dev/modules/tls/outputs.tf b/examples/dev/modules/tls/outputs.tf new file mode 100644 index 0000000..7f0832c --- /dev/null +++ b/examples/dev/modules/tls/outputs.tf @@ -0,0 +1,12 @@ +output "tls_public_certificate" { + value = tls_locally_signed_cert.tls_cert.cert_pem +} + +output "tls_private_key" { + value = tls_private_key.tls_key.private_key_pem + sensitive = true +} + +output "certificate_chain" { + value = tls_self_signed_cert.ca_cert.cert_pem +} diff --git a/examples/dev/modules/tls/variables.tf b/examples/dev/modules/tls/variables.tf new file mode 100644 index 0000000..9bbec36 --- /dev/null +++ b/examples/dev/modules/tls/variables.tf @@ -0,0 +1,6 @@ +variable "domain" { + type = string + description = <<-EOT + The domain name to set as common name and dns sans. + EOT +} diff --git a/examples/dev/modules/tls/versions.tf b/examples/dev/modules/tls/versions.tf new file mode 100644 index 0000000..28763e7 --- /dev/null +++ b/examples/dev/modules/tls/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.5.0" + required_providers { + tls = { + source = "hashicorp/tls" + version = ">= 4.0.5" + } + } +} diff --git a/examples/dev/outputs.tf b/examples/dev/outputs.tf new file mode 100644 index 0000000..d9eca66 --- /dev/null +++ b/examples/dev/outputs.tf @@ -0,0 +1,18 @@ +output "kubeconfig" { + value = module.rancher.kubeconfig + description = <<-EOT + The kubeconfig for the server. + EOT + sensitive = true +} +output "address" { + value = module.rancher.address +} +output "admin_token" { + value = module.rancher.admin_token + sensitive = true +} +output "admin_password" { + value = module.rancher.admin_password + sensitive = true +} diff --git a/examples/dev/variables.tf b/examples/dev/variables.tf new file mode 100644 index 0000000..3d7bc84 --- /dev/null +++ b/examples/dev/variables.tf @@ -0,0 +1,54 @@ +variable "key_name" { + type = string + description = <<-EOT + The name of an AWS key pair to use for SSH access to the instance. + This key should already be added to your ssh agent for server authentication. + EOT +} +variable "key" { + type = string + description = <<-EOT + The contents of an AWS key pair to use for SSH access to the instance. + This is necessary for installing rke2 on the nodes and will be removed after installation. + EOT +} +variable "identifier" { + type = string + description = <<-EOT + A unique identifier for the project, this helps when generating names for infrastructure items." + EOT +} +variable "owner" { + type = string + description = <<-EOT + The owner of the project, this helps when generating names for infrastructure items." + EOT +} +variable "zone" { + type = string + description = <<-EOT + The Route53 DNS zone to deploy the cluster into. + This is used to generate the DNS name for the cluster. + The zone must already exist. + EOT +} +variable "rke2_version" { + type = string + description = <<-EOT + The version of rke2 to install on the nodes. + EOT +} +variable "rancher_version" { + type = string + description = <<-EOT + The version of rancher to install on the rke2 cluster. + EOT + default = "2.9.1" +} +variable "file_path" { + type = string + description = <<-EOT + The path to the file containing the rke2 install script. + EOT + default = "./rke2" +} diff --git a/examples/dev/versions.tf b/examples/dev/versions.tf new file mode 100644 index 0000000..ce73c4d --- /dev/null +++ b/examples/dev/versions.tf @@ -0,0 +1,53 @@ +terraform { + required_version = ">= 1.5.0" + required_providers { + file = { + source = "rancher/file" + version = ">= 2.2" + } + random = { + source = "hashicorp/random" + version = ">= 3.5.1" + } + github = { + source = "integrations/github" + version = ">= 5.44" + } + aws = { + source = "hashicorp/aws" + version = ">= 5.11" + } + http = { + source = "hashicorp/http" + version = ">= 3.4" + } + null = { + source = "hashicorp/null" + version = ">= 3" + } + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } + acme = { + source = "vancluever/acme" + version = ">= 2.0" + } + cloudinit = { + source = "hashicorp/cloudinit" + version = ">= 2.3.3" + } + helm = { + source = "hashicorp/helm" + version = "2.14" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.31.0" + } + time = { + source = "hashicorp/time" + version = ">= 0.12.0" + } + } +} diff --git a/examples/downstream/main.tf b/examples/downstream/main.tf index b469855..c6246de 100644 --- a/examples/downstream/main.tf +++ b/examples/downstream/main.tf @@ -40,9 +40,10 @@ locals { # tflint-ignore: terraform_unused_declarations aws_instance_type = "m5.large" # tflint-ignore: terraform_unused_declarations - node_count = 1 - email = (var.email != "" ? var.email : "${local.identifier}@${local.zone}") - acme_server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" #"https://acme-v02.api.letsencrypt.org/directory" + node_count = 1 + email = (var.email != "" ? var.email : "${local.identifier}@${local.zone}") + # "https://acme-staging-v02.api.letsencrypt.org/directory" #"https://acme-v02.api.letsencrypt.org/directory" + acme_server_url = var.acme_server_url helm_chart_values = { "hostname" = "${local.domain}.${local.zone}" "replicas" = "1" diff --git a/examples/downstream/variables.tf b/examples/downstream/variables.tf index c9fb55b..b2e86be 100644 --- a/examples/downstream/variables.tf +++ b/examples/downstream/variables.tf @@ -91,3 +91,11 @@ variable "email" { EOT default = "" } +variable "acme_server_url" { + type = string + description = <<-EOT + The ACME server URL to use for cert-manager. + This is useful for using the Let's Encrypt staging server for testing. + EOT + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} diff --git a/examples/downstream_splitrole/main.tf b/examples/downstream_splitrole/main.tf index 51aaf53..401bcc6 100644 --- a/examples/downstream_splitrole/main.tf +++ b/examples/downstream_splitrole/main.tf @@ -18,22 +18,23 @@ provider "helm" {} # make sure you set the env variable KUBE_CONFIG_PATH t locals { - identifier = var.identifier - example = "downstream_splitrole" - project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" - username = local.project_name - domain = local.project_name - zone = var.zone - key_name = var.key_name - key = var.key - owner = var.owner - rke2_version = var.rke2_version - local_file_path = var.file_path - runner_ip = chomp(data.http.myip.response_body) # "runner" is the server running Terraform - rancher_version = var.rancher_version - cert_manager_version = "1.18.1" - os = "sle-micro-61" - acme_server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" #"https://acme-v02.api.letsencrypt.org/directory" + identifier = var.identifier + example = "downstream_splitrole" + project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" + username = local.project_name + domain = local.project_name + zone = var.zone + key_name = var.key_name + key = var.key + owner = var.owner + rke2_version = var.rke2_version + local_file_path = var.file_path + runner_ip = chomp(data.http.myip.response_body) # "runner" is the server running Terraform + rancher_version = var.rancher_version + cert_manager_version = "1.18.1" + os = "sle-micro-61" + # "https://acme-staging-v02.api.letsencrypt.org/directory" #"https://acme-v02.api.letsencrypt.org/directory" + acme_server_url = var.acme_server_url aws_access_key_id = var.aws_access_key_id aws_secret_access_key = var.aws_secret_access_key aws_region = var.aws_region diff --git a/examples/downstream_splitrole/variables.tf b/examples/downstream_splitrole/variables.tf index 612abf4..e844eb3 100644 --- a/examples/downstream_splitrole/variables.tf +++ b/examples/downstream_splitrole/variables.tf @@ -91,3 +91,11 @@ variable "email" { EOT default = "" } +variable "acme_server_url" { + type = string + description = <<-EOT + The ACME server URL to use for cert-manager. + This is useful for using the Let's Encrypt staging server for testing. + EOT + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} diff --git a/examples/one/main.tf b/examples/one/main.tf index d4c1daf..5c05e66 100644 --- a/examples/one/main.tf +++ b/examples/one/main.tf @@ -17,15 +17,16 @@ provider "helm" {} # make sure you set the env variable KUBE_CONFIG_PATH t locals { - identifier = var.identifier - example = "basic" - project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" - username = local.project_name - domain = local.project_name - zone = var.zone - key_name = var.key_name - key = var.key - acme_server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" # "https://acme-v02.api.letsencrypt.org/directory" + identifier = var.identifier + example = "basic" + project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" + username = local.project_name + domain = local.project_name + zone = var.zone + key_name = var.key_name + key = var.key + # "https://acme-staging-v02.api.letsencrypt.org/directory" or "https://acme-v02.api.letsencrypt.org/directory" + acme_server_url = var.acme_server_url owner = var.owner rke2_version = var.rke2_version local_file_path = var.file_path diff --git a/examples/one/variables.tf b/examples/one/variables.tf index 3d7bc84..64d307f 100644 --- a/examples/one/variables.tf +++ b/examples/one/variables.tf @@ -52,3 +52,11 @@ variable "file_path" { EOT default = "./rke2" } +variable "acme_server_url" { + type = string + description = <<-EOT + The ACME server URL to use for cert-manager. + This is useful for using the Let's Encrypt staging server for testing. + EOT + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} diff --git a/examples/prod/main.tf b/examples/prod/main.tf index cbf6ae5..8896916 100644 --- a/examples/prod/main.tf +++ b/examples/prod/main.tf @@ -17,16 +17,17 @@ provider "helm" {} # make sure you set the env variable KUBE_CONFIG_PATH t locals { - identifier = var.identifier - example = "prod" - project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" - username = local.project_name - domain = local.project_name - email = var.email - zone = var.zone - key_name = var.key_name - key = var.key - acme_server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" # "https://acme-v02.api.letsencrypt.org/directory" + identifier = var.identifier + example = "prod" + project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" + username = local.project_name + domain = local.project_name + email = var.email + zone = var.zone + key_name = var.key_name + key = var.key + # "https://acme-staging-v02.api.letsencrypt.org/directory" # "https://acme-v02.api.letsencrypt.org/directory" + acme_server_url = var.acme_server_url owner = var.owner rke2_version = var.rke2_version local_file_path = var.file_path diff --git a/examples/prod/variables.tf b/examples/prod/variables.tf index 4dcf01f..421cdf8 100644 --- a/examples/prod/variables.tf +++ b/examples/prod/variables.tf @@ -90,3 +90,11 @@ variable "email" { EOT default = "" } +variable "acme_server_url" { + type = string + description = <<-EOT + The ACME server URL to use for cert-manager. + This is useful for using the Let's Encrypt staging server for testing. + EOT + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} diff --git a/examples/three/main.tf b/examples/three/main.tf index 383a073..eb369c5 100644 --- a/examples/three/main.tf +++ b/examples/three/main.tf @@ -27,15 +27,16 @@ terraform { } locals { - identifier = var.identifier - example = "basic" - project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" - username = local.project_name - domain = local.project_name - zone = var.zone - key_name = var.key_name - key = var.key - acme_server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" + identifier = var.identifier + example = "basic" + project_name = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}" + username = local.project_name + domain = local.project_name + zone = var.zone + key_name = var.key_name + key = var.key + # "https://acme-staging-v02.api.letsencrypt.org/directory" # "https://acme-v02.api.letsencrypt.org/directory" + acme_server_url = var.acme_server_url owner = var.owner rke2_version = var.rke2_version rancher_helm_repo = "https://releases.rancher.com/server-charts" diff --git a/examples/three/variables.tf b/examples/three/variables.tf index 3d7bc84..64d307f 100644 --- a/examples/three/variables.tf +++ b/examples/three/variables.tf @@ -52,3 +52,11 @@ variable "file_path" { EOT default = "./rke2" } +variable "acme_server_url" { + type = string + description = <<-EOT + The ACME server URL to use for cert-manager. + This is useful for using the Let's Encrypt staging server for testing. + EOT + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} diff --git a/flake.lock b/flake.lock index 032e7aa..fa7acf0 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1759322190, - "narHash": "sha256-s+0wBPx9FAphKv8BYRN7OLCiZkiK1Dc8ebbCzczI9S8=", + "lastModified": 1762604901, + "narHash": "sha256-Pr2jpryIaQr9Yx8p6QssS03wqB6UifnnLr3HJw9veDw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ee4cd98afba682ca27242f2d7fdbd8583f6ef51a", + "rev": "f6b44b2401525650256b977063dbcf830f762369", "type": "github" }, "original": { diff --git a/main.tf b/main.tf index 0b21791..0e1f7a3 100644 --- a/main.tf +++ b/main.tf @@ -73,6 +73,7 @@ locals { ip_family = "ipv4" rancher_helm_chart_values = var.rancher_helm_chart_values rancher_helm_chart_use_strategy = var.rancher_helm_chart_use_strategy + install_rancher = var.install_rancher bootstrap_rancher = var.bootstrap_rancher acme_server_url = var.acme_server_url } @@ -117,13 +118,13 @@ module "install_cert_manager" { cert_manager_configuration = local.cert_manager_config } -module "rancher_bootstrap" { +module "install_rancher" { depends_on = [ module.cluster, module.install_cert_manager, ] - count = (local.bootstrap_rancher ? 1 : 0) - source = "./modules/rancher_bootstrap" + count = (local.install_rancher ? 1 : 0) + source = "./modules/install_rancher" path = local.local_file_path project_domain = local.fqdn zone_id = data.aws_route53_zone.zone.zone_id @@ -141,3 +142,17 @@ module "rancher_bootstrap" { rancher_helm_chart_values = local.rancher_helm_chart_values rancher_helm_chart_use_strategy = local.rancher_helm_chart_use_strategy } + +module "bootstrap_rancher" { + depends_on = [ + module.cluster, + module.install_cert_manager, + module.install_rancher, + ] + count = (local.bootstrap_rancher ? 1 : 0) + source = "./modules/bootstrap_rancher" + path = local.local_file_path + rancher_domain = local.fqdn + ca_certs = module.install_rancher[0].ca_certs + admin_password = module.install_rancher[0].rancher_admin_password +} diff --git a/modules/bootstrap_rancher/bootstrap/main.tf b/modules/bootstrap_rancher/bootstrap/main.tf new file mode 100644 index 0000000..64e875e --- /dev/null +++ b/modules/bootstrap_rancher/bootstrap/main.tf @@ -0,0 +1,19 @@ +locals { + rancher_domain = var.rancher_domain + ca_certs = base64decode(var.ca_certs) + admin_password = var.admin_password +} + +provider "rancher2" { + api_url = "https://${local.rancher_domain}" + bootstrap = true + ca_certs = local.ca_certs + timeout = "300s" +} + +resource "rancher2_bootstrap" "admin" { + initial_password = local.admin_password + password = local.admin_password + token_update = true + token_ttl = 7200 # 2 hours +} diff --git a/modules/rancher_bootstrap/rancher_externalTLS/outputs.tf b/modules/bootstrap_rancher/bootstrap/outputs.tf similarity index 53% rename from modules/rancher_bootstrap/rancher_externalTLS/outputs.tf rename to modules/bootstrap_rancher/bootstrap/outputs.tf index 73c2095..aa3ee58 100644 --- a/modules/rancher_bootstrap/rancher_externalTLS/outputs.tf +++ b/modules/bootstrap_rancher/bootstrap/outputs.tf @@ -1,14 +1,8 @@ -output "admin_token" { - value = rancher2_bootstrap.admin.token - sensitive = true -} - output "admin_password" { - value = random_password.password.result + value = rancher2_bootstrap.admin.password sensitive = true } - -output "ca_certs" { - value = local.ca_certs +output "admin_token" { + value = rancher2_bootstrap.admin.token sensitive = true } diff --git a/modules/bootstrap_rancher/bootstrap/variables.tf b/modules/bootstrap_rancher/bootstrap/variables.tf new file mode 100644 index 0000000..7abeb5b --- /dev/null +++ b/modules/bootstrap_rancher/bootstrap/variables.tf @@ -0,0 +1,15 @@ +variable "rancher_domain" { + description = "The domain name for the Rancher server" + type = string +} + +variable "ca_certs" { + description = "Base64 encoded CA certificate chain to trust when connecting to the Rancher server" + type = string + default = "" +} + +variable "admin_password" { + description = "The initial admin password for the Rancher server" + type = string +} diff --git a/modules/bootstrap_rancher/bootstrap/versions.tf b/modules/bootstrap_rancher/bootstrap/versions.tf new file mode 100644 index 0000000..1fc55ee --- /dev/null +++ b/modules/bootstrap_rancher/bootstrap/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.5.0" + required_providers { + rancher2 = { + source = "rancher/rancher2" + version = ">= 5.0.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.6.2" + } + } +} diff --git a/modules/bootstrap_rancher/main.tf b/modules/bootstrap_rancher/main.tf new file mode 100644 index 0000000..cb99360 --- /dev/null +++ b/modules/bootstrap_rancher/main.tf @@ -0,0 +1,39 @@ +locals { + rancher_domain = var.rancher_domain + ca_certs = var.ca_certs + path = var.path + deploy_path = "${local.path}/bootstrap_rancher" + data_path = local.deploy_path + bootstrap_path = "${path.module}/bootstrap" + kubeconfig_path = "../kubeconfig" # relative to deploy path + admin_password = var.admin_password +} + +module "bootstrap" { + source = "../deploy" + deploy_path = local.deploy_path + data_path = local.data_path + template_files = [ + join("/", [local.bootstrap_path, "main.tf"]), + join("/", [local.bootstrap_path, "outputs.tf"]), + join("/", [local.bootstrap_path, "variables.tf"]), + join("/", [local.bootstrap_path, "versions.tf"]), + ] + attempts = 5 + interval = 60 + skip_destroy = true # this is a one way operation, un-bootstrap not supported + # if any of these change, redeploy/update + deploy_trigger = md5(join("-", [ + local.rancher_domain, + local.ca_certs, + ])) + environment_variables = { + KUBECONFIG = local.kubeconfig_path + KUBE_CONFIG_PATH = local.kubeconfig_path + } + inputs = <<-EOT + rancher_domain = "${local.rancher_domain}" + ca_certs = "${base64encode(local.ca_certs)}" + admin_password = "${local.admin_password}" + EOT +} diff --git a/modules/bootstrap_rancher/outputs.tf b/modules/bootstrap_rancher/outputs.tf new file mode 100644 index 0000000..ae168c6 --- /dev/null +++ b/modules/bootstrap_rancher/outputs.tf @@ -0,0 +1,9 @@ +output "admin_token" { + value = module.bootstrap.output.admin_token + sensitive = true +} + +output "admin_password" { + value = module.bootstrap.output.admin_password + sensitive = true +} diff --git a/modules/bootstrap_rancher/variables.tf b/modules/bootstrap_rancher/variables.tf new file mode 100644 index 0000000..707e589 --- /dev/null +++ b/modules/bootstrap_rancher/variables.tf @@ -0,0 +1,16 @@ +variable "path" { + description = "The root path where files will be deployed from" + type = string +} +variable "rancher_domain" { + description = "The domain name to use for Rancher" + type = string +} +variable "ca_certs" { + description = "The CA certificates to trust when accessing Rancher" + type = string +} +variable "admin_password" { + description = "The initial admin password for Rancher" + type = string +} diff --git a/modules/bootstrap_rancher/versions.tf b/modules/bootstrap_rancher/versions.tf new file mode 100644 index 0000000..e44cd41 --- /dev/null +++ b/modules/bootstrap_rancher/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.5.0" + required_providers { + file = { + source = "rancher/file" + version = ">= 2.2" + } + } +} diff --git a/modules/deploy/create.sh.tpl b/modules/deploy/create.sh.tpl index ebb3a82..9adade9 100644 --- a/modules/deploy/create.sh.tpl +++ b/modules/deploy/create.sh.tpl @@ -4,7 +4,9 @@ cd ${deploy_path} pwd ls -lah whoami -. ${deploy_path}/envrc +. envrc +terraform version + TF_CLI_ARGS_init="" TF_CLI_ARGS_apply="" @@ -18,7 +20,7 @@ E1=0 while [ $EXITCODE -gt 0 ] && [ $ATTEMPTS -lt $MAX ]; do A=0 while [ $E -gt 0 ] && [ $A -lt $MAX ]; do - timeout -k 1m ${timeout} terraform apply -var-file="${deploy_path}/inputs.tfvars" -no-color -auto-approve -state="${deploy_path}/tfstate" + timeout -k 1m ${timeout} terraform apply -var-file="inputs.tfvars" -no-color -auto-approve -state="tfstate" E=$? if [ $E -eq 124 ]; then echo "Apply timed out after ${timeout}"; fi A=$((A+1)) @@ -27,7 +29,7 @@ while [ $EXITCODE -gt 0 ] && [ $ATTEMPTS -lt $MAX ]; do if [ $E -gt 0 ] && [ $ATTEMPTS != $((MAX-1)) ]; then A1=0 while [ $E1 -gt 0 ] && [ $A1 -lt $MAX ]; do - timeout -k 1m ${timeout} terraform destroy -var-file="${deploy_path}/inputs.tfvars" -no-color -auto-approve -state="${deploy_path}/tfstate" + timeout -k 1m ${timeout} terraform destroy -var-file="inputs.tfvars" -no-color -auto-approve -state="tfstate" E1=$? if [ $E1 -eq 124 ]; then echo "Apply timed out after ${timeout}"; fi A1=$((A1+1)) @@ -54,7 +56,7 @@ if [ $ATTEMPTS -eq $MAX ]; then echo "max attempts reached..."; fi if [ $EXITCODE -ne 0 ]; then echo "failure, exit code $EXITCODE..."; fi if [ $EXITCODE -eq 0 ]; then echo "success..."; - terraform output -json -state="${deploy_path}/tfstate" > ${deploy_path}/outputs.json + terraform output -json -state="tfstate" > outputs.json fi cd $DIR exit $EXITCODE diff --git a/modules/deploy/destroy.sh.tpl b/modules/deploy/destroy.sh.tpl index c3ff2fd..e6ac08a 100644 --- a/modules/deploy/destroy.sh.tpl +++ b/modules/deploy/destroy.sh.tpl @@ -4,12 +4,14 @@ cd ${deploy_path} pwd ls -lah whoami -. ${deploy_path}/envrc +. envrc +terraform version + TF_CLI_ARGS_init="" TF_CLI_ARGS_apply="" if [ -z "${skip_destroy}" ]; then timeout -k 1m ${timeout} terraform init -upgrade -reconfigure -no-color - timeout -k 1m ${timeout} terraform destroy -var-file="${deploy_path}/inputs.tfvars" -no-color -auto-approve -state="${deploy_path}/tfstate" || true + timeout -k 1m ${timeout} terraform destroy -var-file="inputs.tfvars" -no-color -auto-approve -state="tfstate" || true else echo "Not destroying deployed module, it will no longer be managed here." fi diff --git a/modules/install_cert_manager/main.tf b/modules/install_cert_manager/main.tf index 1375a3c..c069bf9 100644 --- a/modules/install_cert_manager/main.tf +++ b/modules/install_cert_manager/main.tf @@ -15,12 +15,11 @@ locals { cert_manager_path = "${path.module}/${local.cert_manager_configured}" cert_manager_config = var.cert_manager_configuration deploy_path = "${local.path}/install_cert_manager" + kubeconfig_path = "../kubeconfig" # relative to deploy path } module "deploy_cert_manager" { - source = "../deploy" - depends_on = [ - ] + source = "../deploy" deploy_path = local.deploy_path data_path = local.deploy_path template_files = [ @@ -43,8 +42,8 @@ module "deploy_cert_manager" { ]) ) environment_variables = { - KUBE_CONFIG_PATH = "${abspath(local.path)}/kubeconfig" - KUBECONFIG = "${abspath(local.path)}/kubeconfig" + KUBE_CONFIG_PATH = local.kubeconfig_path + KUBECONFIG = local.kubeconfig_path } inputs = <<-EOT cert_manager_version = "${local.cert_manager_version}" diff --git a/modules/rancher_bootstrap/main.tf b/modules/install_rancher/main.tf similarity index 95% rename from modules/rancher_bootstrap/main.tf rename to modules/install_rancher/main.tf index 8b9ecee..4fbaa17 100644 --- a/modules/rancher_bootstrap/main.tf +++ b/modules/install_rancher/main.tf @@ -16,6 +16,7 @@ locals { externalTLS = var.externalTLS rancher_path = (local.externalTLS ? "${path.module}/rancher_externalTLS" : "${path.module}/rancher") deploy_path = "${local.path}/rancher_bootstrap" + kubeconfig_path = "../kubeconfig" # relative to deploy path rancher_helm_chart_values = var.rancher_helm_chart_values rancher_helm_chart_use_strategy = var.rancher_helm_chart_use_strategy cert_public = var.cert_public @@ -59,8 +60,8 @@ module "deploy_rancher" { local.cert_chain, ])) environment_variables = { - KUBECONFIG = "${local.path}/kubeconfig" - KUBE_CONFIG_PATH = "${local.path}/kubeconfig" + KUBECONFIG = local.kubeconfig_path + KUBE_CONFIG_PATH = local.kubeconfig_path } inputs = <<-EOT project_domain = "${local.project_domain}" diff --git a/modules/install_rancher/outputs.tf b/modules/install_rancher/outputs.tf new file mode 100644 index 0000000..aff93ea --- /dev/null +++ b/modules/install_rancher/outputs.tf @@ -0,0 +1,9 @@ +output "ca_certs" { + value = module.deploy_rancher.output.ca_certs + sensitive = true +} + +output "rancher_admin_password" { + value = module.deploy_rancher.output.rancher_admin_password + sensitive = true +} diff --git a/modules/rancher_bootstrap/rancher/main.tf b/modules/install_rancher/rancher/main.tf similarity index 90% rename from modules/rancher_bootstrap/rancher/main.tf rename to modules/install_rancher/rancher/main.tf index e960b53..afabe0e 100644 --- a/modules/rancher_bootstrap/rancher/main.tf +++ b/modules/install_rancher/rancher/main.tf @@ -9,7 +9,7 @@ locals { default_hc_values = { "hostname" = local.rancher_domain "replicas" = "3" - "bootstrapPassword" = "admin" + "bootstrapPassword" = random_password.admin_password.result "ingress.enabled" = "true" "ingress.tls.source" = "letsEncrypt" "tls" = "ingress" @@ -22,11 +22,11 @@ locals { "additionalTrustedCAs" = "true" "ingress.extraAnnotations.cert-manager\\.io\\/issuer" = "rancher" } - helm_chart_values = coalesce( # using coalesce like this essentially gives us a switch function + helm_chart_values = tomap(coalesce( # using coalesce like this essentially gives us a switch function (local.helm_chart_use_strategy == "merge" ? merge(local.default_hc_values, local.rancher_helm_chart_values) : null), (local.helm_chart_use_strategy == "default" ? local.default_hc_values : null), (local.helm_chart_use_strategy == "provide" ? local.rancher_helm_chart_values : null), - ) # WARNING! Some config is necessary, if the result is an empty string the coalesce will fail + )) # WARNING! Some config is necessary, if the result is an empty string the coalesce will fail zone_id = var.zone_id region = var.region email = var.email @@ -34,6 +34,12 @@ locals { acme_server = var.acme_server_url } +resource "random_password" "admin_password" { + length = 16 + special = true + override_special = "!#$%&-_=+" +} + resource "time_sleep" "settle_before_rancher" { create_duration = "30s" } @@ -154,7 +160,8 @@ resource "helm_release" "rancher" { timeout = 1800 # 30m dynamic "set" { - for_each = local.helm_chart_values + # Terraform won't iterate over sensitive values, so we have to wrap it in nonsensitive() + for_each = nonsensitive(local.helm_chart_values) content { name = set.key type = "string" @@ -300,33 +307,3 @@ resource "terraform_data" "get_public_cert_info" { EOT } } - -provider "rancher2" { - api_url = "https://${local.rancher_domain}" - bootstrap = true - ca_certs = data.kubernetes_secret_v1.certificate.data["tls.crt"] - alias = "bootstrap" -} - -resource "random_password" "password" { - length = 16 - special = true - override_special = "!-_=+" -} - -resource "rancher2_bootstrap" "admin" { - depends_on = [ - time_sleep.settle_before_rancher, - terraform_data.wait_for_nginx, - terraform_data.cattle-system, - kubernetes_manifest.issuer, - helm_release.rancher, - terraform_data.wait_for_rancher, - terraform_data.get_public_cert_info, - data.kubernetes_secret_v1.certificate, - kubernetes_secret.rancher_tls_ca, - kubernetes_secret.rancher_tls_ca_additional, - ] - provider = rancher2.bootstrap - password = random_password.password.result -} diff --git a/modules/install_rancher/rancher/outputs.tf b/modules/install_rancher/rancher/outputs.tf new file mode 100644 index 0000000..e311533 --- /dev/null +++ b/modules/install_rancher/rancher/outputs.tf @@ -0,0 +1,9 @@ +output "ca_certs" { + value = data.kubernetes_secret_v1.certificate.data["tls.crt"] + sensitive = true +} + +output "rancher_admin_password" { + value = random_password.admin_password.result + sensitive = true +} diff --git a/modules/rancher_bootstrap/rancher/runningDeployments.sh b/modules/install_rancher/rancher/runningDeployments.sh similarity index 100% rename from modules/rancher_bootstrap/rancher/runningDeployments.sh rename to modules/install_rancher/rancher/runningDeployments.sh diff --git a/modules/rancher_bootstrap/rancher/runningPods.sh b/modules/install_rancher/rancher/runningPods.sh similarity index 100% rename from modules/rancher_bootstrap/rancher/runningPods.sh rename to modules/install_rancher/rancher/runningPods.sh diff --git a/modules/rancher_bootstrap/rancher/variables.tf b/modules/install_rancher/rancher/variables.tf similarity index 99% rename from modules/rancher_bootstrap/rancher/variables.tf rename to modules/install_rancher/rancher/variables.tf index fecedce..b28f716 100644 --- a/modules/rancher_bootstrap/rancher/variables.tf +++ b/modules/install_rancher/rancher/variables.tf @@ -67,6 +67,7 @@ variable "rancher_helm_chart_values" { } EOT default = "{}" + sensitive = true } variable "zone_id" { type = string diff --git a/modules/rancher_bootstrap/rancher/versions.tf b/modules/install_rancher/rancher/versions.tf similarity index 84% rename from modules/rancher_bootstrap/rancher/versions.tf rename to modules/install_rancher/rancher/versions.tf index a598965..de75423 100644 --- a/modules/rancher_bootstrap/rancher/versions.tf +++ b/modules/install_rancher/rancher/versions.tf @@ -5,10 +5,6 @@ terraform { source = "hashicorp/helm" version = "2.14" } - rancher2 = { - source = "rancher/rancher2" - version = ">= 5.0.0" - } kubernetes = { source = "hashicorp/kubernetes" version = ">= 2.31.0" diff --git a/modules/rancher_bootstrap/rancher_externalTLS/main.tf b/modules/install_rancher/rancher_externalTLS/main.tf similarity index 86% rename from modules/rancher_bootstrap/rancher_externalTLS/main.tf rename to modules/install_rancher/rancher_externalTLS/main.tf index 0f931f8..0adc7cb 100644 --- a/modules/rancher_bootstrap/rancher_externalTLS/main.tf +++ b/modules/install_rancher/rancher_externalTLS/main.tf @@ -15,7 +15,7 @@ locals { default_hc_values = { "hostname" = local.rancher_domain # must be an fqdn "replicas" = "3" - "bootstrapPassword" = "admin" + "bootstrapPassword" = random_password.admin_password.result "ingress.enabled" = "true" "ingress.tls.source" = "secret" "ingress.tls.secretName" = "tls-rancher-ingress" @@ -24,15 +24,18 @@ locals { "additionalTrustedCAs" = "true" } helm_chart_values = coalesce( # using coalesce like this essentially gives us a switch function - (local.helm_chart_use_strategy == "default" ? - local.default_hc_values : null), - (local.helm_chart_use_strategy == "merge" ? - merge(local.default_hc_values, local.rancher_helm_chart_values) : null), - (local.helm_chart_use_strategy == "provide" ? - local.rancher_helm_chart_values : null), + (local.helm_chart_use_strategy == "default" ? local.default_hc_values : null), + (local.helm_chart_use_strategy == "merge" ? merge(local.default_hc_values, local.rancher_helm_chart_values) : null), + (local.helm_chart_use_strategy == "provide" ? local.rancher_helm_chart_values : null), ) # WARNING! helm_chart_use_strategy is required and must be "default", "merge", or "provide", if the strategy isn't found, the coalesce will fail } +resource "random_password" "admin_password" { + length = 16 + special = true + override_special = "!#$%&-_=+" +} + resource "file_local" "hcv" { name = "helm_chart_values.txt" contents = jsonencode(local.rancher_helm_chart_values) @@ -191,7 +194,8 @@ resource "helm_release" "rancher" { timeout = 1800 # 30m dynamic "set" { - for_each = local.helm_chart_values + # Terraform won't iterate over sensitive values, so we have to wrap it in nonsensitive() + for_each = nonsensitive(local.helm_chart_values) content { name = set.key type = "string" @@ -220,12 +224,6 @@ resource "terraform_data" "wait_for_rancher" { } } -resource "random_password" "password" { - length = 16 - special = true - override_special = "!$%&-_=+" -} - resource "terraform_data" "get_public_cert_info" { depends_on = [ time_sleep.settle_before_rancher, @@ -252,26 +250,3 @@ resource "terraform_data" "get_public_cert_info" { EOT } } - -provider "rancher2" { - api_url = "https://${local.rancher_domain}" - bootstrap = true - ca_certs = local.ca_certs - alias = "bootstrap" -} - -resource "rancher2_bootstrap" "admin" { - depends_on = [ - time_sleep.settle_before_rancher, - terraform_data.wait_for_nginx, - terraform_data.cattle-system, - kubernetes_secret.tls_rancher_ingress, - kubernetes_secret.rancher_tls_ca, - kubernetes_secret.rancher_tls_ca_additional, - helm_release.rancher, - terraform_data.wait_for_rancher, - terraform_data.get_public_cert_info, - ] - provider = rancher2.bootstrap - password = random_password.password.result -} diff --git a/modules/install_rancher/rancher_externalTLS/outputs.tf b/modules/install_rancher/rancher_externalTLS/outputs.tf new file mode 100644 index 0000000..59b1427 --- /dev/null +++ b/modules/install_rancher/rancher_externalTLS/outputs.tf @@ -0,0 +1,9 @@ +output "ca_certs" { + value = local.ca_certs + sensitive = true +} + +output "rancher_admin_password" { + value = random_password.admin_password.result + sensitive = true +} diff --git a/modules/rancher_bootstrap/rancher_externalTLS/runningDeployments.sh b/modules/install_rancher/rancher_externalTLS/runningDeployments.sh similarity index 100% rename from modules/rancher_bootstrap/rancher_externalTLS/runningDeployments.sh rename to modules/install_rancher/rancher_externalTLS/runningDeployments.sh diff --git a/modules/rancher_bootstrap/rancher_externalTLS/runningPods.sh b/modules/install_rancher/rancher_externalTLS/runningPods.sh similarity index 100% rename from modules/rancher_bootstrap/rancher_externalTLS/runningPods.sh rename to modules/install_rancher/rancher_externalTLS/runningPods.sh diff --git a/modules/rancher_bootstrap/rancher_externalTLS/variables.tf b/modules/install_rancher/rancher_externalTLS/variables.tf similarity index 99% rename from modules/rancher_bootstrap/rancher_externalTLS/variables.tf rename to modules/install_rancher/rancher_externalTLS/variables.tf index c36931e..8028338 100644 --- a/modules/rancher_bootstrap/rancher_externalTLS/variables.tf +++ b/modules/install_rancher/rancher_externalTLS/variables.tf @@ -62,6 +62,7 @@ variable "rancher_helm_chart_values" { } EOT default = "{}" + sensitive = true } variable "ca_certs" { type = string diff --git a/modules/rancher_bootstrap/versions.tf b/modules/install_rancher/rancher_externalTLS/versions.tf similarity index 86% rename from modules/rancher_bootstrap/versions.tf rename to modules/install_rancher/rancher_externalTLS/versions.tf index 113008a..99c0af6 100644 --- a/modules/rancher_bootstrap/versions.tf +++ b/modules/install_rancher/rancher_externalTLS/versions.tf @@ -1,18 +1,10 @@ terraform { required_version = ">= 1.5.0" required_providers { - file = { - source = "rancher/file" - version = ">= 2.2.0" - } helm = { source = "hashicorp/helm" version = "2.14" } - rancher2 = { - source = "rancher/rancher2" - version = ">= 5.0.0" - } random = { source = "hashicorp/random" version = ">= 3.6.2" @@ -29,5 +21,9 @@ terraform { source = "hashicorp/aws" version = ">= 5.11" } + file = { + source = "rancher/file" + version = ">= 2.2.0" + } } } diff --git a/modules/rancher_bootstrap/variables.tf b/modules/install_rancher/variables.tf similarity index 99% rename from modules/rancher_bootstrap/variables.tf rename to modules/install_rancher/variables.tf index 03700e5..c6ab3b1 100644 --- a/modules/rancher_bootstrap/variables.tf +++ b/modules/install_rancher/variables.tf @@ -114,6 +114,7 @@ variable "rancher_helm_chart_values" { } EOT default = {} + sensitive = true } variable "cert_public" { type = string diff --git a/modules/rancher_bootstrap/rancher_externalTLS/versions.tf b/modules/install_rancher/versions.tf similarity index 86% rename from modules/rancher_bootstrap/rancher_externalTLS/versions.tf rename to modules/install_rancher/versions.tf index 6aabb11..cdf82a1 100644 --- a/modules/rancher_bootstrap/rancher_externalTLS/versions.tf +++ b/modules/install_rancher/versions.tf @@ -1,14 +1,14 @@ terraform { required_version = ">= 1.5.0" required_providers { + file = { + source = "rancher/file" + version = ">= 2.2.0" + } helm = { source = "hashicorp/helm" version = "2.14" } - rancher2 = { - source = "rancher/rancher2" - version = ">= 5.0.0" - } random = { source = "hashicorp/random" version = ">= 3.6.2" @@ -25,9 +25,5 @@ terraform { source = "hashicorp/aws" version = ">= 5.11" } - file = { - source = "rancher/file" - version = ">= 2.2.0" - } } } diff --git a/modules/rancher_bootstrap/outputs.tf b/modules/rancher_bootstrap/outputs.tf deleted file mode 100644 index 67f458d..0000000 --- a/modules/rancher_bootstrap/outputs.tf +++ /dev/null @@ -1,14 +0,0 @@ -output "admin_token" { - value = module.deploy_rancher.output.admin_token - sensitive = true -} - -output "admin_password" { - value = module.deploy_rancher.output.admin_password - sensitive = true -} - -output "ca_certs" { - value = module.deploy_rancher.output.ca_certs - sensitive = true -} diff --git a/modules/rancher_bootstrap/rancher/outputs.tf b/modules/rancher_bootstrap/rancher/outputs.tf deleted file mode 100644 index 336cabc..0000000 --- a/modules/rancher_bootstrap/rancher/outputs.tf +++ /dev/null @@ -1,14 +0,0 @@ -output "admin_token" { - value = rancher2_bootstrap.admin.token - sensitive = true -} - -output "admin_password" { - value = random_password.password.result - sensitive = true -} - -output "ca_certs" { - value = data.kubernetes_secret_v1.certificate.data["tls.crt"] - sensitive = true -} diff --git a/outputs.tf b/outputs.tf index 9e4ccf8..e1bcb9d 100644 --- a/outputs.tf +++ b/outputs.tf @@ -11,12 +11,12 @@ output "address" { } output "admin_token" { - value = module.rancher_bootstrap[0].admin_token + value = try(module.bootstrap_rancher[0].admin_token, "") sensitive = true } output "admin_password" { - value = module.rancher_bootstrap[0].admin_password + value = try(module.install_rancher[0].rancher_admin_password, "") sensitive = true } @@ -39,5 +39,5 @@ output "domain_object" { value = module.cluster.project_domain_object } output "tls_certificate_chain" { - value = module.rancher_bootstrap[0].ca_certs + value = try(module.install_rancher[0].ca_certs, "") } diff --git a/run_tests.sh b/run_tests.sh index 4cde16d..6abe87c 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -218,78 +218,17 @@ if [ -z "$cleanup_id" ]; then fi fi -echo "Clearing leftovers with Id $IDENTIFIER in $AWS_REGION..." - -# shellcheck disable=SC2143 -if [ -n "$IDENTIFIER" ]; then - attempts=0 - # shellcheck disable=SC2143 - while [ -n "$(leftovers -d --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" | grep -v 'AccessDenied')" ] && [ $attempts -lt 3 ]; do - leftovers --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" --no-confirm | grep -v 'AccessDenied' || true - sleep 10 - attempts=$((attempts + 1)) - done - - if [ $attempts -eq 3 ]; then - echo "Warning: Failed to clear all resources after 3 attempts." - fi - - # remove key pairs - attempts=0 - # shellcheck disable=SC2143 - while [ -n "$(leftovers -d --iaas=aws --aws-region="$AWS_REGION" --type="ec2-key-pair" --filter="terraform-ci-$IDENTIFIER" | grep -v 'AccessDenied')" ] && [ $attempts -lt 3 ]; do - leftovers --iaas=aws --aws-region="$AWS_REGION" --type="ec2-key-pair" --filter="terraform-ci-$IDENTIFIER" --no-confirm | grep -v 'AccessDenied' || true - sleep 10 - attempts=$((attempts + 1)) - done - - if [ $attempts -eq 3 ]; then - echo "Warning: Failed to clear all EC2 key pairs after 3 attempts." - fi - - # remove s3 storage - attempts=0 - ID="$(aws s3 ls | grep -i "$IDENTIFIER" | awk '{print $3}')" - # shellcheck disable=SC2143 - while [ -n "$(aws s3 ls | grep -i "$IDENTIFIER")" ] && [ $attempts -lt 3 ]; do - echo "found s3 bucket $ID, removing..." - while read -r v; do - if [ -z "$v" ]; then continue; fi; - aws s3api delete-object --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" --key "tfstate" --version-id="$v" - done <<<"$( - aws s3api list-object-versions --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" | jq -r '.Versions[]?.VersionId' - )" - - while read -r v; do - if [ -z "$v" ]; then continue; fi; - aws s3api delete-object --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" --key "tfstate" --version-id="$v"; - done <<<"$( - aws s3api list-object-versions --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" | jq -r '.DeleteMarkers[]?.VersionId' - )" - - aws s3api delete-bucket --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" - - sleep 10 - attempts=$((attempts + 1)) - done +echo "Starting cleanup..." +sh "$REPO_ROOT/cleanup.sh" "$IDENTIFIER" +C=$? +if [ $C -ne 0 ]; then + echo "Cleanup failed with exit code $C" + exit $C +fi +echo "Cleanup completed successfully." - # remove load balancer target groups - attempts=0 - # shellcheck disable=SC2143 - while [ $attempts -lt 3 ]; do - while read -r line; do - if [ -z "$line" ]; then continue; fi - echo "removing load balancer target group, $line..." - aws elbv2 delete-target-group --target-group-arn "$line"; - done <<<"$( - while read -r line; do - if [ -z "$line" ]; then continue; fi - aws elbv2 describe-tags --resource-arns "$line" | jq -r --arg id "$IDENTIFIER" '.TagDescriptions[] | select(any(.Tags[]; .Key == "Id" and .Value == $id)) | .ResourceArn // ""'; - done <<<"$(aws elbv2 describe-target-groups | jq -r '.TargetGroups[]?.TargetGroupArn')" - )" - sleep 10 - attempts=$((attempts + 1)) - done +if [ -n "$cleanup_id" ]; then + exit 0 fi if [ -f "/tmp/${IDENTIFIER}_failed_tests.txt" ]; then diff --git a/test/tests/dev/basic_test.go b/test/tests/dev/basic_test.go new file mode 100644 index 0000000..642b249 --- /dev/null +++ b/test/tests/dev/basic_test.go @@ -0,0 +1,129 @@ +package three + +import ( + "os" + "path/filepath" + "strings" + "testing" + + aws "github.com/gruntwork-io/terratest/modules/aws" + g "github.com/gruntwork-io/terratest/modules/git" + "github.com/gruntwork-io/terratest/modules/ssh" + "github.com/gruntwork-io/terratest/modules/terraform" + util "github.com/rancher/terraform-rancher2-aws/test/tests" +) + +func TestDevBasic(t *testing.T) { + t.Parallel() + id := util.GetId() + region := util.GetRegion() + directory := "dev" + owner := "terraform-ci@suse.com" + acme_server_url := util.SetAcmeServer() + + repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) + if err != nil { + t.Fatalf("Error getting git root directory: %v", err) + } + + exampleDir := repoRoot + "/examples/" + directory + testDir := repoRoot + "/test/tests/data/" + id + + err = util.CreateTestDirectories(t, id) + if err != nil { + os.RemoveAll(testDir) + t.Fatalf("Error creating test data directories: %s", err) + } + keyPair, err := util.CreateKeypair(t, region, owner, id) + if err != nil { + os.RemoveAll(testDir) + t.Fatalf("Error creating test key pair: %s", err) + } + err = os.WriteFile(testDir+"/id_rsa", []byte(keyPair.KeyPair.PrivateKey), 0600) + if err != nil { + err = aws.DeleteEC2KeyPairE(t, keyPair) + if err != nil { + t.Logf("Failed to destroy key pair: %v", err) + } + os.RemoveAll(testDir) + t.Fatalf("Error creating test key pair: %s", err) + } + sshAgent := ssh.SshAgentWithKeyPair(t, keyPair.KeyPair) + t.Logf("Key %s created and added to agent", keyPair.Name) + + backendTerraformOptions, err := util.CreateObjectStorageBackend(t, testDir, id, owner, region) + tfOptions := []*terraform.Options{backendTerraformOptions} + if err != nil { + t.Log("Test failed, tearing down...") + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) + t.Fatalf("Error creating cluster: %s", err) + } + + // use oldest RKE2, remember it releases much more than Rancher + _, _, rke2Version, err := util.GetRke2Releases() + if err != nil { + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) + t.Fatalf("Error getting Rke2 release version: %s", err) + } + + rancherVersion := os.Getenv("RANCHER_VERSION") + if rancherVersion == "" { + // use stable version if not specified + // using stable prevents problems where the Rancher provider hasn't released to fit the latest Rancher + _, rancherVersion, _, err = util.GetRancherReleases() + } + if err != nil { + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) + t.Fatalf("Error getting Rancher release version: %s", err) + } + + terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ + TerraformDir: exampleDir, + // Variables to pass to our Terraform code using -var options + Vars: map[string]interface{}{ + "identifier": id, + "owner": owner, + "key_name": keyPair.Name, + "key": keyPair.KeyPair.PublicKey, + "zone": os.Getenv("ZONE"), + "rke2_version": rke2Version, + "rancher_version": rancherVersion, + "file_path": testDir, + "acme_server_url": acme_server_url, + }, + // Environment variables to set when running Terraform + EnvVars: map[string]string{ + "AWS_DEFAULT_REGION": region, + "AWS_REGION": region, + "TF_DATA_DIR": testDir, + "TF_IN_AUTOMATION": "1", + "TF_CLI_ARGS_init": "-backend-config=\"bucket=" + strings.ToLower(id) + "\"", + "TF_CLI_ARGS_plan": "-no-color", // using remote state from storage backend + "TF_CLI_ARGS_apply": "-no-color -parallelism=5", + "TF_CLI_ARGS_destroy": "-no-color", + "TF_CLI_ARGS_output": "-no-color", + }, + RetryableTerraformErrors: util.GetRetryableTerraformErrors(), + NoColor: true, + SshAgent: sshAgent, + Reconfigure: true, + Upgrade: true, + }) + // we need to prepend the main options because we need to destroy it before the backend + newTfOptions := []*terraform.Options{terraformOptions, backendTerraformOptions} + _, err = terraform.InitAndApplyE(t, terraformOptions) + if err != nil { + t.Log("Test failed, tearing down...") + util.GetErrorLogs(t, testDir+"/kubeconfig") + util.Teardown(t, testDir, exampleDir, newTfOptions, keyPair, sshAgent) + t.Fatalf("Error creating cluster: %s", err) + } + util.CheckReady(t, testDir+"/kubeconfig") + util.CheckRunning(t, testDir+"/kubeconfig") + if t.Failed() { + t.Log("Test failed...") + } else { + t.Log("Test passed...") + } + util.Teardown(t, testDir, exampleDir, newTfOptions, keyPair, sshAgent) +} diff --git a/test/tests/downstream/basic_test.go b/test/tests/downstream/basic_test.go index 3ce9f0d..9817753 100644 --- a/test/tests/downstream/basic_test.go +++ b/test/tests/downstream/basic_test.go @@ -21,7 +21,7 @@ func TestDownstreamBasic(t *testing.T) { sessionToken := util.GetAwsSessionToken() directory := "downstream" owner := "terraform-ci@suse.com" - util.SetAcmeServer() + acme_server_url := util.SetAcmeServer() repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) if err != nil { @@ -87,6 +87,7 @@ func TestDownstreamBasic(t *testing.T) { "aws_secret_access_key": secretKey, "aws_session_token": sessionToken, "aws_region": region, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/downstream/splitrole_test.go b/test/tests/downstream/splitrole_test.go index 1562c36..8e42bf6 100644 --- a/test/tests/downstream/splitrole_test.go +++ b/test/tests/downstream/splitrole_test.go @@ -21,7 +21,7 @@ func TestDownstreamSplitrole(t *testing.T) { sessionToken := util.GetAwsSessionToken() directory := "downstream_splitrole" owner := "terraform-ci@suse.com" - util.SetAcmeServer() + acme_server_url := util.SetAcmeServer() repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) if err != nil { @@ -89,6 +89,7 @@ func TestDownstreamSplitrole(t *testing.T) { "aws_secret_access_key": secretKey, "aws_session_token": sessionToken, "aws_region": region, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/one/basic_test.go b/test/tests/one/basic_test.go index ec6169f..8f4c35c 100644 --- a/test/tests/one/basic_test.go +++ b/test/tests/one/basic_test.go @@ -18,7 +18,7 @@ func TestOneBasic(t *testing.T) { region := util.GetRegion() directory := "one" owner := "terraform-ci@suse.com" - util.SetAcmeServer() + acme_server_url := util.SetAcmeServer() repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) if err != nil { @@ -81,6 +81,7 @@ func TestOneBasic(t *testing.T) { "rke2_version": rke2Version, "rancher_version": rancherVersion, "file_path": testDir, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/prod/basic_test.go b/test/tests/prod/basic_test.go index 78861e7..73baffb 100644 --- a/test/tests/prod/basic_test.go +++ b/test/tests/prod/basic_test.go @@ -21,7 +21,7 @@ func TestProdBasic(t *testing.T) { sessionToken := util.GetAwsSessionToken() directory := "prod" owner := "terraform-ci@suse.com" - util.SetAcmeServer() + acme_server_url := util.SetAcmeServer() repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) if err != nil { @@ -89,6 +89,7 @@ func TestProdBasic(t *testing.T) { "aws_secret_access_key": secretKey, "aws_session_token": sessionToken, "aws_region": region, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/three/basic_test.go b/test/tests/three/basic_test.go index 44a20ea..cef0735 100644 --- a/test/tests/three/basic_test.go +++ b/test/tests/three/basic_test.go @@ -19,7 +19,7 @@ func TestThreeBasic(t *testing.T) { region := util.GetRegion() directory := "three" owner := "terraform-ci@suse.com" - util.SetAcmeServer() + acme_server_url := util.SetAcmeServer() repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) if err != nil { @@ -89,6 +89,7 @@ func TestThreeBasic(t *testing.T) { "rke2_version": rke2Version, "rancher_version": rancherVersion, "file_path": testDir, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/three/state_test.go b/test/tests/three/state_test.go index 1efe42e..23585bd 100644 --- a/test/tests/three/state_test.go +++ b/test/tests/three/state_test.go @@ -16,7 +16,7 @@ import ( // This test is the same as basic but it also tests that the state is correctly stored in S3 and can be used to re-create the cluster func TestThreeState(t *testing.T) { t.Parallel() - util.SetAcmeServer() + acme_server_url := til.SetAcmeServer() id := util.GetId() region := util.GetRegion() @@ -89,6 +89,7 @@ func TestThreeState(t *testing.T) { "rke2_version": rke2Version, "rancher_version": rancherVersion, "file_path": testDir, + "acme_server_url": acme_server_url, }, // Environment variables to set when running Terraform EnvVars: map[string]string{ diff --git a/test/tests/util.go b/test/tests/util.go index dfd5faa..e7e6b4c 100644 --- a/test/tests/util.go +++ b/test/tests/util.go @@ -249,7 +249,7 @@ func removeZeroPadding(v *[]string) { func CreateKeypair(t *testing.T, region string, owner string, id string) (*aws.Ec2Keypair, error) { t.Log("Creating keypair...") // Create an EC2 KeyPair that we can use for SSH access - keyPairName := fmt.Sprintf("terraform-ci-%s", id) + keyPairName := fmt.Sprintf("%s", id) keyPair := aws.CreateAndImportEC2KeyPair(t, region, keyPairName) // tag the key pair so we can find in the access module diff --git a/variables.tf b/variables.tf index a8d7d02..e506a4f 100644 --- a/variables.tf +++ b/variables.tf @@ -209,7 +209,7 @@ variable "rancher_helm_channel" { EOT default = "stable" } -variable "bootstrap_rancher" { +variable "install_rancher" { type = bool description = <<-EOT Whether or not to install Rancher, defaults to true. @@ -218,6 +218,15 @@ variable "bootstrap_rancher" { EOT default = true } +variable "bootstrap_rancher" { + type = bool + description = <<-EOT + Whether or not to bootstrap Rancher, defaults to true. + When this is false the module will install Rancher using helm, but it won't instantiate the rancher provider. + This mostly exists to provide a convenient way to test provider changes. + EOT + default = true +} variable "cert_manager_configuration" { type = object({ aws_access_key_id = string @@ -269,6 +278,7 @@ variable "rancher_helm_chart_values" { } EOT default = {} + sensitive = true } variable "acme_server_url" { type = string diff --git a/versions.tf b/versions.tf index 9434f99..c269279 100644 --- a/versions.tf +++ b/versions.tf @@ -37,10 +37,6 @@ terraform { source = "hashicorp/helm" version = "2.14" } - rancher2 = { - source = "rancher/rancher2" - version = ">= 5.0.0" - } kubernetes = { source = "hashicorp/kubernetes" version = ">= 2.31.0"