fix: separate install from bootstrap

cleanup.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+# This script is run by the run_tests.sh script to clean up AWS resources created during testing.
+# It can also be run independently to clean up resources by providing a cleanup ID.
+cleanup_id="$1"
+if [ -z "$cleanup_id" ]; then
+  echo "No cleanup Id provided. Exiting."
+  exit 1
+fi
+echo "Starting cleanup for Id: $cleanup_id"
+IDENTIFIER="$cleanup_id"
+AWS_REGION="${AWS_REGION:-us-west-2}"
+
+echo "Clearing leftovers with Id $IDENTIFIER in $AWS_REGION..."
+
+max_attempts=3
+
+attempts=0
+resources_to_clear="$(leftovers -d --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" | grep -v 'AccessDenied')"
+while [ -n "$resources_to_clear" ] && [ $attempts -lt $max_attempts ]; do
+  echo -e "found these resources to clear:\n $resources_to_clear\n"
+  leftovers --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" --no-confirm | grep -v 'AccessDenied' || true
+  sleep 10
+  resources_to_clear="$(leftovers -d --iaas=aws --aws-region="$AWS_REGION" --filter="Id:$IDENTIFIER" | grep -v 'AccessDenied')"
+  if [ -n "$resources_to_clear" ]; then
+    echo "Some resources failed to clear, retrying in $((attempts * 10)) seconds..."
+  fi
+  sleep $((attempts * 10))
+  attempts=$((attempts + 1))
+done
+
+if [ $attempts -eq $max_attempts ]; then
+  echo "Warning: Failed to clear all resources after $max_attempts attempts."
+fi
+
+# remove secrets
+attempts=0
+while [ $attempts -lt $max_attempts ]; do
+  while read -r arn; do
+    if [ -z "$arn" ]; then
+      continue
+    fi
+    echo "removing secret $arn..."
+    aws secretsmanager delete-secret --secret-id "$arn" --force-delete-without-recovery
+  done <<<"$(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "secretsmanager:secret" --tag-filters "Key=Id,Values=$IDENTIFIER" | jq -r '.ResourceTagMappingList[]?.ResourceARN')"
+  sleep $((attempts * 10))
+  attempts=$((attempts + 1))
+done
+
+# remove s3 storage
+attempts=0
+while [ $attempts -lt $max_attempts ]; do
+  while read -r id; do
+    if [ -z "$id" ]; then
+      continue
+    fi
+    echo "removing s3 bucket $id..."
+    aws s3 rb "s3://$id" --force
+  done <<<"$(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "s3:bucket" --tag-filters "Key=Id,Values=$IDENTIFIER" | jq -r '.ResourceTagMappingList[]?.ResourceARN' | awk -F'arn:aws:s3:::' '{print $2}')"
+  sleep $((attempts * 10))
+  attempts=$((attempts + 1))
+done
+
+# remove key pairs
+attempts=0
+while [ $attempts -lt $max_attempts ]; do
+  while read -r id; do
+    if [ -z "$id" ]; then
+      continue
+    fi
+    echo "removing ec2 key pair $id..."
+    aws ec2 delete-key-pair --key-pair-id "$id"
+  done <<<"$(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "ec2:key-pair" --tag-filters "Key=Id,Values=$IDENTIFIER" | jq -r '.ResourceTagMappingList[]?.ResourceARN' | awk -F'/' '{print $2}')"
+  sleep $((attempts * 10))
+  attempts=$((attempts + 1))
+done
+
+# remove server certificates
+# unfortunately get-resources doesn't support iam server certificates
+attempts=0
+while [ $attempts -lt $max_attempts ]; do
+  while read -r name; do
+    if [ -z "$name" ]; then
+      continue
+    fi
+    if aws iam list-server-certificate-tags --server-certificate-name "$name" | jq -e --arg ID "$IDENTIFIER" '.Tags[] | select(.Key=="Id" and .Value==$ID)' > /dev/null; then
+      echo "removing iam server certificate $name..."
+      aws iam delete-server-certificate --server-certificate-name "$name"
+    fi
+  done <<<"$(aws iam list-server-certificates | jq -r '.ServerCertificateMetadataList[].ServerCertificateName')"
+  sleep $((attempts * 10))
+  attempts=$((attempts + 1))
+done
+
+# remove load balancer target groups
+attempts=0
+while [ $attempts -lt $max_attempts ]; do
+  while read -r arn; do
+    if [ -z "$arn" ]; then
+      continue
+    fi
+    echo "removing load balancer target group $arn..."
+    aws elbv2 delete-target-group --target-group-arn "$arn";
+  done <<<"$(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "elasticloadbalancing:targetgroup" --tag-filters "Key=Id,Values=$IDENTIFIER" | jq -r '.ResourceTagMappingList[]?.ResourceARN')"
+  sleep $((attempts * 10))
+  attempts=$((attempts + 1))
+done
+
+echo "Cleanup completed."
+
+# These examples find Ids that need to be cleaned up by looking for resources owned by CI and extracting their Id tags.
+# This is useful if you happen to come across leftover resources and want to clean up anything that might have been missed with their specific Id.
+# For example, if you hit a quota limit and notice there a bunch of leftover secrets or target groups, you can run these commands to clean up all resources with the same Id as the leftover resources.
+# for id in $(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "elasticloadbalancing:targetgroup" --tag-filters "Key=Owner,Values=terraform-ci@suse.com" | jq -r '.ResourceTagMappingList[]?.Tags[] | select(.Key=="Id") | .Value'); do ./cleanup.sh "$id"; done
+# for id in $(aws resourcegroupstaggingapi get-resources --no-cli-pager --resource-type-filters "secretsmanager:secret" --tag-filters "Key=Owner,Values=terraform-ci@suse.com" | jq -r '.ResourceTagMappingList[]?.Tags[] | select(.Key=="Id") | .Value'); do ./cleanup.sh "$id"; done
+# for id in $(for name in $(aws iam list-server-certificates | jq -r '.ServerCertificateMetadataList[].ServerCertificateName'); do echo "$(aws iam list-server-certificate-tags --server-certificate-name "$name" | jq -r '.Tags[] | select(.Key=="Id").Value')"; done); do echo "$id"; done

examples/dev/README.md

@@ -0,0 +1,7 @@
+# Development Example
+
+This example is specifically designed to help test the Rancher provider.
+The idea is to have an example that builds up everything you need to get started with the provider without actually implementing it.
+This example stops after installing Rancher with the Helm chart.
+
+The Rancher provider isn't instantiated in this example.

examples/dev/main.tf

@@ -0,0 +1,122 @@
+provider "aws" {
+  default_tags {
+    tags = {
+      Id    = local.identifier
+      Owner = local.owner
+    }
+  }
+}
+
+provider "github" {}
+provider "kubernetes" {} # make sure you set the env variable KUBE_CONFIG_PATH to local_file_path (file_path variable)
+provider "helm" {}       # make sure you set the env variable KUBE_CONFIG_PATH to local_file_path (file_path variable)
+
+
+terraform {
+  backend "s3" {
+    # This needs to be set in the backend configs on the command line or somewhere that your identifier can be set.
+    # terraform init -reconfigure -backend-config="bucket=<identifier>"
+    # https://developer.hashicorp.com/terraform/language/backend/s3
+    # https://developer.hashicorp.com/terraform/language/backend#partial-configuration
+    key = "tfstate"
+  }
+}
+
+locals {
+  identifier           = var.identifier
+  example              = "dev"
+  project_name         = "tf-${substr(md5(join("-", [local.example, local.identifier])), 0, 5)}"
+  username             = local.project_name
+  domain               = local.project_name
+  zone                 = var.zone
+  key_name             = var.key_name
+  key                  = var.key
+  owner                = var.owner
+  rke2_version         = var.rke2_version
+  rancher_helm_repo    = "https://releases.rancher.com/server-charts"
+  rancher_helm_channel = "stable"
+  helm_chart_strategy  = "provide"
+  # These options use the Let's Encrypt cert that the module generates for you when you deploy the VPC and Domain.
+  # WARNING! "hostname" must be an fqdn
+  helm_chart_values = {
+    "hostname"               = "${local.domain}.${local.zone}"
+    "replicas"               = "1"
+    "bootstrapPassword"      = random_password.admin_password.result
+    "tls"                    = "ingress"
+    "ingress.enabled"        = "true"
+    "ingress.tls.source"     = "secret"
+    "ingress.tls.secretName" = "tls-rancher-ingress"
+    "certmanager.version"    = local.cert_manager_version
+    "agentTLSMode"           = "strict"
+    "privateCA"              = "true"
+    "additionalTrustedCAs"   = "true"
+  }
+  node_configuration = {
+    "rancher" = {
+      type            = "all-in-one"
+      size            = "xxl"
+      os              = local.os
+      indirect_access = true
+      initial         = true
+    }
+  }
+  local_file_path      = var.file_path
+  runner_ip            = chomp(data.http.myip.response_body) # "runner" is the server running Terraform
+  rancher_version      = var.rancher_version
+  cert_manager_version = "1.18.1"
+  os                   = "sle-micro-61"
+}
+
+resource "random_password" "admin_password" {
+  length           = 16
+  special          = true
+  override_special = "!#$%&-_=+"
+}
+
+
+data "http" "myip" {
+  url = "https://ipinfo.io/ip"
+}
+
+# you shouldn't do this in production, I am trying to show/prove self-signed certificates working with the Rancher configuration
+# this could easily be replaced by some secret resource from Vault or if you are using Terraform 1.11+ you should use the ephemeral resources
+module "tls" {
+  source = "./modules/tls"
+  domain = "${local.domain}.${local.zone}"
+}
+
+module "rancher" {
+  depends_on = [
+    module.tls,
+  ]
+  source = "../../"
+  # project
+  identifier   = local.identifier
+  owner        = local.owner
+  project_name = local.project_name
+  domain       = local.domain
+  zone         = local.zone
+  # access
+  key_name = local.key_name
+  key      = local.key
+  username = local.username
+  admin_ip = local.runner_ip
+  # rke2
+  rke2_version       = local.rke2_version
+  local_file_path    = local.local_file_path
+  install_method     = "rpm" # rpm only for now, need to figure out local helm chart installs otherwise
+  cni                = "canal"
+  node_configuration = local.node_configuration
+  # rancher
+  cert_manager_version            = local.cert_manager_version
+  cert_use_strategy               = "supply"
+  tls_public_cert                 = module.tls.tls_public_certificate # just the cert, not any CA
+  tls_private_key                 = module.tls.tls_private_key
+  tls_public_chain                = module.tls.certificate_chain # just the chain, it should not include the cert itself
+  rancher_version                 = local.rancher_version
+  rancher_helm_repo               = local.rancher_helm_repo
+  rancher_helm_channel            = local.rancher_helm_channel
+  rancher_helm_chart_use_strategy = local.helm_chart_strategy
+  rancher_helm_chart_values       = local.helm_chart_values
+  bootstrap_rancher               = false
+}

examples/dev/modules/tls/main.tf

@@ -0,0 +1,57 @@
+
+locals {
+  domain = var.domain
+}
+
+resource "tls_private_key" "ca_key" {
+  algorithm = "RSA"
+  rsa_bits  = 2048
+}
+
+resource "tls_self_signed_cert" "ca_cert" {
+  private_key_pem = tls_private_key.ca_key.private_key_pem
+
+  subject {
+    common_name  = "Example CA"
+    organization = "Example"
+  }
+
+  validity_period_hours = 8760
+  is_ca_certificate     = true
+
+  allowed_uses = [
+    "cert_signing",
+    "crl_signing",
+  ]
+}
+
+// TLS Certificate
+resource "tls_private_key" "tls_key" {
+  algorithm = "RSA"
+  rsa_bits  = 2048
+}
+
+resource "tls_cert_request" "tls_csr" {
+  private_key_pem = tls_private_key.tls_key.private_key_pem
+
+  subject {
+    common_name  = local.domain
+    organization = "Example"
+  }
+
+  dns_names = [local.domain]
+}
+
+resource "tls_locally_signed_cert" "tls_cert" {
+  cert_request_pem   = tls_cert_request.tls_csr.cert_request_pem
+  ca_private_key_pem = tls_private_key.ca_key.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.ca_cert.cert_pem
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "server_auth",
+  ]
+}

examples/dev/modules/tls/outputs.tf

@@ -0,0 +1,12 @@
+output "tls_public_certificate" {
+  value = tls_locally_signed_cert.tls_cert.cert_pem
+}
+
+output "tls_private_key" {
+  value     = tls_private_key.tls_key.private_key_pem
+  sensitive = true
+}
+
+output "certificate_chain" {
+  value = tls_self_signed_cert.ca_cert.cert_pem
+}

examples/dev/modules/tls/variables.tf

@@ -0,0 +1,6 @@
+variable "domain" {
+  type        = string
+  description = <<-EOT
+    The domain name to set as common name and dns sans.
+  EOT
+}

examples/dev/modules/tls/versions.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">= 1.5.0"
+  required_providers {
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0.5"
+    }
+  }
+}

examples/dev/outputs.tf

@@ -0,0 +1,18 @@
+output "kubeconfig" {
+  value       = module.rancher.kubeconfig
+  description = <<-EOT
+    The kubeconfig for the server.
+  EOT
+  sensitive   = true
+}
+output "address" {
+  value = module.rancher.address
+}
+output "admin_token" {
+  value     = module.rancher.admin_token
+  sensitive = true
+}
+output "admin_password" {
+  value     = module.rancher.admin_password
+  sensitive = true
+}