Merge branch 'main' into feature/73-bundle-provisioner-into-management

Author: HiranAdikari

modules/management/namespace-credential-provisioner/scripts/reconcile.sh

@@ -6,12 +6,26 @@
 # 1. Namespace watch (Harvester)
 #    Watches tenant namespaces on the Harvester cluster. For each new namespace
 #    that belongs to a Rancher project it creates:
+#
+#    Cloud-provider credentials (for RKE2 Harvester cloud provider):
 #      - ServiceAccount harvester-cloud-provider-<ns>
 #      - RoleBinding to harvesterhci.io:cloudprovider in the tenant namespace
 #      - Optional RoleBinding to view in the project's network namespace
 #      - Long-lived SA token secret
+#
+#    Consumer VM-access credentials (for tenant teams provisioning VMs/clusters):
+#      - ServiceAccount harvester-vm-access-<ns>
+#      - RoleBinding to harvesterhci.io:edit in the tenant namespace
+#      - RoleBinding to edit (k8s built-in) in the tenant namespace
+#      - RoleBinding to view in harvester-public (for shared OS images)
+#      - Long-lived SA token secret
+#      - Secret "harvester-vm-kubeconfig" in the tenant namespace containing a
+#        namespace-scoped kubeconfig. The platform team retrieves this once at
+#        onboarding and hands it to the tenant team.
+#
 #    On namespace deletion it deletes any harvesterconfig-* secrets on Rancher
-#    whose kubeconfig was built from that namespace's SA token.
+#    whose kubeconfig was built from that namespace's SA token, and cleans up
+#    the harvester-public RoleBinding for the VM-access SA.
 #
 # 2. Cluster watch (Rancher)
 #    Watches clusters.provisioning.cattle.io on the Rancher cluster. For each
@@ -22,9 +36,6 @@
 #        v2prov-secret-authorized-for-cluster already set at creation time
 #    On cluster deletion it removes harvesterconfig-<cluster-name>.
 #
-# Consumers (tenant teams) only need the rancher2 provider. No Harvester or
-# Rancher kubeconfig required on their side.
-#
 # Environment variables (injected by the Deployment):
 #   HARVESTER_API_SERVER  — Harvester Kubernetes API server URL
 #   RANCHER_KUBECONFIG    — Path to kubeconfig for Rancher's local cluster
@@ -131,6 +142,114 @@ $(echo "$kubeconfig" | sed 's/^/    /')
 EOF
 }
 
+# Build a namespace-scoped VM-access kubeconfig for tenant teams and write it
+# as the well-known "harvester-vm-kubeconfig" Secret in the tenant namespace.
+# Consumers retrieve this secret once at onboarding to provision VMs and RKE2
+# clusters using the workloads/vm and workloads/k8s-cluster OCD modules.
+# Args: ns
+write_vm_kubeconfig() {
+  local ns="$1"
+  local sa_name="harvester-vm-access-${ns}"
+  local secret_name="harvester-vm-kubeconfig"
+
+  # ServiceAccount in tenant namespace.
+  kubectl create serviceaccount "$sa_name" -n "$ns" \
+    --dry-run=client -o yaml | kubectl apply -f -
+
+  # RoleBinding — Harvester VM lifecycle (VMs, keypairs, images, NADs, backups).
+  kubectl create rolebinding "${sa_name}" \
+    --clusterrole=harvesterhci.io:edit \
+    --serviceaccount="${ns}:${sa_name}" \
+    -n "$ns" --dry-run=client -o yaml | kubectl apply -f -
+
+  # RoleBinding — Kubernetes resource edit (Secrets, PVCs, ConfigMaps).
+  kubectl create rolebinding "${sa_name}-k8s-edit" \
+    --clusterrole=edit \
+    --serviceaccount="${ns}:${sa_name}" \
+    -n "$ns" --dry-run=client -o yaml | kubectl apply -f -
+
+  # RoleBinding — read shared OS images in default namespace.
+  kubectl create rolebinding "${ns}-${sa_name}-default-view" \
+    --clusterrole=view \
+    --serviceaccount="${ns}:${sa_name}" \
+    -n "default" --dry-run=client -o yaml | kubectl apply -f -
+
+  # RoleBinding — read shared OS images in harvester-public.
+  kubectl create rolebinding "${ns}-${sa_name}-public-view" \
+    --clusterrole=view \
+    --serviceaccount="${ns}:${sa_name}" \
+    -n "harvester-public" --dry-run=client -o yaml | kubectl apply -f -
+
+  # Long-lived token secret.
+  kubectl apply -f - <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${sa_name}-token
+  namespace: ${ns}
+  annotations:
+    kubernetes.io/service-account.name: ${sa_name}
+type: kubernetes.io/service-account-token
+EOF
+
+  # Wait for the token to be populated by the token controller.
+  local token=""
+  for _ in $(seq 1 20); do
+    token=$(kubectl get secret "${sa_name}-token" -n "$ns" \
+      -o jsonpath='{.data.token}' 2>/dev/null || true)
+    [[ -n "$token" ]] && break
+    sleep 1
+  done
+  if [[ -z "$token" ]]; then
+    log "  ERROR: VM access token not populated for ${sa_name} in ${ns}"
+    return 1
+  fi
+
+  local token_decoded ca_cert_b64
+  token_decoded=$(echo "$token" | base64 -d)
+  ca_cert_b64=$(kubectl get configmap kube-root-ca.crt -n kube-system \
+    -o jsonpath='{.data.ca\.crt}' | base64 | tr -d '\n')
+
+  local kubeconfig
+  kubeconfig=$(cat <<EOF
+apiVersion: v1
+kind: Config
+clusters:
+- name: harvester
+  cluster:
+    certificate-authority-data: ${ca_cert_b64}
+    server: ${HARVESTER_API_SERVER}
+users:
+- name: ${ns}
+  user:
+    token: ${token_decoded}
+contexts:
+- name: ${ns}@harvester
+  context:
+    cluster: harvester
+    namespace: ${ns}
+    user: ${ns}
+current-context: ${ns}@harvester
+EOF
+)
+
+  kubectl apply -f - <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${secret_name}
+  namespace: ${ns}
+  annotations:
+    platform.wso2.com/vm-access-sa: "${sa_name}"
+type: Opaque
+stringData:
+  kubeconfig: |
+$(echo "$kubeconfig" | sed 's/^/    /')
+EOF
+
+  log "  [ns] VM access kubeconfig ready: ${secret_name} in ${ns}"
+}
+
 # ── Namespace watch handlers ───────────────────────────────────────────────────
 
 on_added_namespace() {
@@ -179,6 +298,11 @@ type: kubernetes.io/service-account-token
 EOF
 
   log "  [ns] SA ready: ${sa_name} in ${ns}"
+
+  # Consumer VM-access kubeconfig — separate SA with broader permissions.
+  # Explicit return propagates failure to the caller so the namespace is NOT
+  # marked processed; the watch loop will retry on the next event.
+  write_vm_kubeconfig "$ns" || return 1
 }
 
 on_deleted_namespace() {
@@ -216,6 +340,14 @@ on_deleted_namespace() {
         kubectl delete rolebinding "$rb_name_found" -n "$rb_ns" 2>/dev/null \
           && log "  [ns] deleted rolebinding ${rb_name_found} from ${rb_ns}"
       done || true
+
+  # Delete the VM-access SA's cross-namespace RoleBindings.
+  # (Resources inside the deleted namespace are cleaned up by Kubernetes.)
+  local vm_sa_name="harvester-vm-access-${ns}"
+  kubectl delete rolebinding "${ns}-${vm_sa_name}-default-view" -n "default" \
+    2>/dev/null && log "  [ns] deleted default RoleBinding for ${vm_sa_name}" || true
+  kubectl delete rolebinding "${ns}-${vm_sa_name}-public-view" -n "harvester-public" \
+    2>/dev/null && log "  [ns] deleted harvester-public RoleBinding for ${vm_sa_name}" || true
 }
 
 # ── Cluster watch handlers ─────────────────────────────────────────────────────
@@ -408,9 +540,26 @@ kubectl get namespaces -o json | jq -r '
   [[ -z "$project_id" ]] && continue
   is_system_namespace "$ns" && continue
   [[ "$role" == "network-namespace" ]] && continue
-  log "INIT namespace: ${ns} (project: ${project_id})"
-  if on_added_namespace "$ns" "$project_id"; then
-    echo "$ns" >> "$PROCESSED_NS_FILE"
+  sa_name="harvester-cloud-provider-${ns}"
+  if kubectl get secret "${sa_name}-token" -n "$ns" &>/dev/null; then
+    # Cloud-provider credentials already exist. Check for the VM-access kubeconfig
+    # separately — may be absent on pods that ran before this feature was added.
+    if kubectl get secret "harvester-vm-kubeconfig" -n "$ns" &>/dev/null; then
+      log "INIT namespace: ${ns} — already provisioned, skipping"
+      echo "$ns" >> "$PROCESSED_NS_FILE"
+    else
+      log "INIT namespace: ${ns} — backfilling VM access kubeconfig"
+      if write_vm_kubeconfig "$ns"; then
+        echo "$ns" >> "$PROCESSED_NS_FILE"
+      else
+        log "  WARN: VM access kubeconfig backfill failed for ${ns} — will retry on next watch event"
+      fi
+    fi
+  else
+    log "INIT namespace: ${ns} (project: ${project_id})"
+    if on_added_namespace "$ns" "$project_id"; then
+      echo "$ns" >> "$PROCESSED_NS_FILE"
+    fi
   fi
 done
 

modules/management/tenant-space/main.tf

@@ -13,8 +13,11 @@ locals {
   # index. Only relevant when vyos_endpoint is set and exactly one VLAN is given.
   # Auto-routed environments (physical switch / DigiOps-issued VLANs) use multiple
   # VLANs with route_mode=auto; no explicit subnets needed.
-  use_vyos       = var.vlan_id != null && length(var.vlan_id) > 0 && var.vyos_endpoint != null
-  tenant_subnet  = local.use_vyos ? cidrsubnet("10.0.0.0/8", 15, var.vlan_id[0] - 1000) : null
+  use_vyos = var.vlan_id != null && length(var.vlan_id) > 0 && var.vyos_endpoint != null
+  # vlan_id[0] must be >= 1000 when VyOS is used — enforced by the precondition below.
+  # max(..., 0) prevents a negative cidrsubnet index from causing a plan-time panic
+  # before the precondition fires.
+  tenant_subnet  = local.use_vyos ? cidrsubnet("10.0.0.0/8", 15, max(var.vlan_id[0] - 1000, 0)) : null
   tenant_gateway = local.use_vyos ? cidrhost(local.tenant_subnet, 1) : null
 }
 
@@ -59,6 +62,14 @@ resource "rancher2_project" "this" {
       condition     = !local.use_vyos || length(var.vlan_id) == 1
       error_message = "VyOS path requires exactly one VLAN ID. Set vyos_endpoint = null for multi-VLAN auto-route configurations."
     }
+    precondition {
+      condition     = !local.use_vyos || var.vlan_id[0] >= 1000
+      error_message = "VyOS IPAM uses VLAN IDs >= 1000 (index = vlan_id - 1000). Set vyos_endpoint = null for VLANs below 1000."
+    }
+    precondition {
+      condition     = var.vlan_id == null || length(var.vlan_id) == 0 || local.create_net_ns
+      error_message = "vlan_id requires the network namespace to exist. This should never happen since create_net_ns is always true when vlan_id is set — if you see this, do not set create_network_namespace = false alongside vlan_id."
+    }
   }
 }
 
@@ -138,6 +149,49 @@ module "vyos_tenant" {
   vyos_api_key  = var.vyos_api_key
 }
 
+# ── Consumer VM access kubeconfig (read from provisioner-created secret) ───────
+# The namespace-credential-provisioner automatically creates a "harvester-vm-kubeconfig"
+# Secret in each tenant namespace containing a namespace-scoped Harvester kubeconfig.
+# This data source surfaces it as a Terraform output so the platform team can
+# retrieve it once at onboarding and hand it to the tenant team:
+#
+#   terraform output -raw <tenant>_vm_kubeconfig > <team>.harvester.kubeconfig.secret
+#
+# Requires the kubernetes.harvester provider to be passed in (set expose_vm_kubeconfig = true
+# and configure kubernetes.harvester in the caller's provider block).
+#
+# IMPORTANT — two-pass apply required:
+# This data source races with the out-of-band reconciler. On a fresh namespace the
+# provisioner may not have created the secret yet when Terraform runs the data read.
+# Apply in two steps:
+#   1. terraform apply                  # creates namespaces; provisioner reconciles
+#   2. terraform apply                  # reads the now-present secret
+# Alternatively, run `terraform apply -target=rancher2_namespace.this` first, wait
+# for the provisioner, then run the full apply.
+
+locals {
+  # Primary workload namespace for the VM kubeconfig.
+  # Uses var.vm_access_namespace when explicitly set; falls back to the first
+  # resolved namespace (or project_name when no namespaces are configured).
+  vm_access_ns = var.vm_access_namespace != null ? var.vm_access_namespace : (
+    length(local.namespaces) > 0 ? local.namespaces[0] : var.project_name
+  )
+}
+
+data "kubernetes_secret_v1" "vm_access_kubeconfig" {
+  count    = var.expose_vm_kubeconfig ? 1 : 0
+  provider = kubernetes.harvester
+  metadata {
+    name      = "harvester-vm-kubeconfig"
+    namespace = local.vm_access_ns
+  }
+  depends_on = [rancher2_namespace.this]
+}
+
+locals {
+  vm_access_kubeconfig = var.expose_vm_kubeconfig ? data.kubernetes_secret_v1.vm_access_kubeconfig[0].data["kubeconfig"] : null
+}
+
 # ── One binding per (group, role) pair. ───────────────────────────────────────
 resource "rancher2_project_role_template_binding" "this" {
   for_each = {

modules/management/tenant-space/outputs.tf

@@ -37,3 +37,9 @@ output "gateway_ip" {
   value       = local.tenant_gateway
   description = "VyOS gateway IP for this tenant (first host in subnet_cidr). Non-null only when vlan_id and vyos_endpoint are both set."
 }
+
+output "vm_access_kubeconfig" {
+  value       = local.vm_access_kubeconfig
+  sensitive   = true
+  description = "Namespace-scoped Harvester kubeconfig for the tenant team. Non-null when expose_vm_kubeconfig = true and the namespace-credential-provisioner has already created the secret. Hand to the tenant team once at onboarding. See examples/consumer-workloads in wso2-datacenter-project for usage."
+}

modules/management/tenant-space/variables.tf

@@ -170,6 +170,22 @@ variable "group_role_bindings" {
   default     = []
 }
 
+variable "expose_vm_kubeconfig" {
+  type        = bool
+  description = "When true, reads the 'harvester-vm-kubeconfig' Secret created by the namespace-credential-provisioner and exposes it via the vm_access_kubeconfig output. Requires the kubernetes.harvester provider alias to be configured in the caller. The provisioner must have run before apply."
+  default     = false
+}
+
+variable "vm_access_namespace" {
+  type        = string
+  description = "Namespace from which to read the 'harvester-vm-kubeconfig' Secret when expose_vm_kubeconfig = true. Defaults to the first resolved namespace (or project_name when no namespaces are configured). Set explicitly when the tenant has multiple namespaces and the kubeconfig should target a specific one."
+  default     = null
+  validation {
+    condition     = var.vm_access_namespace == null ? true : trimspace(var.vm_access_namespace) != ""
+    error_message = "vm_access_namespace must be null or a non-empty namespace name."
+  }
+}
+
 variable "vyos_endpoint" {
   type        = string
   description = "VyOS HTTPS API endpoint (e.g. 'https://172.22.100.50'). Required when vlan_id is set."

modules/management/tenant-space/versions.tf

@@ -9,5 +9,10 @@ terraform {
       source  = "harvester/harvester"
       version = "~> 1.7"
     }
+    kubernetes = {
+      source                = "hashicorp/kubernetes"
+      version               = "~> 2.35"
+      configuration_aliases = [kubernetes.harvester]
+    }
   }
 }

modules/monitoring/examples/basic/main.tf

@@ -23,17 +23,15 @@ terraform {
 }
 
 provider "kubernetes" {
-  config_path    = var.kubeconfig_path
-  config_context = var.kubeconfig_context
+  config_path = var.kubeconfig_path
 }
 
 module "monitoring" {
   source = "../../"
 
   # Identifiers
-  environment        = var.environment
-  kubeconfig_path    = var.kubeconfig_path
-  kubeconfig_context = var.kubeconfig_context
+  environment     = var.environment
+  kubeconfig_path = var.kubeconfig_path
 
   # Notification
   google_chat_webhook_url = var.google_chat_webhook_url
@@ -71,11 +69,6 @@ variable "kubeconfig_path" {
   default = "~/.kube/harvester-lk.yaml"
 }
 
-variable "kubeconfig_context" {
-  type    = string
-  default = "local"
-}
-
 variable "google_chat_webhook_url" {
   type      = string
   sensitive = true

modules/monitoring/main.tf

@@ -304,14 +304,11 @@ resource "null_resource" "alertmanager_base_config" {
     command = <<-BASH
       set -e
       kubectl create secret generic alertmanager-rancher-monitoring-alertmanager \
-        --context '${var.kubeconfig_context}' \
         -n '${local.ns}' \
         --from-file=alertmanager.yaml=<(base64 -d <<< "$AM_CONFIG_B64") \
         --from-file=rancher_defaults.tmpl=<(base64 -d <<< "$AM_TMPL_B64") \
         --dry-run=client -o yaml \
-      | kubectl apply \
-          --context '${var.kubeconfig_context}' \
-          -f -
+      | kubectl apply -f -
     BASH
   }
 }
@@ -337,14 +334,11 @@ resource "null_resource" "calert_config" {
     command = <<-BASH
       set -e
       kubectl create secret generic calert-config \
-        --context '${var.kubeconfig_context}' \
         -n '${local.ns}' \
         --from-file=config.toml=<(base64 -d <<< "$CONFIG_B64") \
         --from-file=message.tmpl=<(base64 -d <<< "$TMPL_B64") \
         --dry-run=client -o yaml \
-      | kubectl apply \
-          --context '${var.kubeconfig_context}' \
-          -f -
+      | kubectl apply -f -
     BASH
   }
 }