One RSS per namespace. Add role/binding for secrets (#375)

Change the ApplicationSet module so that each RunnerScaleSet is executed
in its own namespace.
Add a role to get read access to secrets and a rolebinding in each
namespace.
Also update the RSS name to reflect the OS/version of the runner
container image, rather than the containing VM.

Signed-off-by: Andrea Frittoli <andrea.frittoli@uk.ibm.com>

Author: afrittoli

arc/aws/391835788720/us-east-1/03_argocd/runner-set-lf-aws.tf

@@ -7,7 +7,6 @@ module "argocd-runner-scale-set" {
   token         = local.argocd_ready ? local.argocd_terraform_sa_token : null
   organization  = "lf"
   cluster       = "in-cluster"
-  namespace     = "lf-aws"
   provider_path = "argocd/aws/391835788720/us-east-1"
   git_revision  = var.git_revision
 }

arc/modules/argocd-runner-scale-set/arc_runner_scale_set.tf

@@ -24,16 +24,53 @@ locals {
   arc_private_key = data.aws_secretsmanager_secret_version.arc_secrets_private_key.secret_string
 }
 
+# ClusterRole with secret read permissions
+# This is never used cluster-wide, only in specific namespaces
+resource "kubernetes_cluster_role" "secret_reader" {
+  metadata {
+    name = "secret-reader"
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["secrets"]
+    verbs      = ["get", "list", "watch"]
+  }
+}
+
+/*
+ * A few resources need to be provisioned for each RunnerScaleSet
+ * i.e. for each folder under ${var.provider_path}/${var.cluster}
+ * 
+ * - A namespace
+ * - The GitHub secret
+ * - A RoleBinding the allows access to the secret 
+ */
+
+// Find folders directly under argocd/${var.provider_path}/${var.cluster}
+// and create a JSON map of results for next resources to loop on
+data "external" "runner_scale_sets" {
+  program = ["bash", "-c", <<-EOT
+    REPO_ROOT=$(git rev-parse --show-toplevel)
+    find "$REPO_ROOT/${var.provider_path}/${var.cluster}" -mindepth 1 -maxdepth 1 -type d -type d -exec basename {} \; | jq -R -s -c 'split("\n")[:-1] | map({key: ., value: .}) | from_entries'
+  EOT
+  ]
+}
+
 resource "kubernetes_namespace" "arc_runners" {
+  for_each = data.external.runner_scale_sets.result
+
   metadata {
-    name = var.namespace
+    name = "${var.organization}-${each.value}"
   }
 }
 
 resource "kubernetes_secret" "github_app" {
+  for_each = data.external.runner_scale_sets.result
+
   metadata {
     name      = "github-config"
-    namespace = var.namespace
+    namespace = "${var.organization}-${each.value}"
   }
 
   data = {
@@ -45,6 +82,27 @@ resource "kubernetes_secret" "github_app" {
   type = "Opaque"
 }
 
+resource "kubernetes_role_binding" "secret_access" {
+  for_each = data.external.runner_scale_sets.result
+
+  metadata {
+    name      = "secret-reader-binding"
+    namespace = "${var.organization}-${each.value}"
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = kubernetes_cluster_role.secret_reader.metadata[0].name
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = var.arc_controller_sa
+    namespace = var.arc_controller_sa_namespace
+  }
+}
+
 // This secret is required by ArgoCD to enable using an OCI repo
 // for helm charts. It's deployed once in the argocd namespace.
 resource "kubernetes_secret" "arc_runner_scale_set_oci_repo" {
@@ -123,18 +181,17 @@ resource "argocd_project" "arc_rss_project" {
  * - the helm chart
  * - the value files
  *
- * Value files can be found under argocd/cloud/tenant/region/cluster/namespace/<runner-scale-set-name>.
- * Each cloud/tenant/region/cluster/namespace combination has its ApplicationSet.
- * Various RunnerSets in the same namespace belong to the same ApplicationSet, with one application each.
+ * Value files can be found under argocd/cloud/tenant/region/cluster/<runner-scale-set-name>.
+ * Each cloud/tenant/region/cluster combination has its ApplicationSet.
+ * Each RunnerSets runs in its own namespace belong to the ApplicationSet, with one application each.
  *
  * The cluster folder is used to select the cluster name in ArgoCD
- * The namespace folder is used to define the target namespace in ArgoCD
  * Each resource in the folder is applied as a dedicated Application
  */
 resource "argocd_application_set" "arc_runner_scale_set" {
 
   metadata {
-    name      = "arc-rss-${var.cluster}-${var.namespace}"
+    name      = "arc-rss-${var.organization}-${var.cluster}"
     namespace = "argocd"
   }
 
@@ -147,7 +204,7 @@ resource "argocd_application_set" "arc_runner_scale_set" {
         repo_url = "https://github.com/pytorch/ci-infra"
         revision = var.git_revision
         directory {
-            path = "${var.provider_path}/${var.cluster}/${var.namespace}/*"
+            path = "${var.provider_path}/${var.cluster}/*"
         }
       }
     }
@@ -170,6 +227,11 @@ resource "argocd_application_set" "arc_runner_scale_set" {
             value_files = [
               "$values/{{.path.path}}/values.yaml"
             ]
+            values = <<-EOT
+              controllerServiceAccount:
+                name: ${var.arc_controller_sa}
+                namespace: ${var.arc_controller_sa_namespace}
+            EOT
           }
         }
         source {
@@ -180,7 +242,7 @@ resource "argocd_application_set" "arc_runner_scale_set" {
 
         destination {
           name      = "{{index .path.segments 4}}"
-          namespace = "{{index .path.segments 5}}"
+          namespace = "${var.organization}-{{index .path.segments 5}}"
         }
 
         sync_policy {

arc/modules/argocd-runner-scale-set/variables.tf

@@ -14,12 +14,6 @@ variable "organization" {
     default     = "lf"
 }
 
-variable "namespace" {
-    description = "The namespace for in the format <org>-<cloud>"
-    type        = string
-    default     = "lf-aws"
-}
-
 variable "cluster" {
     description = "The name of the cluster as defined in ArgoCD"
     type        = string
@@ -50,3 +44,15 @@ variable "aws_secret_key_name" {
     type        = string
     default     = "pytorch-arc-github-app-private-key"
 }
+
+variable "arc_controller_sa" {
+    description = "Service account used by the ARC controller"
+    type        = string
+    default     = "arc-gha-rs-controller"
+}
+
+variable "arc_controller_sa_namespace" {
+    description = "Namespace where the service account used by the ARC controller is defined"
+    type        = string
+    default     = "arc-system"
+}

…s/linux-20250911-amd64-2c-4g/values.yaml …luster/ubuntu-22-amd64-2c-4g/values.yamlargocd/aws/391835788720/us-east-1/in-cluster/lf-aws/linux-20250911-amd64-2c-4g/values.yaml renamed to argocd/aws/391835788720/us-east-1/in-cluster/ubuntu-22-amd64-2c-4g/values.yaml argocd/aws/391835788720/us-east-1/in-cluster/lf-aws/linux-20250911-amd64-2c-4g/values.yaml renamed to argocd/aws/391835788720/us-east-1/in-cluster/ubuntu-22-amd64-2c-4g/values.yaml

@@ -11,18 +11,14 @@ maxRunners: 2
 ## calculated as a sum of minRunners and the number of jobs assigned to the scale set.
 minRunners: 0
 
-runnerGroup: "linux-amd64-2c-4g"
+# Using the default runner group for now
+# runnerGroup: "ubuntu-amd64-2c-4g"
 
-runnerScaleSetName: "lf-aws-linux-20250911-amd64-2c-4g"
+runnerScaleSetName: "lf-aws-ubuntu-22-amd64-2c-4g"
 
 template:
   spec:
     containers:
       - name: runner
         image: ghcr.io/actions/actions-runner:latest
         command: ["/home/runner/run.sh"]
-
-# Service account of the controller
-controllerServiceAccount:
-  name: "arc-gha-rs-controller"
-  namespace: "arc-systems"