percona · nogueiraanderson · Jul 2, 2026
@@ -19,4 +19,4 @@ The repo is public. Two security-critical EKS inputs (the API endpoint allowlist
 - The reviewable surface (who is cluster admin, where the allowlist lives, its guards) is committed. Only the IP values stay out of the repo, in one shared, audited location instead of N operator laptops.
 - Resolved values still transit Terraform state (private S3 bucket), which is accepted.
 - A fresh fork or DR rebuild must seed its parameters before the first plan (`ParameterNotFound` otherwise). See [docs/runbooks/eks-api-access.md](../runbooks/eks-api-access.md). CI is unaffected: `tofu validate` reads no data sources.
-- The break-glass :22 allowlist (ADR 0032) joins as `/<cluster>/allowlist/master-ssh` once its CIDR owners are confirmed. Until then it stays a module default plus per-master overrides.
+- The break-glass :22 allowlist (ADR 0032) joined as `/<cluster>/allowlist/master-ssh`: the module default is now empty (no committed IPs) and every `master-*.tf` passes the SSM-resolved list. The seed value is the union of the previously committed 6-CIDR baseline and the psmdb/pmm extras, so nobody loses access at cutover.
@@ -48,6 +48,19 @@ aws ssm put-parameter --region us-east-1 \
   --tags Key=iit-billing-tag,Value=<cluster> Key=repo,Value=github.com/Percona/percona-cd-platform
 ```
 
-## Planned second tenant
+## Second tenant: master-ssh
 
-`/percona-ci-platform/allowlist/master-ssh`, the [ADR 0032](../adr/0032-ec2-instance-connect-over-ssm-for-operator-ssh.md) break-glass :22 list, joins once its CIDR owners are confirmed. That gate protects master SSH, not the EKS API. See `master-shell-access.md`.
+`/percona-ci-platform/allowlist/master-ssh` is the [ADR 0032](../adr/0032-ec2-instance-connect-over-ssm-for-operator-ssh.md) break-glass :22 list for the EC2 Jenkins masters. That gate protects master SSH, not the EKS API. See `master-shell-access.md`.
+
+Same mechanics as eks-api: amend with `just allowlist-set master-ssh "<full,list>"`, then `just tf-plan` and `just tf-apply`. The module default is empty, so a master with no parameter value gets NO :22 ingress, and the plan fails closed if the parameter is missing, empty, or contains `0.0.0.0/0`.
+
+Bootstrap (must exist before the first plan that includes this change):
+
+```bash
+aws ssm put-parameter --region us-east-1 \
+  --name /<cluster>/allowlist/master-ssh --type StringList \
+  --value "<operator-egress-cidrs,comma-separated>" \
+  --tags Key=iit-billing-tag,Value=<cluster> Key=repo,Value=github.com/Percona/percona-cd-platform
+```
+
+The cutover seed is the union of the previously committed module default and the psmdb/pmm per-master extras (8 CIDRs). Total reachability is preserved, and the two per-master extras gain :22 on every master where they previously reached one master each; the six baseline CIDRs are unchanged.
@@ -197,37 +197,41 @@ tf-plan-masters: _require-aws-profile
     tofu -chdir=terraform plan "${targets[@]}"
 
 # ---------- operator allowlists (SSM-backed) ----------
-# Source of truth for the EKS API allowlist (and future operator allowlists).
+# Source of truth for the operator allowlists (eks-api, master-ssh).
 # Values live only in SSM Parameter Store (CloudTrail-audited), never in git.
 # A put changes nothing live until `just tf-plan && just tf-apply`.
 # Ops: docs/runbooks/eks-api-access.md. Decision: docs/adr/0036.
 allowlist-show: _require-aws-profile
     #!/usr/bin/env bash
     set -euo pipefail
-    for p in eks-api; do
+    for p in eks-api master-ssh; do
       echo "== /{{cluster}}/allowlist/${p}"
       aws ssm get-parameter --name "/{{cluster}}/allowlist/${p}" \
         --region "${AWS_REGION:-us-east-1}" \
         --query Parameter.Value --output text | tr ',' '\n'
     done
 
 # Usage: just allowlist-set eks-api "198.51.100.5/32,203.0.113.0/24"
-# The value is the FULL comma-separated list, not a delta. The master-ssh
-# allowlist joins the case list when its parameter lands (docs/adr/0036).
+# The value is the FULL comma-separated list, not a delta. Operator input is
+# bound ONCE via quote() into shell variables before any other use; embedding
+# quote() inside a double-quoted string would let $(...) re-evaluate, so only
+# the two assignments below may interpolate the operands (docs/adr/0036).
 allowlist-set name cidrs: _require-aws-profile
     #!/usr/bin/env bash
     set -euo pipefail
-    case "{{name}}" in eks-api) ;; *) echo "unknown allowlist '{{name}}' (expected: eks-api)" >&2; exit 2 ;; esac
-    param="/{{cluster}}/allowlist/{{name}}"
+    name={{quote(name)}}
+    cidrs={{quote(cidrs)}}
+    case "$name" in eks-api|master-ssh) ;; *) printf 'unknown allowlist %s (expected: eks-api, master-ssh)\n' "$name" >&2; exit 2 ;; esac
+    param="/{{cluster}}/allowlist/$name"
     echo "current:"
-    aws ssm get-parameter --name "${param}" --region "${AWS_REGION:-us-east-1}" \
+    aws ssm get-parameter --name "$param" --region "${AWS_REGION:-us-east-1}" \
       --query Parameter.Value --output text | tr ',' '\n'
     echo "new:"
-    tr ',' '\n' <<< "{{cidrs}}"
-    read -r -p "overwrite ${param}? [y/N] " a
-    [ "${a}" = "y" ]
-    aws ssm put-parameter --name "${param}" --type StringList \
-      --value "{{cidrs}}" --overwrite --region "${AWS_REGION:-us-east-1}"
+    tr ',' '\n' <<< "$cidrs"
+    read -r -p "overwrite $param? [y/N] " a
+    [ "$a" = "y" ]
+    aws ssm put-parameter --name "$param" --type StringList \
+      --value "$cidrs" --overwrite --region "${AWS_REGION:-us-east-1}"
     echo "saved. Apply it: just tf-plan && just tf-apply"
 
 # ---------- gitops / yaml ----------

@@ -10,10 +10,10 @@
 # feed. That is accepted. Ops: docs/runbooks/eks-api-access.md.
 # Decision: docs/adr/0036-operator-allowlists-in-ssm-and-dynamic-sso-access-entry.md.
 #
-# Parameters are StringList in the cluster region. Today only the EKS API
-# allowlist lives here. The master break-glass :22 allowlist
-# (/<cluster>/allowlist/master-ssh) is the planned second tenant once its
-# CIDR owners are confirmed (docs/adr/0032 keeps that path break-glass only).
+# Parameters are StringList in the cluster region. Two tenants: the EKS API
+# allowlist and the master break-glass :22 allowlist
+# (/<cluster>/allowlist/master-ssh, consumed by every master-*.tf; docs/adr/0032
+# keeps that path break-glass only).
 #
 # The postcondition FAILS the plan (check blocks only warn in OpenTofu 1.11),
 # keeping the hardening invariant fail-closed: the parameter must exist,
@@ -34,11 +34,29 @@ data "aws_ssm_parameter" "eks_api_allowlist" {
   }
 }
 
+data "aws_ssm_parameter" "master_ssh_allowlist" {
+  name = "/${local.cluster_name}/allowlist/master-ssh"
+
+  lifecycle {
+    postcondition {
+      condition = alltrue([
+        length(compact([for c in split(",", self.value) : trimspace(c)])) > 0,
+        !contains([for c in split(",", self.value) : trimspace(c)], "0.0.0.0/0"),
+        alltrue([for c in compact([for x in split(",", self.value) : trimspace(x)]) : can(cidrhost(c, 0))]),
+      ])
+      error_message = "The master-ssh allowlist parameter must hold at least one valid CIDR and never 0.0.0.0/0 (docs/adr/0032, 0036). Fix with `just allowlist-set master-ssh <full,list>`."
+    }
+  }
+}
+
 locals {
   # aws_ssm_parameter values are sensitive-by-default; unwrapped like the
   # amis.tf consumers so the EKS endpoint plan diff stays readable.
   eks_api_allowed_cidrs = distinct(concat(
     compact([for c in split(",", nonsensitive(data.aws_ssm_parameter.eks_api_allowlist.value)) : trimspace(c)]),
     var.api_public_access_cidrs,
   ))
+
+  # Break-glass :22 fleet allowlist, passed to every jenkins-master module.
+  master_ssh_allowed_cidrs = compact([for c in split(",", nonsensitive(data.aws_ssm_parameter.master_ssh_allowlist.value)) : trimspace(c)])
 }
@@ -86,6 +86,12 @@ resource "aws_s3_bucket" "lgtm" {
   # below (per-component `expiration.days`), which is the right tool for
   # log/metric/trace chunks (constant churn, short shelf life).
 
+  lifecycle {
+    # Holds up to 395-day metrics: a for_each key rename must never
+    # destroy-and-recreate the bucket (matches ps3-home-archive and EFS).
+    prevent_destroy = true
+  }
+
   tags = merge(local.tags, {
     Name      = "${local.cluster_name}-${each.value.suffix}"
     component = each.key

@@ -33,6 +33,7 @@ module "cloud" {
   vpc_cidr                = "10.177.0.0/22"
   ami_id                  = nonsensitive(data.aws_ssm_parameter.al2023_minimal_euw1.value) # latest AL2023 minimal (amis.tf)
   master_profile          = "eks_observability"
+  ssh_allowed_cidrs       = local.master_ssh_allowed_cidrs
   jenkins_package_version = "2.541.3"
   # cloud workers have no S3 build cache: the CFN-era default named
   # cloud-build-cache, a bucket this account never created and that exists

@@ -95,18 +95,7 @@ module "pmm" {
     "vadim.yalovets",
   ]
 
-  # The 6-CIDR fleet baseline (module default) plus the pmm QA engineer's
-  # address. This is the "pmm differs" delta the module variable description
-  # references; codified so an apply does not strip the live rule.
-  ssh_allowed_cidrs = [
-    "46.149.86.84/32",
-    "54.214.47.252/32",
-    "54.214.47.254/32",
-    "154.192.11.141/32",
-    "176.37.55.60/32",
-    "188.163.20.103/32",
-    "213.159.239.48/32",
-  ]
+  ssh_allowed_cidrs = local.master_ssh_allowed_cidrs
 
   # Declarative init.groovy.d delivered via the module-created S3 bucket
   # (jenkins-pmm-amzn2-init-config). Content moved byte-identically off the

@@ -24,6 +24,7 @@ module "ps57" {
   vpc_cidr                = "10.157.0.0/22"
   ami_id                  = nonsensitive(data.aws_ssm_parameter.al2023_minimal_euc1.value) # latest AL2023 minimal (amis.tf)
   master_profile          = "eks_observability"
+  ssh_allowed_cidrs       = local.master_ssh_allowed_cidrs
   jenkins_package_version = "2.541.3"
   cache_bucket_name       = "ps-build-cache"
 

@@ -27,6 +27,7 @@ module "ps80" {
   vpc_cidr                = "10.155.0.0/22"
   ami_id                  = nonsensitive(data.aws_ssm_parameter.al2023_minimal_usw2.value) # latest AL2023 minimal (amis.tf)
   master_profile          = "eks_observability"
+  ssh_allowed_cidrs       = local.master_ssh_allowed_cidrs
   jenkins_package_version = "2.541.3"
   cache_bucket_name       = "ps-build-cache"
   ebs_size                = 300 # grown live from 200; EBS cannot shrink, keep >= live size

@@ -30,17 +30,7 @@ module "psmdb" {
   master_profile          = "eks_observability"
   jenkins_package_version = "2.541.3"
 
-  # Baseline 6-CIDR fleet allowlist plus an operator's :22 source CIDR;
-  # codified so an apply does not strip the live rule (matches the pmm pattern).
-  ssh_allowed_cidrs = [
-    "46.149.86.84/32",
-    "54.214.47.252/32",
-    "54.214.47.254/32",
-    "176.37.55.60/32",
-    "188.163.20.103/32",
-    "213.159.239.48/32",
-    "86.127.231.233/32",
-  ]
+  ssh_allowed_cidrs = local.master_ssh_allowed_cidrs
 
   # Retained CFN data volume vol-090299a14ad3da940 is 300 GiB gp2 in
   # us-west-2b. ebs_type must be gp2 (not the module default gp3) so the

@@ -33,6 +33,7 @@ module "pxb" {
   vpc_cidr                = "10.179.0.0/22"
   ami_id                  = nonsensitive(data.aws_ssm_parameter.al2023_minimal_usw2.value)
   master_profile          = "eks_observability"
+  ssh_allowed_cidrs       = local.master_ssh_allowed_cidrs
   jenkins_package_version = "2.541.3"
   # pxb workers do not use an S3 build cache (null disables the worker S3 IAM
   # policy), matching the legacy pxb worker role.

@@ -26,6 +26,7 @@ module "pxc" {
   vpc_cidr                = "10.156.0.0/22"
   ami_id                  = nonsensitive(data.aws_ssm_parameter.al2023_minimal_usw1.value) # latest AL2023 minimal (amis.tf)
   master_profile          = "eks_observability"
+  ssh_allowed_cidrs       = local.master_ssh_allowed_cidrs
   jenkins_package_version = "2.541.3"
   cache_bucket_name       = "pxc-build-cache"
 

@@ -25,6 +25,7 @@ module "rel" {
   vpc_cidr                = "10.199.0.0/22"
   ami_id                  = nonsensitive(data.aws_ssm_parameter.al2023_minimal_euw1.value) # latest AL2023 minimal (amis.tf)
   master_profile          = "eks_observability"
+  ssh_allowed_cidrs       = local.master_ssh_allowed_cidrs
   jenkins_package_version = "2.541.3"
   # rel workers have no S3 build cache: the CFN-era default named a
   # rel-build-cache bucket that was never created. null drops the dead

@@ -134,16 +134,9 @@ variable "extra_http_ingress" {
 }
 
 variable "ssh_allowed_cidrs" {
-  description = "Source CIDRs allowed on :22. Default is the 6-CIDR fleet baseline; pmm differs."
+  description = "Source CIDRs allowed on :22. No committed baseline: the fleet allowlist lives in SSM (/<cluster>/allowlist/master-ssh, ADR 0036) and every master-*.tf passes the resolved list. Empty means no :22 ingress."
   type        = list(string)
-  default = [
-    "46.149.86.84/32",
-    "54.214.47.252/32",
-    "54.214.47.254/32",
-    "176.37.55.60/32",
-    "188.163.20.103/32",
-    "213.159.239.48/32",
-  ]
+  default     = []
 }
 
 variable "create_eip" {

@@ -298,6 +298,9 @@ import {
   id       = each.value.id
 }
 
+# Workers SSH to the test VMs over their public IPs from arbitrary egress
+# points (AWS workers, Hetzner, ARM fleet), so 22/tcp and ICMP echo stay
+# world-open by design; mirrors create.yml exactly (see header above).
 #trivy:ignore:AVD-AWS-0107
 resource "aws_security_group" "molecule_tests_pxc_runtime" {
   for_each = local.molecule_tests_pxc_runtime_sgs