Add opstool environment for SRE tooling AKS cluster

Author: trevorwilliams2025

.github/workflows/aro-hcp-cd.yml

@@ -37,6 +37,7 @@ on:
     - 'image-sync/**/'
     - 'tooling/templatize/**'
     - 'config/*'
+    - 'topology-opstool.yaml'
     types:
     - closed
 concurrency:
@@ -157,3 +158,15 @@ jobs:
     with:
       deploy_env: cspr
       deploy_cs_pr_check_deps: true
+  deploy_opstool_environment_infra:
+    name: 'Deploy opstool infrastructure'
+    if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
+    needs:
+    - deploy_global_rg
+    permissions:
+      id-token: 'write'
+      contents: 'read'
+    secrets: inherit
+    uses: ./.github/workflows/environment-infra-cd.yml
+    with:
+      deploy_env: opstool

.github/workflows/environment-infra-cd.yml

@@ -119,6 +119,9 @@ jobs:
         az config set bicep.use_binary_from_path=false
         az bicep install
         cd dev-infrastructure/
+        if [ "${{ inputs.deploy_env }}" = "opstool" ]; then
+          export TOPOLOGY_CONFIG=topology-opstool.yaml
+        fi
         PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} make svc
     - name: 'Az CLI login again'
       uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0

Makefile

@@ -356,10 +356,12 @@ PERSIST ?= "false"
 TIMING_OUTPUT ?= timing/steps.yaml
 ENTRYPOINT_JUNIT_OUTPUT ?= _artifacts/junit_entrypoint.xml
 
+TOPOLOGY_CONFIG ?= topology.yaml
+
 local-run: $(TEMPLATIZE)
 	$(TEMPLATIZE) entrypoint run --config-file "${CONFIG_FILE}" \
 								     --config-file-override "${OVERRIDE_CONFIG_FILE}" \
-	                                 --topology-config topology.yaml \
+	                                 --topology-config $(TOPOLOGY_CONFIG) \
 	                                 --dev-settings-file tooling/templatize/settings.yaml \
 	                                 --dev-environment $(DEPLOY_ENV) \
 	                                 $(WHAT) $(EXTRA_ARGS) \

config/config.yaml

@@ -1180,6 +1180,42 @@ clouds:
           svc:
             subscription:
               key: ARO HCP nightly service (EA Subscription)
+      opstool:
+        # this is an environment for SRE tooling AKS cluster
+        defaults:
+          regionRG: "{{ .ctx.region }}-shared-resources"
+          serviceKeyVault:
+            name: "arohcp{{ .ctx.environment }}-svc-{{ .ctx.regionShort }}" # [globally-unique]
+            rg: "{{ .ctx.region }}-shared-resources"
+            region: "{{ .ctx.region }}"
+            softDelete: false
+          monitoring:
+            svcWorkspaceName: 'aro-hcp-{{ .ctx.environment }}-svc-{{ .ctx.regionShort }}'
+          # DNS
+          dns:
+            regionalSubdomain: '{{ .ctx.regionShort }}-{{ .ctx.environment }}'
+          frontend:
+            cosmosDB:
+              deploy: false
+          clustersService:
+            postgres:
+              deploy: false
+          maestro:
+            eventGrid:
+              name: "arohcp-{{ .ctx.environment }}-maestro-{{ .ctx.regionShort }}"
+              maxClientSessionsPerAuthName: 6
+              private: false
+            certIssuer: OneCertV2-PrivateCA
+            postgres:
+              deploy: false
+          adminApi:
+            managedIdentityName: admin-api
+            k8s:
+              namespace: aro-hcp
+              serviceAccountName: admin-api
+            cert:
+              name: admin-api-cert-{{ .ctx.environment }}-{{ .ctx.regionShort }}
+              issuer: Self
       pers:
         # Regional overrides
         regions:

config/dev.digests.yaml

@@ -10,6 +10,9 @@ clouds:
       ntly:
         regions:
           uksouth: 97fc3fe690f8c37aa2435ad84e840aaabd7b3728346d9468961f29db32bee887
+      opstool:
+        regions:
+          uksouth: aa554e097384d03595b471ca2161acadcdb01362b5c0bbd7215a40146f99ab74
       perf:
         regions:
           westus3: dbcc359647553e4925da830e947ce33dc2a6d20dd4e86938448433d6ca467e5c

dev-infrastructure/templates/svc-cluster.bicep

@@ -627,7 +627,7 @@ module rpCosmosDb '../modules/rp-cosmos.bicep' = if (deployFrontendCosmos) {
   }
 }
 
-module rpCosmosdbPrivateEndpoint '../modules/private-endpoint.bicep' = {
+module rpCosmosdbPrivateEndpoint '../modules/private-endpoint.bicep' = if (deployFrontendCosmos) {
   name: 'rp-pe-${uniqueString(deployment().name)}'
   params: {
     location: location
@@ -983,17 +983,23 @@ module svcNSP '../modules/network/nsp.bicep' = {
   }
 }
 
+var nspAssociatedResources = deployFrontendCosmos
+  ? [
+      svcCluster.outputs.etcKeyVaultId
+      rpCosmosDb.outputs.cosmosDBAccountId
+    ]
+  : [
+      svcCluster.outputs.etcKeyVaultId
+    ]
+
 module svcClusterNSPProfile '../modules/network/nsp-profile.bicep' = {
   name: 'profile-${uniqueString(resourceGroup().name)}'
   params: {
     accessMode: svcNSPAccessMode
     nspName: svcNSPName
     profileName: svcNSPName
     location: location
-    associatedResources: [
-      svcCluster.outputs.etcKeyVaultId
-      rpCosmosDb.outputs.cosmosDBAccountId
-    ]
+    associatedResources: nspAssociatedResources
     // TODO Add EV2 access here
     subscriptions: [
       subscription().id

templatize.sh

@@ -108,22 +108,24 @@ elif [ $PIPELINE_MODE == "inspect" ] && [ -n "${SERVICE_GROUP+x}" ] && [ -n "${P
         ${LOG_VERBOSITY_OPTION} \
         --format makefile
 elif [ $PIPELINE_MODE == "run" ] && [ -n "${SERVICE_GROUP+x}" ] && [ -n "${PIPELINE_STEP+x}" ]; then
+    TOPOLOGY_FILE="${TOPOLOGY_FILE:-${PROJECT_ROOT_DIR}/topology.yaml}"
     $TEMPLATIZE pipeline run \
         --config-file="${CONFIG_FILE}" \
         --dev-settings-file="${PROJECT_ROOT_DIR}/tooling/templatize/settings.yaml" \
         --dev-environment="${DEPLOY_ENV}" "${REGION:+"--region=${REGION}"}" \
-        --topology-file="${PROJECT_ROOT_DIR}/topology.yaml" \
+        --topology-file="${TOPOLOGY_FILE}" \
         --service-group="${SERVICE_GROUP}" \
         --step="${PIPELINE_STEP}" \
         ${PERSIST_FLAG} \
         ${LOG_VERBOSITY_OPTION} \
         ${DRY_RUN}
 elif [ $PIPELINE_MODE == "run" ] && [ -n "${SERVICE_GROUP+x}" ]; then
+    TOPOLOGY_FILE="${TOPOLOGY_FILE:-${PROJECT_ROOT_DIR}/topology.yaml}"
     $TEMPLATIZE pipeline run \
         --config-file="${CONFIG_FILE}" \
         --dev-settings-file="${PROJECT_ROOT_DIR}/tooling/templatize/settings.yaml" \
         --dev-environment="${DEPLOY_ENV}" "${REGION:+"--region=${REGION}"}" \
-        --topology-file="${PROJECT_ROOT_DIR}/topology.yaml" \
+        --topology-file="${TOPOLOGY_FILE}" \
         --service-group="${SERVICE_GROUP}" \
         ${PERSIST_FLAG} \
         ${LOG_VERBOSITY_OPTION} \

tooling/templatize/settings.yaml

@@ -64,3 +64,12 @@ environments:
     ev2Cloud: public
     cxStamp: 1
     regionShortSuffix: "s${USER:0:4}"
+- name: opstool
+  description: |
+    Used for SRE tooling AKS cluster deployment.
+  defaults:
+    region: uksouth
+    cloud: dev
+    ev2Cloud: public
+    cxStamp: 1
+    regionShortSuffix: "t"

topology-opstool.yaml

@@ -0,0 +1,48 @@
+entrypoints:
+- identifier: 'Microsoft.Azure.ARO.HCP.Global'
+  metadata:
+    name: Global
+    scopeDoc: high-level-architecture.md
+    incremental: "true"
+- identifier: 'Microsoft.Azure.ARO.HCP.Region'
+  metadata:
+    name: Region
+    scopeDoc: high-level-architecture.md#regional-scope
+- identifier: 'Microsoft.Azure.ARO.HCP.Service.Infra'
+  metadata:
+    name: Service Cluster
+    scopeDoc: high-level-architecture.md#service-cluster
+services:
+- serviceGroup: Microsoft.Azure.ARO.HCP.Global
+  children:
+  - serviceGroup: Microsoft.Azure.ARO.HCP.Region
+    children:
+    - serviceGroup: Microsoft.Azure.ARO.HCP.Service.Infra
+      children:
+      pipelinePath: dev-infrastructure/svc-pipeline.yaml
+      purpose: Deploy the service cluster and supporting infrastructure.
+    - serviceGroup: Microsoft.Azure.ARO.HCP.Monitoring
+      pipelinePath: dev-infrastructure/monitoring-pipeline.yaml
+      purpose: Deploy the Monitoring resources
+    pipelinePath: dev-infrastructure/region-pipeline.yaml
+    purpose: Deploy regional shared infrastructure.
+  pipelinePath: dev-infrastructure/global-pipeline.yaml
+  purpose: Deploy global shared infrastructure.
+# Cleanup pipelines
+- serviceGroup: Microsoft.Azure.ARO.HCP.Service.Delete
+  pipelinePath: dev-infrastructure/cleanup/delete.svc.pipeline.yaml
+  purpose: Delete the service resources and service resource group
+- serviceGroup: Microsoft.Azure.ARO.HCP.Region.Delete
+  pipelinePath: dev-infrastructure/cleanup/delete.region.pipeline.yaml
+  purpose: Delete the region resources and resource group
+# Dev-only pipelines
+- serviceGroup: Microsoft.Azure.ARO.HCP.Observability
+  pipelinePath: observability/tracing/pipeline.yaml
+  purpose: Deploy the development tracing stack.
+# Kusto Infra pipeline
+- serviceGroup: Microsoft.Azure.ARO.HCP.Log.Infra
+  pipelinePath: dev-infrastructure/kusto-pipeline.yaml
+  purpose: Deploy the kusto log infrastructure.
+- serviceGroup: Microsoft.Azure.ARO.HCP.Service.Kubeconfig
+  pipelinePath: dev-infrastructure/svc-kubeconfig.yaml
+  purpose: Grant access to AKS SVC AKS Clusters, mainly intended for E2E test setup.