build pipeline dynamically based on list of dbs

Author: aguard-redgate

Gitlab-Templatized/.env.example

@@ -3,7 +3,7 @@
 # Copy this file to .env and fill in your values.
 
 GITLAB_RUNNER_REGISTRATION_TOKEN=your_runner_registration_token
-GITLAB_URL=http://172.17.0.2
+GITLAB_URL=http://172.30.0.2
 GITLAB_EXTERNAL_URL=http://localhost:8080
 
 # SQL Server (SQLEXPRESS) — used by Flyway CI/CD variables
@@ -13,6 +13,15 @@ SQL_PORT=1433
 SQL_USER=sa
 SQL_PASSWORD=your_sql_password
 
+# Flyway Pipeline Generator - registry connection
+# These map to the same SQL Server used for Flyway targets.
+# In GitLab CI/CD, set these as project/group CI/CD variables instead.
+REGISTRY_SERVER=host.docker.internal
+REGISTRY_PORT=1433
+REGISTRY_USER=sa
+REGISTRY_PASSWORD=your_sql_password
+REGISTRY_DATABASE=flyway_registry
+
 # Flyway Enterprise licensing (Personal Access Token)
 # See: https://documentation.red-gate.com/fd/licensing-164167730.html#Licensing-AccessToken(PAT)
 FLYWAY_EMAIL=your_email@example.com

Gitlab-Templatized/.gitignore

@@ -1,2 +1,57 @@
+# .gitignore for GitLab Flyway Pipeline Project
+
+# Active pipeline file (users create this by copying an example)
+# Uncomment the line below if you want to keep .gitlab-ci.yml local
+# and not commit it to the repository
+#.gitlab-ci.yml
+
+# User-specific SQL migrations (optional - comment out if you want to track them)
+# sql/*.sql
+
+# Flyway metadata
+flyway.conf
+
+# Environment files (if used)
 .env
-!.env.example
+.env.local
+!.env.example
+
+# IDE and Editor files
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# OS specific files
+Thumbs.db
+Desktop.ini
+
+# Build artifacts
+dist/
+build/
+*.log
+
+# Temporary files
+tmp/
+temp/
+*.tmp
+
+# Personal notes (users might create these)
+notes.md
+TODO.md
+scratch.txt
+
+# Backup files
+*.bak
+*.backup
+
+# GitLab Runner specific
+.gitlab-runner/
+
+# Docker volumes (if testing locally)
+gitlab-config/
+gitlab-logs/
+gitlab-data/
+rancher-data/

Gitlab-Templatized/.gitlab-ci.yml

@@ -0,0 +1,77 @@
+# =============================================================================
+# Dynamic Pipeline — Registry-Driven Flyway Deployments
+# =============================================================================
+# Uses generate_pipeline.py to query a SQL Server registry database,
+# discover target databases, and produce a child pipeline with one
+# Flyway migrate job per target.
+#
+# CI/CD Variables required (Settings → CI/CD → Variables):
+#   REGISTRY_SERVER            = SQL Server hostname for the registry DB
+#   REGISTRY_USER              = Login for the registry DB
+#   REGISTRY_PASSWORD          = *** (Protected + Masked)
+#   TARGET_DATABASE_USER       = Login for target databases
+#   TARGET_DATABASE_PASSWORD   = *** (Protected + Masked)
+#   FLYWAY_EMAIL               = your@email.com
+#   FLYWAY_TOKEN               = *** (Protected + Masked)
+#
+# Optional CI/CD Variables:
+#   REGISTRY_DATABASE          = flyway_registry  (default)
+#   REGISTRY_PORT              = 1433             (default)
+#   JDBC_PORT                  = 1433             (default)
+#   FILTER_LOCATION            = all              (default — or e.g. "London")
+#   INCLUDE_REPLICAS           = false            (default)
+#   FLYWAY_LOCATIONS           = filesystem:./migrations (default)
+#   RUNNER_TAG_MAP             = JSON map of location → runner tag overrides
+#
+# Consumer repo structure:
+#   my-flyway-project/
+#   ├── migrations/
+#   │   ├── V1__create_schema.sql
+#   │   └── V2__add_users_table.sql
+#   └── .gitlab-ci.yml          ← this file
+#
+# Optional CI/CD Variables for template repo override:
+#   TEMPLATE_PROJECT           = root/templatized-with-parser (default)
+#   TEMPLATE_REF               = main                        (default)
+# =============================================================================
+
+stages:
+  - generate
+  - deploy
+
+variables:
+  TEMPLATE_PROJECT: "root/templatized-with-parser"
+  TEMPLATE_REF: "main"
+
+# ---------------------------------------------------------------------------
+# Stage 1: Generate the child pipeline YAML from the registry
+# ---------------------------------------------------------------------------
+generate-pipeline:
+  stage: generate
+  image: python:3.12-slim
+  script:
+    - apt-get update -qq && apt-get install -y -qq git > /dev/null
+    - git clone --depth 1 --branch "$TEMPLATE_REF"
+        "http://gitlab-ci-token:${CI_JOB_TOKEN}@172.30.0.2/${TEMPLATE_PROJECT}.git"
+        /tmp/templates
+    - pip install --quiet -r /tmp/templates/scripts/requirements.txt
+    - python /tmp/templates/scripts/generate_pipeline.py
+  artifacts:
+    paths:
+      - dynamic-pipeline.yml
+    expire_in: 1 hour
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
+
+# ---------------------------------------------------------------------------
+# Stage 2: Trigger the generated child pipeline
+# ---------------------------------------------------------------------------
+deploy-targets:
+  stage: deploy
+  trigger:
+    include:
+      - artifact: dynamic-pipeline.yml
+        job: generate-pipeline
+    strategy: depend
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"

Gitlab-Templatized/README.md

@@ -142,9 +142,32 @@ Optional `generate_pipeline.py` settings:
 | `FILTER_LOCATION` | `all` | Region filter (`London`, `New York`, `Tokyo`) |
 | `INCLUDE_REPLICAS` | `false` | Include replicated databases |
 | `JDBC_PORT` | `1433` | Port in generated JDBC URLs |
+| `RUNNER_TAG_DEFAULT` | _(empty)_ | Single runner tag for all generated jobs (overrides per-location tags) |
 | `TEMPLATE_PROJECT` | _(empty)_ | Templates repo path for child pipeline include |
 | `TEMPLATE_REF` | `main` | Git ref for cross-project child pipeline include |
 
+## Runner Tags
+
+Generated pipeline jobs are assigned runner tags based on each target's `location`. By default, a location like `Production` produces the tag `runner-production`.
+
+**Override all tags with one value** — set `RUNNER_TAG_DEFAULT` as a CI/CD variable:
+
+```
+RUNNER_TAG_DEFAULT = local-runner
+```
+
+Every generated job will use `local-runner` instead of per-location tags.
+
+**Per-location overrides** — set `RUNNER_TAG_MAP` as a JSON CI/CD variable:
+
+```
+RUNNER_TAG_MAP = {"Production": "prod-runner", "Development": "dev-runner"}
+```
+
+Or use individual variables: `RUNNER_TAG_LONDON`, `RUNNER_TAG_NEW_YORK`, `RUNNER_TAG_TOKYO`.
+
+**Priority order:** `RUNNER_TAG_DEFAULT` → `RUNNER_TAG_MAP` → individual `RUNNER_TAG_*` → auto-generated from location name.
+
 ## Usage Examples
 
 See [`usage-examples/`](usage-examples/) for complete `.gitlab-ci.yml` files:

Gitlab-Templatized/scripts/generate_pipeline.py

@@ -39,6 +39,7 @@
 TEMPLATE_PROJECT     Templates repo path for child pipeline include
 TEMPLATE_REF         Templates repo ref for child pipeline include
 OUTPUT_FILE          Output YAML filename          (default: dynamic-pipeline.yml)
+RUNNER_TAG_DEFAULT   Single runner tag for all jobs (overrides per-location tags)
 RUNNER_TAG_MAP       JSON map of location->runner tag overrides
 RUNNER_TAG_LONDON    Runner tag for London         (default: runner-london)
 RUNNER_TAG_NEW_YORK  Runner tag for New York       (default: runner-new-york)
@@ -53,6 +54,37 @@
 import yaml
 
 
+# ---------------------------------------------------------------------------
+# Load .env file if present (local development)
+# In GitLab CI/CD, variables come from project/group CI/CD settings natively.
+# ---------------------------------------------------------------------------
+
+def load_dotenv(path=None):
+    """Load a .env file into os.environ without overwriting existing values."""
+    if path is None:
+        # Look for .env in the repo root (one level up from scripts/)
+        path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, ".env")
+    path = os.path.normpath(path)
+    if not os.path.isfile(path):
+        return
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            if "=" not in line:
+                continue
+            key, _, value = line.partition("=")
+            key = key.strip()
+            value = value.strip()
+            # Don't overwrite — GitLab CI/CD variables take precedence
+            if key not in os.environ:
+                os.environ[key] = value
+
+
+load_dotenv()
+
+
 # ---------------------------------------------------------------------------
 # Configuration from environment
 # ---------------------------------------------------------------------------
@@ -75,7 +107,8 @@
 TEMPLATE_PROJECT = os.environ.get("TEMPLATE_PROJECT", "")
 TEMPLATE_REF     = os.environ.get("TEMPLATE_REF", "main")
 
-# Runner tags — JSON map takes priority, then individual env vars, then defaults
+# Runner tags — default overrides all, then JSON map, then individual env vars, then fallbacks
+RUNNER_TAG_DEFAULT = os.environ.get("RUNNER_TAG_DEFAULT", "")
 _runner_tag_map_raw = os.environ.get("RUNNER_TAG_MAP", "")
 if _runner_tag_map_raw:
     RUNNER_TAGS = json.loads(_runner_tag_map_raw)
@@ -178,6 +211,8 @@ def safe_job_name(value):
 
 
 def runner_tag_for(location):
+    if RUNNER_TAG_DEFAULT:
+        return RUNNER_TAG_DEFAULT
     return RUNNER_TAGS.get(location, f"runner-{location.lower().replace(' ', '-')}")
 
 

Gitlab-Templatized/startup-services.ps1

@@ -14,6 +14,17 @@ if (Test-Path $envFile) {
 Write-Host "Starting services..." -ForegroundColor Yellow
 Write-Host ""
 
+# Create a dedicated Docker network with a fixed subnet (idempotent)
+$networkName = "gitlab-net"
+$subnet = "172.30.0.0/24"
+$networkExists = docker network ls --filter "name=^${networkName}$" --format '{{.Name}}' 2>&1
+if ($networkExists -ne $networkName) {
+    Write-Host "Creating Docker network '$networkName' (subnet $subnet)..." -ForegroundColor Cyan
+    docker network create --subnet $subnet $networkName | Out-Null
+} else {
+    Write-Host "Docker network '$networkName' already exists." -ForegroundColor Gray
+}
+
 # Ensure Rancher Desktop is running
 if (-not (Get-Process "Rancher Desktop" -ErrorAction SilentlyContinue)) {
     Write-Host "Rancher Desktop is not running. Starting it..." -ForegroundColor Yellow
@@ -38,6 +49,7 @@ if (-not (Get-Process "Rancher Desktop" -ErrorAction SilentlyContinue)) {
 Write-Host "Starting Rancher..." -ForegroundColor Cyan
 docker run -d --restart=unless-stopped `
     --name rancher `
+    --network gitlab-net --ip 172.30.0.10 `
     -p 80:80 -p 443:443 `
     --privileged `
     rancher/rancher:latest
@@ -46,6 +58,7 @@ docker run -d --restart=unless-stopped `
 Write-Host "Starting GitLab..." -ForegroundColor Cyan
 docker run -d --restart=unless-stopped `
     --name gitlab `
+    --network gitlab-net --ip 172.30.0.2 `
     -p 8080:80 -p 8443:443 -p 2222:22 `
     -v gitlab-config:/etc/gitlab `
     -v gitlab-logs:/var/log/gitlab `
@@ -56,6 +69,7 @@ docker run -d --restart=unless-stopped `
 Write-Host "Starting GitLab Runner..." -ForegroundColor Cyan
 docker run -d --restart=unless-stopped `
     --name gitlab-runner `
+    --network gitlab-net --ip 172.30.0.3 `
     -v gitlab-runner-config:/etc/gitlab-runner `
     -v //var/run/docker.sock:/var/run/docker.sock `
     gitlab/gitlab-runner:latest
@@ -74,10 +88,13 @@ if ($runnerConfig -notmatch '\[\[runners\]\]') {
             --executor "docker" `
             --docker-image "redgate/flyway:12-enterprise-alpine" `
             --description "local-runner" `
+            --tag-list "local-runner" `
             --run-untagged="true" `
-            --locked="false"
+            --locked="false" `
+            --docker-network-mode "gitlab-net" `
+            --clone-url $url
     } else {
-        Write-Host "WARNING: GITLAB_RUNNER_REGISTRATION_TOKEN or GITLAB_URL not set in .env — skipping registration." -ForegroundColor Yellow
+        Write-Host "WARNING: GITLAB_RUNNER_REGISTRATION_TOKEN or GITLAB_URL not set in .env - skipping registration." -ForegroundColor Yellow
     }
 }