work in progress templatized. current python builds out the pipeline

Author: aguard-redgate

Gitlab-Templatized/.gitignore

@@ -0,0 +1,57 @@
+# .gitignore for GitLab Flyway Pipeline Project
+
+# Active pipeline file (users create this by copying an example)
+# Uncomment the line below if you want to keep .gitlab-ci.yml local
+# and not commit it to the repository
+#.gitlab-ci.yml
+
+# User-specific SQL migrations (optional - comment out if you want to track them)
+# sql/*.sql
+
+# Flyway metadata
+flyway.conf
+
+# Environment files (if used)
+.env
+.env.local
+*.env
+
+# IDE and Editor files
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# OS specific files
+Thumbs.db
+Desktop.ini
+
+# Build artifacts
+dist/
+build/
+*.log
+
+# Temporary files
+tmp/
+temp/
+*.tmp
+
+# Personal notes (users might create these)
+notes.md
+TODO.md
+scratch.txt
+
+# Backup files
+*.bak
+*.backup
+
+# GitLab Runner specific
+.gitlab-runner/
+
+# Docker volumes (if testing locally)
+gitlab-config/
+gitlab-logs/
+gitlab-data/
+rancher-data/

Gitlab-Templatized/.gitlab-ci-example-all-regions.yml

@@ -0,0 +1,80 @@
+# =============================================================================
+# Pipeline: Deploy ALL regions at once (London + New York + Tokyo)
+# =============================================================================
+# This pipeline queries the jdbc_table_store registry and generates
+# a single child pipeline containing jobs for EVERY available database across
+# all three regions. Each job is automatically tagged to run on the runner
+# serving that region.
+#
+# SETUP REQUIRED – GitLab Settings → CI/CD → Variables
+# -------------------------------------------------------
+# Registry database (used by the Python generator):
+#   REGISTRY_SERVER   = <hostname of your flyway_registry SQL Server>
+#   REGISTRY_USER     = <login on that server>
+#   REGISTRY_PASSWORD = <password>  ← Protected + Masked
+#
+# Target databases (used by Flyway jobs at runtime):
+#   TARGET_DATABASE_USER     = flyway_user
+#   TARGET_DATABASE_PASSWORD = <password>  ← Protected + Masked
+#
+# RUNNER TAGS – change these to match your actual runner tag names
+# The defaults below follow the convention set in .gitlab/ci/generate.yml
+# =============================================================================
+
+include:
+  - local: '.gitlab/ci/generate.yml'   # .generate_targets base job
+  - local: '.gitlab/ci/flyway.yml'     # Flyway templates (used by child pipeline)
+
+stages:
+  - generate
+  - deploy
+
+# ---------------------------------------------------------------------------
+# Configurable runner tags
+# Change these values to match the tags registered on your GitLab runners.
+# One-line change per region – that's all you need.
+# ---------------------------------------------------------------------------
+variables:
+  RUNNER_TAG_LONDON:   "runner-london"
+  RUNNER_TAG_NEW_YORK: "runner-new-york"
+  RUNNER_TAG_TOKYO:    "runner-tokyo"
+
+  # Registry connection (set sensitive values in GitLab CI/CD Variables)
+  REGISTRY_DATABASE:   "flyway_registry"
+  REGISTRY_PORT:       "1433"
+  SQL_SERVER_PORT:     "1433"
+
+  # Set to "0" to target primary databases only (skip replicas)
+  INCLUDE_REPLICAS:    "1"
+
+# ---------------------------------------------------------------------------
+# Stage 1 – generate: build the child pipeline YAML from the registry
+# ---------------------------------------------------------------------------
+generate:all-regions:
+  extends: .generate_targets
+  stage: generate
+  variables:
+    FILTER_LOCATION: "all"
+    OUTPUT_FILE:     "generated-all-regions.yml"
+  artifacts:
+    paths:
+      - generated-all-regions.yml
+    expire_in: 2 hours
+  only:
+    - main
+    - tags
+
+# ---------------------------------------------------------------------------
+# Stage 2 – deploy: trigger the generated child pipeline
+# strategy: depend  → parent waits for all child jobs to finish
+# ---------------------------------------------------------------------------
+deploy:all-regions:
+  stage: deploy
+  trigger:
+    include:
+      - artifact: generated-all-regions.yml
+        job:      generate:all-regions
+    strategy: depend
+  only:
+    - main
+    - tags

Gitlab-Templatized/.gitlab-ci-example-dev.yml

@@ -1,5 +1,11 @@
 # Example Pipeline 1: Development Environment
 # This pipeline validates and deploys migrations to the DEV database
+#
+# SETUP REQUIRED:
+# Configure these CI/CD variables in GitLab (Settings → CI/CD → Variables):
+#   - TARGET_DATABASE_JDBC = jdbc:postgresql://dev-db.example.com:5432/myapp_dev
+#   - TARGET_DATABASE_USER = flyway_user
+#   - TARGET_DATABASE_PASSWORD = your-password (mark as Protected and Masked)
 
 include:
   - local: '.gitlab/ci/flyway.yml'
@@ -8,10 +14,7 @@ stages:
   - validate
   - deploy
 
-variables:
-  DB_URL: "jdbc:postgresql://dev-db.example.com:5432/myapp_dev"
-  DB_USER: "flyway_user"
-  # DB_PASSWORD should be set as a CI/CD variable in GitLab UI
+# No additional variables needed - uses TARGET_DATABASE_* from GitLab CI/CD settings
 
 # Validate migrations on every commit
 validate:dev:

Gitlab-Templatized/.gitlab-ci-example-matrix.yml

@@ -0,0 +1,74 @@
+# Example Pipeline 4: Matrix-Based Multi-Database Deployment
+# This pipeline uses GitLab's parallel:matrix feature to deploy to N databases
+# Best for: 10-100 databases where explicit jobs become impractical
+#
+# SETUP REQUIRED:
+# Configure numbered CI/CD variables in GitLab (Settings → CI/CD → Variables):
+#   - TARGET_DATABASE_JDBC_1 through TARGET_DATABASE_JDBC_N
+#   - TARGET_DATABASE_USER_1 through TARGET_DATABASE_USER_N
+#   - TARGET_DATABASE_PASSWORD_1 through TARGET_DATABASE_PASSWORD_N
+#
+# Example for 50 databases:
+#   TARGET_DATABASE_JDBC_1 = jdbc:postgresql://tenant1-db:5432/app
+#   TARGET_DATABASE_USER_1 = flyway
+#   TARGET_DATABASE_PASSWORD_1 = secret1
+#   ...
+#   TARGET_DATABASE_JDBC_50 = jdbc:postgresql://tenant50-db:5432/app
+#   TARGET_DATABASE_USER_50 = flyway
+#   TARGET_DATABASE_PASSWORD_50 = secret50
+
+include:
+  - local: '.gitlab/ci/flyway.yml'
+
+stages:
+  - validate
+  - deploy
+
+# Validate once using first database
+validate:matrix:
+  extends: .flyway_validate
+  stage: validate
+  variables:
+    FLYWAY_URL: "${TARGET_DATABASE_JDBC_1}"
+    FLYWAY_USER: "${TARGET_DATABASE_USER_1}"
+    FLYWAY_PASSWORD: "${TARGET_DATABASE_PASSWORD_1}"
+
+# Deploy to N databases in parallel
+# Update the DB_NUMBER list to match your database count
+migrate:all-databases:
+  extends: .flyway_migrate
+  stage: deploy
+  parallel:
+    matrix:
+      - DB_NUMBER: ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]
+        # Add more numbers as needed: "11", "12", ..., "100"
+  variables:
+    # GitLab will create separate jobs for each DB_NUMBER
+    FLYWAY_URL: "${TARGET_DATABASE_JDBC_${DB_NUMBER}}"
+    FLYWAY_USER: "${TARGET_DATABASE_USER_${DB_NUMBER}}"
+    FLYWAY_PASSWORD: "${TARGET_DATABASE_PASSWORD_${DB_NUMBER}}"
+  only:
+    - main
+  environment:
+    name: database-${DB_NUMBER}
+  # Optional: Control parallelism to avoid overwhelming database servers
+  # Maximum 50 jobs run concurrently by default
+
+# Alternative: Named databases instead of numbered
+# Uncomment and modify if you prefer named databases like "users", "orders", etc.
+#
+# migrate:named-databases:
+#   extends: .flyway_migrate
+#   stage: deploy
+#   parallel:
+#     matrix:
+#       - DB_NAME: ["users", "orders", "products", "analytics", "reports"]
+#   variables:
+#     FLYWAY_URL: "${TARGET_DATABASE_JDBC_${DB_NAME}}"
+#     FLYWAY_USER: "${TARGET_DATABASE_USER_${DB_NAME}}"
+#     FLYWAY_PASSWORD: "${TARGET_DATABASE_PASSWORD_${DB_NAME}}"
+#   environment:
+#     name: database-${DB_NAME}
+
+# Note: For 100+ databases, consider using dynamic child pipelines
+# See SETUP_GUIDE.md "Scenario 3: Deploy to 100+ Databases"

Gitlab-Templatized/.gitlab-ci-example-multi-db.yml

@@ -1,27 +1,48 @@
-# Example Pipeline 3: Multi-Database Deployment
-# This pipeline deploys to multiple databases using the same shared template
+# Example Pipeline 3: Multi-Database Deployment (Explicit Jobs)
+# This pipeline deploys to multiple databases using shared templates
+# Best for: 2-10 databases that you want to deploy explicitly
+#
+# SETUP REQUIRED:
+# Configure these CI/CD variables in GitLab (Settings → CI/CD → Variables):
+#   Database 1:
+#     - TARGET_DATABASE_JDBC_1 = jdbc:postgresql://db1.example.com:5432/users_db
+#     - TARGET_DATABASE_USER_1 = flyway_user
+#     - TARGET_DATABASE_PASSWORD_1 = password1
+#   Database 2:
+#     - TARGET_DATABASE_JDBC_2 = jdbc:postgresql://db2.example.com:5432/orders_db
+#     - TARGET_DATABASE_USER_2 = flyway_user
+#     - TARGET_DATABASE_PASSWORD_2 = password2
+#   Database 3:
+#     - TARGET_DATABASE_JDBC_3 = jdbc:postgresql://db3.example.com:5432/analytics_db
+#     - TARGET_DATABASE_USER_3 = flyway_user
+#     - TARGET_DATABASE_PASSWORD_3 = password3
 
 include:
   - local: '.gitlab/ci/flyway.yml'
 
 stages:
   - validate
-  - deploy-db1
-  - deploy-db2
-  - deploy-db3
+  - deploy
 
-# Validate all migrations
+# Validate all migrations once (using default variables or DB1)
 validate:all:
   extends: .flyway_validate
   stage: validate
+  variables:
+    FLYWAY_URL: "${TARGET_DATABASE_JDBC_1}"
+    FLYWAY_USER: "${TARGET_DATABASE_USER_1}"
+    FLYWAY_PASSWORD: "${TARGET_DATABASE_PASSWORD_1}"
 
 # Database 1: Users DB
 migrate:users-db:
   extends: .flyway_migrate
-  stage: deploy-db1
+  stage: deploy
   variables:
-    FLYWAY_URL: "${DB_URL_USERS}"
-    FLYWAY_LOCATIONS: "filesystem:./sql/users"
+    FLYWAY_URL: "${TARGET_DATABASE_JDBC_1}"
+    FLYWAY_USER: "${TARGET_DATABASE_USER_1}"
+    FLYWAY_PASSWORD: "${TARGET_DATABASE_PASSWORD_1}"
+    # Optional: Use separate SQL folder for this database
+    # FLYWAY_LOCATIONS: "filesystem:./sql/users"
   only:
     - main
   environment:
@@ -30,10 +51,12 @@ migrate:users-db:
 # Database 2: Orders DB
 migrate:orders-db:
   extends: .flyway_migrate
-  stage: deploy-db2
+  stage: deploy
   variables:
-    FLYWAY_URL: "${DB_URL_ORDERS}"
-    FLYWAY_LOCATIONS: "filesystem:./sql/orders"
+    FLYWAY_URL: "${TARGET_DATABASE_JDBC_2}"
+    FLYWAY_USER: "${TARGET_DATABASE_USER_2}"
+    FLYWAY_PASSWORD: "${TARGET_DATABASE_PASSWORD_2}"
+    # FLYWAY_LOCATIONS: "filesystem:./sql/orders"
   only:
     - main
   environment:
@@ -42,12 +65,16 @@ migrate:orders-db:
 # Database 3: Analytics DB
 migrate:analytics-db:
   extends: .flyway_migrate
-  stage: deploy-db3
+  stage: deploy
   variables:
-    FLYWAY_URL: "${DB_URL_ANALYTICS}"
-    FLYWAY_LOCATIONS: "filesystem:./sql/analytics"
+    FLYWAY_URL: "${TARGET_DATABASE_JDBC_3}"
+    FLYWAY_USER: "${TARGET_DATABASE_USER_3}"
+    FLYWAY_PASSWORD: "${TARGET_DATABASE_PASSWORD_3}"
+    # FLYWAY_LOCATIONS: "filesystem:./sql/analytics"
   only:
     - main
   environment:
     name: analytics-database
-  when: manual  # Analytics can be deployed separately
+  when: manual  # Analytics can be deployed separately if needed
+
+# Add more databases by copying the pattern above and incrementing the number