gitlab templatized pipeline v1.0

Author: aguard-redgate

Gitlab-Templatized/.env.example

@@ -0,0 +1,19 @@
+# Local GitLab environment configuration
+# These values are for the local Docker-based GitLab instance only.
+# Copy this file to .env and fill in your values.
+
+GITLAB_RUNNER_REGISTRATION_TOKEN=your_runner_registration_token
+GITLAB_URL=http://172.17.0.2
+GITLAB_EXTERNAL_URL=http://localhost:8080
+
+# SQL Server (SQLEXPRESS) — used by Flyway CI/CD variables
+# From Docker containers, use host.docker.internal (Rancher Desktop VM)
+SQL_HOST=host.docker.internal
+SQL_PORT=1433
+SQL_USER=sa
+SQL_PASSWORD=your_sql_password
+
+# Flyway Enterprise licensing (Personal Access Token)
+# See: https://documentation.red-gate.com/fd/licensing-164167730.html#Licensing-AccessToken(PAT)
+FLYWAY_EMAIL=your_email@example.com
+FLYWAY_TOKEN=your_flyway_personal_access_token

Gitlab-Templatized/.gitignore

@@ -1,57 +1,2 @@
-# .gitignore for GitLab Flyway Pipeline Project
-
-# Active pipeline file (users create this by copying an example)
-# Uncomment the line below if you want to keep .gitlab-ci.yml local
-# and not commit it to the repository
-#.gitlab-ci.yml
-
-# User-specific SQL migrations (optional - comment out if you want to track them)
-# sql/*.sql
-
-# Flyway metadata
-flyway.conf
-
-# Environment files (if used)
 .env
-.env.local
-*.env
-
-# IDE and Editor files
-.vscode/
-.idea/
-*.swp
-*.swo
-*~
-.DS_Store
-
-# OS specific files
-Thumbs.db
-Desktop.ini
-
-# Build artifacts
-dist/
-build/
-*.log
-
-# Temporary files
-tmp/
-temp/
-*.tmp
-
-# Personal notes (users might create these)
-notes.md
-TODO.md
-scratch.txt
-
-# Backup files
-*.bak
-*.backup
-
-# GitLab Runner specific
-.gitlab-runner/
-
-# Docker volumes (if testing locally)
-gitlab-config/
-gitlab-logs/
-gitlab-data/
-rancher-data/
+!.env.example

Gitlab-Templatized/.gitlab-ci-example-all-regions.yml

@@ -22,11 +22,10 @@
 # =============================================================================
 
 include:
-  - local: '.gitlab/ci/generate.yml'   # .fetch_targets and .generate_pipeline templates
+  - local: '.gitlab/ci/generate.yml'   # .generate_pipeline template
   - local: '.gitlab/ci/flyway.yml'     # Flyway templates (used by child pipeline)
 
 stages:
-  - fetch
   - generate
   - deploy
 
@@ -43,37 +42,20 @@ variables:
   # Registry connection (set sensitive values in GitLab CI/CD Variables)
   REGISTRY_DATABASE:   "flyway_registry"
   REGISTRY_PORT:       "1433"
-  SQL_SERVER_PORT:     "1433"
-
-  # Set to "0" to target primary databases only (skip replicas)
-  INCLUDE_REPLICAS:    "1"
-
-# ---------------------------------------------------------------------------
-# Stage 1 – fetch: query the registry, write targets.json artifact
-# ---------------------------------------------------------------------------
-fetch:all-regions:
-  extends: .fetch_targets
-  variables:
-    FILTER_LOCATION: "all"
-  only:
-    - main
-    - tags
 
 # ---------------------------------------------------------------------------
-# Stage 2 – generate: read targets.json, write dynamic-pipeline.yml artifact
-# No database connection required at this stage.
+# Stage 1 – generate: query registry, build JDBCs, write dynamic-pipeline.yml
 # ---------------------------------------------------------------------------
 generate:all-regions:
   extends: .generate_pipeline
-  needs:
-    - job: fetch:all-regions
-      artifacts: true
-  only:
-    - main
-    - tags
+  variables:
+    FILTER_LOCATION: "all"
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
+    - if: $CI_COMMIT_TAG
 
 # ---------------------------------------------------------------------------
-# Stage 3 – deploy: trigger the generated child pipeline
+# Stage 2 – deploy: trigger the generated child pipeline
 # strategy: depend → parent waits for all child jobs to finish
 # ---------------------------------------------------------------------------
 deploy:all-regions:
@@ -83,6 +65,6 @@ deploy:all-regions:
       - artifact: dynamic-pipeline.yml
         job:      generate:all-regions
     strategy: depend
-  only:
-    - main
-    - tags
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
+    - if: $CI_COMMIT_TAG

Gitlab-Templatized/.gitlab-ci-example-dev.yml

@@ -20,33 +20,34 @@ stages:
 validate:dev:
   extends: .flyway_validate
   stage: validate
-  only:
-    - dev
-    - merge_requests
+  rules:
+    - if: $CI_COMMIT_BRANCH == "dev"
+    - if: $CI_MERGE_REQUEST_IID
 
 # Show migration info
 info:dev:
   extends: .flyway_info
   stage: validate
-  only:
-    - dev
-    - merge_requests
+  rules:
+    - if: $CI_COMMIT_BRANCH == "dev"
+    - if: $CI_MERGE_REQUEST_IID
 
 # Deploy migrations to DEV
 migrate:dev:
   extends: .flyway_migrate
   stage: deploy
-  only:
-    - dev
   environment:
     name: development
     url: https://dev.myapp.com
+  rules:
+    - if: $CI_COMMIT_BRANCH == "dev"
 
 # Manual repair option
 repair:dev:
   extends: .flyway_repair
   stage: deploy
-  only:
-    - dev
   environment:
     name: development
+  rules:
+    - if: $CI_COMMIT_BRANCH == "dev"
+      when: manual

Gitlab-Templatized/.gitlab-ci-example-matrix.yml

@@ -47,10 +47,10 @@ migrate:all-databases:
     FLYWAY_URL: "${TARGET_DATABASE_JDBC_${DB_NUMBER}}"
     FLYWAY_USER: "${TARGET_DATABASE_USER_${DB_NUMBER}}"
     FLYWAY_PASSWORD: "${TARGET_DATABASE_PASSWORD_${DB_NUMBER}}"
-  only:
-    - main
   environment:
     name: database-${DB_NUMBER}
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
   # Optional: Control parallelism to avoid overwhelming database servers
   # Maximum 50 jobs run concurrently by default
 

Gitlab-Templatized/.gitlab-ci-example-multi-db.yml

@@ -41,12 +41,12 @@ migrate:users-db:
     FLYWAY_URL: "${TARGET_DATABASE_JDBC_1}"
     FLYWAY_USER: "${TARGET_DATABASE_USER_1}"
     FLYWAY_PASSWORD: "${TARGET_DATABASE_PASSWORD_1}"
-    # Optional: Use separate SQL folder for this database
-    # FLYWAY_LOCATIONS: "filesystem:./sql/users"
-  only:
-    - main
+    # Optional: Use separate migrations folder for this database
+    # FLYWAY_LOCATIONS: "filesystem:./migrations/users"
   environment:
     name: users-database
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
 
 # Database 2: Orders DB
 migrate:orders-db:
@@ -56,11 +56,11 @@ migrate:orders-db:
     FLYWAY_URL: "${TARGET_DATABASE_JDBC_2}"
     FLYWAY_USER: "${TARGET_DATABASE_USER_2}"
     FLYWAY_PASSWORD: "${TARGET_DATABASE_PASSWORD_2}"
-    # FLYWAY_LOCATIONS: "filesystem:./sql/orders"
-  only:
-    - main
+    # FLYWAY_LOCATIONS: "filesystem:./migrations/orders"
   environment:
     name: orders-database
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
 
 # Database 3: Analytics DB
 migrate:analytics-db:
@@ -70,11 +70,11 @@ migrate:analytics-db:
     FLYWAY_URL: "${TARGET_DATABASE_JDBC_3}"
     FLYWAY_USER: "${TARGET_DATABASE_USER_3}"
     FLYWAY_PASSWORD: "${TARGET_DATABASE_PASSWORD_3}"
-    # FLYWAY_LOCATIONS: "filesystem:./sql/analytics"
-  only:
-    - main
+    # FLYWAY_LOCATIONS: "filesystem:./migrations/analytics"
   environment:
     name: analytics-database
-  when: manual  # Analytics can be deployed separately if needed
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
+      when: manual  # Analytics can be deployed separately if needed
 
 # Add more databases by copying the pattern above and incrementing the number

Gitlab-Templatized/.gitlab-ci-example-prod.yml

@@ -29,36 +29,33 @@ stages:
 validate:prod:
   extends: .flyway_validate
   stage: validate
-  only:
-    - main
-    - tags
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
+    - if: $CI_COMMIT_TAG
 
 # Deploy to STAGING first
 migrate:staging:
   extends: .flyway_migrate
   stage: staging
-  only:
-    - main
   environment:
     name: staging
     url: https://staging.myapp.com
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
 
 # Show production info before migration
 info:production:
   extends: .flyway_info
   stage: production
-  only:
-    - tags
   environment:
     name: production
+  rules:
+    - if: $CI_COMMIT_TAG
 
 # Deploy to PRODUCTION (manual approval required)
 migrate:production:
   extends: .flyway_migrate
   stage: production
-  only:
-    - tags
-  when: manual
   environment:
     name: production
     url: https://myapp.com
@@ -69,13 +66,18 @@ migrate:production:
     - echo "Database URL: ${FLYWAY_URL}"
     - echo "This requires manual approval. Click 'Play' button to proceed."
     - echo "==========================================="
+  rules:
+    - if: $CI_COMMIT_TAG
+      when: manual
 
 # Manual repair for production (use with caution)
 repair:production:
   extends: .flyway_repair
   stage: production
-  only:
-    - main
-    - tags
   environment:
     name: production
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
+      when: manual
+    - if: $CI_COMMIT_TAG
+      when: manual

Gitlab-Templatized/.gitlab-ci-example-region-london.yml

@@ -24,7 +24,6 @@ include:
   - local: '.gitlab/ci/flyway.yml'
 
 stages:
-  - fetch
   - generate
   - deploy
 
@@ -39,28 +38,17 @@ variables:
 
   REGISTRY_DATABASE:   "flyway_registry"
   REGISTRY_PORT:       "1433"
-  SQL_SERVER_PORT:     "1433"
-  INCLUDE_REPLICAS:    "1"
 
 # ---------------------------------------------------------------------------
-# Stage 1 – fetch: query the registry for London targets only
-# ---------------------------------------------------------------------------
-fetch:london:
-  extends: .fetch_targets
-  variables:
-    FILTER_LOCATION: "London"
-
-# ---------------------------------------------------------------------------
-# Stage 2 – generate: produce dynamic-pipeline.yml from targets.json
+# Stage 1 – generate: query registry, build JDBCs, write dynamic-pipeline.yml
 # ---------------------------------------------------------------------------
 generate:london:
   extends: .generate_pipeline
-  needs:
-    - job: fetch:london
-      artifacts: true
+  variables:
+    FILTER_LOCATION: "London"
 
 # ---------------------------------------------------------------------------
-# Stage 3 – deploy
+# Stage 2 – deploy: trigger the generated child pipeline
 # ---------------------------------------------------------------------------
 deploy:london:
   stage: deploy