Skip to content

Commit 963d3c7

Browse files
aguard-redgateaguard-redgate
committed
templatize pipelines
1 parent ce92762 commit 963d3c7

9 files changed

Lines changed: 409 additions & 215 deletions

‎Gitlab-Templatized/.gitlab-ci-example-all-regions.yml‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,11 @@
2222
# =============================================================================
2323

2424
include:
25-
- local: '.gitlab/ci/generate.yml' # .generate_targets base job
25+
- local: '.gitlab/ci/generate.yml' # .fetch_targets and .generate_pipeline templates
2626
- local: '.gitlab/ci/flyway.yml' # Flyway templates (used by child pipeline)
2727

2828
stages:
29+
- fetch
2930
- generate
3031
- deploy
3132

@@ -48,31 +49,38 @@ variables:
4849
INCLUDE_REPLICAS: "1"
4950

5051
# ---------------------------------------------------------------------------
51-
# Stage 1 – generate: build the child pipeline YAML from the registry
52+
# Stage 1 – fetch: query the registry, write targets.json artifact
5253
# ---------------------------------------------------------------------------
53-
generate:all-regions:
54-
extends: .generate_targets
55-
stage: generate
54+
fetch:all-regions:
55+
extends: .fetch_targets
5656
variables:
5757
FILTER_LOCATION: "all"
58-
OUTPUT_FILE: "generated-all-regions.yml"
59-
artifacts:
60-
paths:
61-
- generated-all-regions.yml
62-
expire_in: 2 hours
6358
only:
6459
- main
6560
- tags
6661

6762
# ---------------------------------------------------------------------------
68-
# Stage 2 – deploy: trigger the generated child pipeline
69-
# strategy: depend → parent waits for all child jobs to finish
63+
# Stage 2 – generate: read targets.json, write dynamic-pipeline.yml artifact
64+
# No database connection required at this stage.
65+
# ---------------------------------------------------------------------------
66+
generate:all-regions:
67+
extends: .generate_pipeline
68+
needs:
69+
- job: fetch:all-regions
70+
artifacts: true
71+
only:
72+
- main
73+
- tags
74+
75+
# ---------------------------------------------------------------------------
76+
# Stage 3 – deploy: trigger the generated child pipeline
77+
# strategy: depend → parent waits for all child jobs to finish
7078
# ---------------------------------------------------------------------------
7179
deploy:all-regions:
7280
stage: deploy
7381
trigger:
7482
include:
75-
- artifact: generated-all-regions.yml
83+
- artifact: dynamic-pipeline.yml
7684
job: generate:all-regions
7785
strategy: depend
7886
only:

‎Gitlab-Templatized/.gitlab-ci-example-region-london.yml‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ include:
2424
- local: '.gitlab/ci/flyway.yml'
2525

2626
stages:
27+
- fetch
2728
- generate
2829
- deploy
2930

@@ -33,8 +34,6 @@ stages:
3334
# ---------------------------------------------------------------------------
3435
variables:
3536
RUNNER_TAG_LONDON: "runner-london"
36-
# New York / Tokyo tags are passed through to the generator for
37-
# consistency; they are not used by any job in this pipeline.
3837
RUNNER_TAG_NEW_YORK: "runner-new-york"
3938
RUNNER_TAG_TOKYO: "runner-tokyo"
4039

@@ -44,26 +43,29 @@ variables:
4443
INCLUDE_REPLICAS: "1"
4544

4645
# ---------------------------------------------------------------------------
47-
# Stage 1 – generate
46+
# Stage 1 – fetch: query the registry for London targets only
4847
# ---------------------------------------------------------------------------
49-
generate:london:
50-
extends: .generate_targets
51-
stage: generate
48+
fetch:london:
49+
extends: .fetch_targets
5250
variables:
5351
FILTER_LOCATION: "London"
54-
OUTPUT_FILE: "generated-london.yml"
55-
artifacts:
56-
paths:
57-
- generated-london.yml
58-
expire_in: 2 hours
5952

6053
# ---------------------------------------------------------------------------
61-
# Stage 2 – deploy
54+
# Stage 2 – generate: produce dynamic-pipeline.yml from targets.json
55+
# ---------------------------------------------------------------------------
56+
generate:london:
57+
extends: .generate_pipeline
58+
needs:
59+
- job: fetch:london
60+
artifacts: true
61+
62+
# ---------------------------------------------------------------------------
63+
# Stage 3 – deploy
6264
# ---------------------------------------------------------------------------
6365
deploy:london:
6466
stage: deploy
6567
trigger:
6668
include:
67-
- artifact: generated-london.yml
69+
- artifact: dynamic-pipeline.yml
6870
job: generate:london
6971
strategy: depend

‎Gitlab-Templatized/.gitlab-ci-example-region-new-york.yml‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ include:
2424
- local: '.gitlab/ci/flyway.yml'
2525

2626
stages:
27+
- fetch
2728
- generate
2829
- deploy
2930

@@ -42,26 +43,29 @@ variables:
4243
INCLUDE_REPLICAS: "1"
4344

4445
# ---------------------------------------------------------------------------
45-
# Stage 1 – generate
46+
# Stage 1 – fetch: query the registry for New York targets only
4647
# ---------------------------------------------------------------------------
47-
generate:new-york:
48-
extends: .generate_targets
49-
stage: generate
48+
fetch:new-york:
49+
extends: .fetch_targets
5050
variables:
5151
FILTER_LOCATION: "New York"
52-
OUTPUT_FILE: "generated-new-york.yml"
53-
artifacts:
54-
paths:
55-
- generated-new-york.yml
56-
expire_in: 2 hours
5752

5853
# ---------------------------------------------------------------------------
59-
# Stage 2 – deploy
54+
# Stage 2 – generate: produce dynamic-pipeline.yml from targets.json
55+
# ---------------------------------------------------------------------------
56+
generate:new-york:
57+
extends: .generate_pipeline
58+
needs:
59+
- job: fetch:new-york
60+
artifacts: true
61+
62+
# ---------------------------------------------------------------------------
63+
# Stage 3 – deploy
6064
# ---------------------------------------------------------------------------
6165
deploy:new-york:
6266
stage: deploy
6367
trigger:
6468
include:
65-
- artifact: generated-new-york.yml
69+
- artifact: dynamic-pipeline.yml
6670
job: generate:new-york
6771
strategy: depend

‎Gitlab-Templatized/.gitlab-ci-example-region-tokyo.yml‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ include:
2424
- local: '.gitlab/ci/flyway.yml'
2525

2626
stages:
27+
- fetch
2728
- generate
2829
- deploy
2930

@@ -42,26 +43,29 @@ variables:
4243
INCLUDE_REPLICAS: "1"
4344

4445
# ---------------------------------------------------------------------------
45-
# Stage 1 – generate
46+
# Stage 1 – fetch: query the registry for Tokyo targets only
4647
# ---------------------------------------------------------------------------
47-
generate:tokyo:
48-
extends: .generate_targets
49-
stage: generate
48+
fetch:tokyo:
49+
extends: .fetch_targets
5050
variables:
5151
FILTER_LOCATION: "Tokyo"
52-
OUTPUT_FILE: "generated-tokyo.yml"
53-
artifacts:
54-
paths:
55-
- generated-tokyo.yml
56-
expire_in: 2 hours
5752

5853
# ---------------------------------------------------------------------------
59-
# Stage 2 – deploy
54+
# Stage 2 – generate: produce dynamic-pipeline.yml from targets.json
55+
# ---------------------------------------------------------------------------
56+
generate:tokyo:
57+
extends: .generate_pipeline
58+
needs:
59+
- job: fetch:tokyo
60+
artifacts: true
61+
62+
# ---------------------------------------------------------------------------
63+
# Stage 3 – deploy
6064
# ---------------------------------------------------------------------------
6165
deploy:tokyo:
6266
stage: deploy
6367
trigger:
6468
include:
65-
- artifact: generated-tokyo.yml
69+
- artifact: dynamic-pipeline.yml
6670
job: generate:tokyo
6771
strategy: depend

‎Gitlab-Templatized/.gitlab/ci/generate.yml‎

Lines changed: 79 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,103 @@
1-
# Shared template for dynamic pipeline generation from the jdbc_table_store registry.
2-
# Include this alongside flyway.yml to get the .generate_targets base job.
1+
# Shared templates for dynamic pipeline generation from the jdbc_table_store registry.
2+
#
3+
# Two job templates are provided to keep concerns separate:
4+
#
5+
# .fetch_targets -- queries the registry DB, writes targets.json artifact
6+
# .generate_pipeline -- reads targets.json, writes dynamic-pipeline.yml artifact
7+
#
8+
# Usage in your pipeline file:
39
#
4-
# Usage:
510
# include:
6-
# - local: '.gitlab/ci/flyway.yml'
7-
# - local: '.gitlab/ci/generate.yml'
11+
# - local: ".gitlab/ci/flyway.yml"
12+
# - local: ".gitlab/ci/generate.yml"
13+
#
14+
# stages: [fetch, generate, deploy]
815
#
9-
# Then extend .generate_targets and set:
10-
# FILTER_LOCATION: 'London' | 'New York' | 'Tokyo' | 'all'
11-
# OUTPUT_FILE: name of the generated YAML artifact
16+
# fetch:my-region:
17+
# extends: .fetch_targets
18+
# variables:
19+
# FILTER_LOCATION: "London" # or "New York" / "Tokyo" / "all"
20+
#
21+
# generate:my-region:
22+
# extends: .generate_pipeline
23+
# needs:
24+
# - job: fetch:my-region
25+
# artifacts: true
26+
#
27+
# deploy:my-region:
28+
# stage: deploy
29+
# trigger:
30+
# include:
31+
# - artifact: dynamic-pipeline.yml
32+
# job: generate:my-region
33+
# strategy: depend
1234

1335
# ---------------------------------------------------------------------------
14-
# Base job: generates the child pipeline YAML from the registry database.
15-
# Extend this job and override FILTER_LOCATION + OUTPUT_FILE as needed.
36+
# Stage 1 fetch
37+
# Queries the registry stored procedure and writes targets.json.
1638
# ---------------------------------------------------------------------------
17-
.generate_targets:
18-
stage: generate
39+
.fetch_targets:
40+
stage: fetch
1941
image: python:3.11-slim
2042
before_script:
2143
- pip install --quiet -r scripts/requirements.txt
2244
script:
23-
- echo "Generating pipeline for location: ${FILTER_LOCATION}"
24-
- python scripts/generate_pipeline.py
25-
- echo "--- Preview of generated pipeline ---"
26-
- cat "${OUTPUT_FILE}"
45+
- echo "Fetching targets for location: ${FILTER_LOCATION}"
46+
- python scripts/fetch_targets.py
47+
- echo "--- targets.json ---"
48+
- cat targets.json
2749
variables:
2850
# -------------------------------------------------------------------------
2951
# Registry database connection
3052
# Set REGISTRY_SERVER, REGISTRY_USER, REGISTRY_PASSWORD in
31-
# GitLab Settings → CI/CD → Variables (mark password as Protected + Masked)
53+
# GitLab Settings -> CI/CD -> Variables (password: Protected + Masked)
3254
# -------------------------------------------------------------------------
33-
REGISTRY_DATABASE: "flyway_registry"
34-
REGISTRY_PORT: "1433"
55+
REGISTRY_DATABASE: "flyway_registry"
56+
REGISTRY_PORT: "1433"
57+
SQL_SERVER_PORT: "1433"
58+
INCLUDE_REPLICAS: "1"
59+
OUTPUT_FILE: "targets.json"
3560

36-
# -------------------------------------------------------------------------
37-
# Regional runner tags
38-
# Override in GitLab Variables or at the top of your pipeline file
39-
# to match the tag names configured on your actual GitLab runners.
40-
# -------------------------------------------------------------------------
41-
RUNNER_TAG_LONDON: "runner-london"
42-
RUNNER_TAG_NEW_YORK: "runner-new-york"
43-
RUNNER_TAG_TOKYO: "runner-tokyo"
61+
# Subclasses must set this:
62+
# FILTER_LOCATION: all | London | New York | Tokyo
63+
artifacts:
64+
paths:
65+
- targets.json
66+
expire_in: never # keep so it can be reused across pipeline runs
67+
rules:
68+
- when: always
69+
70+
71+
# ---------------------------------------------------------------------------
72+
# Stage 2 generate
73+
# Reads targets.json and produces dynamic-pipeline.yml.
74+
# No database connection required.
75+
# ---------------------------------------------------------------------------
76+
.generate_pipeline:
77+
stage: generate
78+
image: python:3.11-slim
79+
before_script:
80+
- pip install --quiet -r scripts/requirements.txt
81+
script:
82+
- echo "Generating dynamic-pipeline.yml from targets.json"
83+
- python scripts/generate_pipeline.py
84+
- echo "--- dynamic-pipeline.yml ---"
85+
- cat dynamic-pipeline.yml
86+
variables:
87+
FLYWAY_LOCATIONS: "filesystem:./sql"
88+
TARGETS_FILE: "targets.json"
89+
OUTPUT_FILE: "dynamic-pipeline.yml"
4490

4591
# -------------------------------------------------------------------------
46-
# Target database defaults
47-
# Credentials are resolved at runtime from GitLab CI/CD variables
92+
# Regional runner tags
93+
# Override to match the tag names on your actual GitLab runners.
4894
# -------------------------------------------------------------------------
49-
SQL_SERVER_PORT: "1433"
50-
FLYWAY_LOCATIONS: "filesystem:./sql"
51-
INCLUDE_REPLICAS: "1"
52-
53-
# Subclasses must set these:
54-
# FILTER_LOCATION: all | London | New York | Tokyo
55-
# OUTPUT_FILE: generated-<name>.yml
95+
RUNNER_TAG_LONDON: "runner-london"
96+
RUNNER_TAG_NEW_YORK: "runner-new-york"
97+
RUNNER_TAG_TOKYO: "runner-tokyo"
5698
artifacts:
5799
paths:
58-
- "*.yml" # capture generated-*.yml files
100+
- dynamic-pipeline.yml
59101
expire_in: 2 hours
60102
rules:
61103
- when: always

0 commit comments

Comments
 (0)