Skip to content

Commit 74604e7

Browse files
committed
fix(ops): wire Fivetran secrets and connection env on Cloud Run deploy
Mount FIVETRAN_API_KEY/SECRET from Secret Manager by default, pass connection IDs and ALLOW_WRITES at deploy time, and support optional min-instances. Correct dbt deployment.yml comment (GCS not Sheets).
1 parent e9f601b commit 74604e7

2 files changed

Lines changed: 9 additions & 2 deletions

File tree

backend/deployment/deploy.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
"ARTIFACT_BUCKET", "DRYLAB_PROVENANCE_BUCKET", "BQ_DATASET",
6060
"MEMORY_SERVICE_URI", "FIVETRAN_TRANSFORMATION_ID", "LAB_ENDPOINT_URL",
6161
"FIVETRAN_ACTIVATION_ID", "FIVETRAN_ACTIVATION_BASE_URL", "FIVETRAN_ACTIVATION_TARGET",
62+
"FIVETRAN_CONNECTION_ID", "FIVETRAN_SAMPLES_CONNECTION_ID", "FIVETRAN_ALLOW_WRITES",
6263
)
6364

6465
# Secret Manager refs injected as env at deploy: {ENV_VAR: secret_name}. SESSION_SERVICE_URI is pulled from a
@@ -75,6 +76,9 @@
7576
_activation_token_secret = os.environ.get("CENSUS_TOKEN_SECRET", "drylab-activation-token")
7677
if os.environ.get("WIRE_ACTIVATION_SECRET") == "1":
7778
_SECRET_REFS["FIVETRAN_ACTIVATION_TOKEN"] = _activation_token_secret
79+
if os.environ.get("WIRE_FIVETRAN_SECRETS", "1") != "0":
80+
_SECRET_REFS["FIVETRAN_API_KEY"] = os.environ.get("FIVETRAN_API_KEY_SECRET", "fivetran-api-key")
81+
_SECRET_REFS["FIVETRAN_API_SECRET"] = os.environ.get("FIVETRAN_API_SECRET_SECRET", "fivetran-api-secret")
7882

7983
_project_number_cache: dict[str, str] = {}
8084

@@ -148,6 +152,9 @@ def _commands() -> list[list[str]]:
148152
"--cpu", "2",
149153
"--timeout", "3600", # an SSE turn (full loop) can run minutes; default 300s would cut it off
150154
]
155+
min_inst = os.environ.get("CLOUD_RUN_MIN_INSTANCES")
156+
if min_inst:
157+
deploy += ["--min-instances", min_inst]
151158
if env_pairs:
152159
deploy += ["--set-env-vars", ",".join(env_pairs)]
153160
if secret_pairs:

transformations/dry_lab_dbt/deployment.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Fivetran Transformations for dbt Core — job + schedule (infrastructure-as-code for the transform edge).
22
# Docs: https://fivetran.com/docs/transformations/dbt/setup-guide/deployment-yaml-file
3-
# The "integrated" schedule runs this transform automatically AFTER the upstream Sheets connectors sync.
4-
# Fill the connection IDs once the Fivetran Google Sheets connectors exist (see docs/DATA_SPINE.md).
3+
# The "integrated" schedule runs this transform automatically AFTER the upstream GCS connectors sync.
4+
# Connection IDs below match the live Fivetran GCS → BigQuery spine (see docs/DATA_SPINE.md).
55
jobs:
66
- name: dry_lab_spine
77
schedule:

0 commit comments

Comments
 (0)