economic-data-project/dagster.yaml at main · C00ldudeNoonan/economic-data-project · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
# Dagster instance configuration
# Uses DefaultRunLauncher (runs execute inside user code container)
# No Docker-in-Docker needed — works on Windows, Mac, and Linux

run_launcher:
  module: dagster.core.launcher
  class: DefaultRunLauncher

run_storage:
  module: dagster_postgres.run_storage
  class: PostgresRunStorage
  config:
    postgres_db:
      username:
        env: DAGSTER_POSTGRES_USER
      password:
        env: DAGSTER_POSTGRES_PASSWORD
      hostname:
        env: DAGSTER_POSTGRES_HOSTNAME
      db_name:
        env: DAGSTER_POSTGRES_DB
      port:
        env: DAGSTER_POSTGRES_PORT

event_log_storage:
  module: dagster_postgres.event_log
  class: PostgresEventLogStorage
  config:
    postgres_db:
      username:
        env: DAGSTER_POSTGRES_USER
      password:
        env: DAGSTER_POSTGRES_PASSWORD
      hostname:
        env: DAGSTER_POSTGRES_HOSTNAME
      db_name:
        env: DAGSTER_POSTGRES_DB
      port:
        env: DAGSTER_POSTGRES_PORT

schedule_storage:
  module: dagster_postgres.schedule_storage
  class: PostgresScheduleStorage
  config:
    postgres_db:
      username:
        env: DAGSTER_POSTGRES_USER
      password:
        env: DAGSTER_POSTGRES_PASSWORD
      hostname:
        env: DAGSTER_POSTGRES_HOSTNAME
      db_name:
        env: DAGSTER_POSTGRES_DB
      port:
        env: DAGSTER_POSTGRES_PORT

compute_logs:
  module: dagster.core.storage.local_compute_log_manager
  class: LocalComputeLogManager
  config:
    base_dir: /opt/dagster/dagster_home/storage/logs

local_artifact_storage:
  module: dagster.core.storage.root
  class: LocalArtifactStorage
  config:
    base_dir: /opt/dagster/dagster_home/storage/artifacts

python_logs:
  managed_python_loggers:
    - root
  python_log_level: INFO

# Run coordinator configuration
# QueuedRunCoordinator enables:
# - Parallel run execution with configurable limits
# - Run prioritization
# - Resource-aware scheduling
run_coordinator:
  module: dagster.core.run_coordinator
  class: QueuedRunCoordinator
  config:
    max_concurrent_runs: 4
    tag_concurrency_limits:
      - key: "dagster/backfill"
        limit: 2
      - key: "series_code"
        limit: 4
      - key: "job_type"
        value: "daily_ingestion"
        limit: 1
    dequeue_interval_seconds: 5

# Run monitoring — detect and fail hung runs
run_monitoring:
  enabled: true
  start_timeout_seconds: 600
  cancel_timeout_seconds: 300
  max_runtime_seconds: 7200  # 2 hours — kill runs that exceed this
  max_resume_run_attempts: 0  # DefaultRunLauncher doesn't support run resumption
  poll_interval_seconds: 60   # Faster detection of recovered container

# Run retries — automatically retry on worker crashes or asset/op failures
run_retries:
  max_retries: 2
  retry_on_asset_or_op_failure: true

# Tick retention (issue #132) — Dagster's supported mechanism for pruning the
# high-volume schedule/sensor tick tables. Successful ticks are kept so
# automation history stays auditable; skipped/failed ticks age out. This does
# NOT touch materialization/observation/asset-check state.
retention:
  schedule:
    purge_after_days: 90
  sensor:
    purge_after_days:
      skipped: 7
      failure: 30
      success: 90

telemetry:
  enabled: false