economic-data-project/dbt_project/models/sources.yml at fd2a31b2077e6fb7ae8fd1c8742b5118734a14ba · C00ldudeNoonan/economic-data-project · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
version: 2

sources:
  - name: staging
    database: "{{ env_var('BIGQUERY_PROJECT', env_var('BIGQUERY_PROJECT_ID', 'econ-data-project-478800')) }}"
    schema: economics_raw
    tables:
      # FRED Data
      - name: fred_raw
        config:
          loaded_at_field: date
          freshness:
            warn_after: {count: 7, period: day}
            error_after: {count: 14, period: day}

      # Financial Conditions Index
      - name: financial_conditions_index

      # Housing Data
      - name: housing_inventory_raw

      - name: housing_pulse_raw

      # Realtor.com Housing Data
      - name: realtor_country_raw
        description: Monthly country-level housing data from Realtor.com (via Google Drive)

      - name: realtor_state_raw
        description: Monthly state-level housing data from Realtor.com (via Google Drive)

      - name: realtor_metro_raw
        description: Monthly metro-level housing data from Realtor.com (via Google Drive)

      - name: realtor_county_raw
        description: Monthly county-level housing data from Realtor.com (via Google Drive)

      - name: realtor_zip_raw
        description: Monthly ZIP-level housing data from Realtor.com (via Google Drive)

      # Treasury Yields Data
      - name: treasury_yields_raw

      # MarketStack Data
      - name: us_sector_etfs_raw
        config:
          loaded_at_field: date
          freshness:
            warn_after: {count: 5, period: day}
            error_after: {count: 10, period: day}
      - name: currency_etfs_raw
        config:
          loaded_at_field: date
          freshness:
            warn_after: {count: 5, period: day}
            error_after: {count: 10, period: day}
      - name: commodity_etfs_raw
        config:
          loaded_at_field: date
          freshness:
            warn_after: {count: 5, period: day}
            error_after: {count: 10, period: day}
      - name: major_indices_raw
        config:
          loaded_at_field: date
          freshness:
            warn_after: {count: 5, period: day}
            error_after: {count: 10, period: day}
      - name: fixed_income_etfs_raw
        config:
          loaded_at_field: date
          freshness:
            warn_after: {count: 5, period: day}
            error_after: {count: 10, period: day}
      - name: global_markets_raw
        config:
          loaded_at_field: date
          freshness:
            warn_after: {count: 5, period: day}
            error_after: {count: 10, period: day}
      - name: sp500_companies_prices_raw
        config:
          loaded_at_field: date
          freshness:
            warn_after: {count: 5, period: day}
            error_after: {count: 10, period: day}
      - name: nasdaq_companies_prices_raw
        config:
          loaded_at_field: date
          freshness:
            warn_after: {count: 5, period: day}
            error_after: {count: 10, period: day}
      - name: energy_commodities_raw
        config:
          loaded_at_field: date
          freshness:
            warn_after: {count: 5, period: day}
            error_after: {count: 10, period: day}
      - name: input_commodities_raw
        config:
          loaded_at_field: date
          freshness:
            warn_after: {count: 5, period: day}
            error_after: {count: 10, period: day}
      - name: agriculture_commodities_raw
        config:
          loaded_at_field: date
          freshness:
            warn_after: {count: 5, period: day}
            error_after: {count: 10, period: day}

      # News Feed Data
      - name: reddit_posts_raw
        description: Raw Reddit posts from financial/economic subreddits scraped from old.reddit.com
      - name: reddit_post_content_raw
        description: Full post content (selftext, links) fetched per Reddit post
      - name: reddit_comments_raw
        description: Comments scraped per Reddit post from old.reddit.com
      - name: reddit_content_embeddings
        description: nomic-embed-text embeddings for Reddit post text and comment text
      - name: reddit_sentiment_scored
        description: VADER sentiment scores for Reddit posts and comments
      - name: reddit_ticker_mentions
        description: Stock ticker mentions ($AAPL etc.) extracted from Reddit posts and comments

      - name: fomc_minutes_metadata
        description: Metadata for FOMC meeting minutes stored in GCS (full documents in gs://bucket/fomc/)

      # FOMC Transcript Data
      - name: fomc_transcripts
        description: Raw FOMC meeting transcripts with full text

      - name: transcript_sections
        description: Parsed transcript sections with speaker attribution

      - name: fomc_meeting_summaries
        description: AI-generated summaries of FOMC meetings

      - name: transcript_topics
        description: Topics and sentiment extracted from transcripts

      - name: transcript_search_index
        description: Full-text search index for transcripts

      - name: member_voting_history
        description: FOMC member voting records and positions

      - name: fomc_meetings_enhanced
        description: Enhanced FOMC meeting calendar with transcript availability

      - name: fomc_sentiment_scores
        description: Hawkish/dovish sentiment scores for FOMC transcript sections

      # Events Calendar Data
      - name: economic_calendar
        description: Economic calendar events from Fair Economy CDN

      - name: earnings_calendar
        description: Earnings announcements from Financial Modeling Prep API

      # Stock Splits
      - name: sp500_splits_raw
        description: Stock split history from MarketStack /splits API for S&P 500 companies

      # Company Metadata
      - name: sp500_companies_raw
        description: >
          S&P 500 company metadata from Wikipedia with SCD Type 2 history.
          Active constituents have date_ended IS NULL. Use stg_sp500_companies_active
          for current members only.