From e794ae08ff12f0388b611335620df9a4f7f3491b Mon Sep 17 00:00:00 2001 From: Alex Noonan <48368867+C00ldudeNoonan@users.noreply.github.com> Date: Thu, 11 Jun 2026 12:31:21 -0400 Subject: [PATCH 1/4] feat: prepare dbt platform fusion orchestration --- .env.example | 16 + .github/workflows/dagster-ci.yml | 23 +- dbt_project/.gitignore | 3 +- dbt_project/catalogs.yml | 10 +- dbt_project/models/commodities/schema.yml | 33 +- dbt_project/models/markets/schema.yml | 66 ++-- dbt_project/models/sources.yml | 101 +++--- .../models/staging/telemetry/sources.yml | 2 +- dbt_project/profiles.yml | 6 +- docs/README.md | 1 + docs/dbt_project/README.md | 6 + .../dbt-platform-fusion-onboarding.md | 163 +++++++++ .../defs/transformation/__init__.py | 14 +- .../macro_agents/defs/transformation/dbt.py | 332 ++++++++++-------- 14 files changed, 540 insertions(+), 236 deletions(-) create mode 100644 docs/dbt_project/dbt-platform-fusion-onboarding.md diff --git a/.env.example b/.env.example index d8daaff..3f8a56e 100644 --- a/.env.example +++ b/.env.example @@ -97,6 +97,18 @@ REALTOR_GDRIVE_FOLDER_ID=your_realtor_folder_id_here # DBT # ============================================================================= DBT_TARGET=dev # dev, local, or prod + +# dbt Platform orchestration from Dagster +# Leave DBT_ORCHESTRATION_MODE=local to run dbt through DbtCliResource. +# Set to dbt_platform after the dbt Platform project/environment is configured. +DBT_ORCHESTRATION_MODE=local +DBT_PLATFORM_ACCOUNT_ID=your-dbt-platform-account-id +DBT_PLATFORM_PROJECT_ID=your-dbt-platform-project-id +DBT_PLATFORM_ENVIRONMENT_ID=your-dbt-platform-environment-id +DBT_PLATFORM_TOKEN=your-dbt-platform-service-token +DBT_PLATFORM_ACCESS_URL=https://cloud.getdbt.com +DBT_PLATFORM_ADHOC_JOB_NAME=economic-data-dagster-adhoc +DBT_PLATFORM_POLL_INTERVAL_SECONDS=60 # DBT_PROJECT_DIR=/path/to/dbt_project # Optional, auto-detected # ============================================================================= @@ -144,3 +156,7 @@ GMAIL_SENDER=you@gmail.com GMAIL_OAUTH_TOKEN_PATH=/path/to/gmail_token.json # Recipient inbox for alert emails (same as sender for self-alerts). ALERT_RECIPIENT=you@gmail.com +DBT_CLOUD_API_TOKEN= +DBT_CLOUD_ACCOUNT= +DBT_CLOUD_PROJECT_ID= +DBT_CLOUD_ENVIRONMENT_ID= diff --git a/.github/workflows/dagster-ci.yml b/.github/workflows/dagster-ci.yml index 44f39f0..43a3fb8 100644 --- a/.github/workflows/dagster-ci.yml +++ b/.github/workflows/dagster-ci.yml @@ -4,11 +4,13 @@ on: push: branches: [main, develop] paths: + - ".github/workflows/dagster-ci.yml" - "macro_agents/**" - "dbt_project/**" pull_request: branches: [main, develop] paths: + - ".github/workflows/dagster-ci.yml" - "macro_agents/**" - "dbt_project/**" @@ -16,6 +18,13 @@ jobs: test: name: Test Dagster runs-on: ubuntu-latest + env: + DBT_PLATFORM_ACCOUNT_ID: ${{ secrets.DBT_PLATFORM_ACCOUNT_ID }} + DBT_PLATFORM_PROJECT_ID: ${{ secrets.DBT_PLATFORM_PROJECT_ID }} + DBT_PLATFORM_ENVIRONMENT_ID: ${{ secrets.DBT_PLATFORM_ENVIRONMENT_ID }} + DBT_PLATFORM_TOKEN: ${{ secrets.DBT_PLATFORM_TOKEN }} + DBT_PLATFORM_ACCESS_URL: ${{ vars.DBT_PLATFORM_ACCESS_URL || 'https://cloud.getdbt.com' }} + DBT_PLATFORM_ADHOC_JOB_NAME: ${{ vars.DBT_PLATFORM_ADHOC_JOB_NAME || 'economic-data-dagster-ci' }} steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 @@ -54,13 +63,25 @@ jobs: cd macro_agents uv run ty check --extra-search-path src src/macro_agents - - name: Test Dagster definitions + - name: Test Dagster definitions with dbt Platform + if: ${{ env.DBT_PLATFORM_TOKEN != '' }} env: BIGQUERY_PROJECT: econ-data-project-478800 + DBT_ORCHESTRATION_MODE: dbt_platform run: | cd macro_agents uv run dg check defs + - name: Test Dagster definitions with local dbt fallback + if: ${{ env.DBT_PLATFORM_TOKEN == '' }} + env: + BIGQUERY_PROJECT: econ-data-project-478800 + DBT_ORCHESTRATION_MODE: local + run: | + echo "DBT_PLATFORM_TOKEN is not configured; using local dbt parse fallback." + cd macro_agents + uv run dg check defs + - name: Test with pytest run: | cd macro_agents diff --git a/dbt_project/.gitignore b/dbt_project/.gitignore index c730168..5207900 100644 --- a/dbt_project/.gitignore +++ b/dbt_project/.gitignore @@ -1,4 +1,5 @@ /target /logs .user.yml -/dbt_packages \ No newline at end of file +/dbt_packages +dbt_internal_packages/ diff --git a/dbt_project/catalogs.yml b/dbt_project/catalogs.yml index b4ae84a..48fb07a 100644 --- a/dbt_project/catalogs.yml +++ b/dbt_project/catalogs.yml @@ -1,11 +1,9 @@ -version: 2 - catalogs: - name: iceberg_catalog + active_write_integration: biglake write_integrations: - name: biglake catalog_type: biglake_metastore - adapter_properties: - connection_id: "projects/{{ env_var('BIGQUERY_PROJECT') }}/locations/{{ env_var('BIGQUERY_LOCATION', 'US') }}/connections/biglake-iceberg" - gcs_bucket: "{{ env_var('ICEBERG_BUCKET_NAME', 'econ-project-iceberg-data') }}" - gcs_path_prefix: "iceberg" + table_format: iceberg + file_format: parquet + external_volume: "gs://{{ env_var('ICEBERG_BUCKET_NAME', 'econ-project-iceberg-data') }}" diff --git a/dbt_project/models/commodities/schema.yml b/dbt_project/models/commodities/schema.yml index 9e486af..a40d519 100644 --- a/dbt_project/models/commodities/schema.yml +++ b/dbt_project/models/commodities/schema.yml @@ -5,7 +5,8 @@ models: description: "Summary statistics for energy commodities across different time periods" tests: - unique_combination: - combination_of_columns: ['commodity_name', 'time_period'] + arguments: + combination_of_columns: ['commodity_name', 'time_period'] columns: - name: commodity_name description: "Name of the commodity" @@ -28,19 +29,22 @@ models: description: "Annualized volatility percentage" tests: - value_in_range: - min_value: 0 + arguments: + min_value: 0 - name: win_rate_pct description: "Percentage of days with positive price changes" tests: - value_in_range: - min_value: 0 - max_value: 100 + arguments: + min_value: 0 + max_value: 100 - name: input_commodities_summary description: "Summary statistics for input/industrial commodities across different time periods" tests: - unique_combination: - combination_of_columns: ['commodity_name', 'time_period'] + arguments: + combination_of_columns: ['commodity_name', 'time_period'] columns: - name: commodity_name description: "Name of the commodity" @@ -63,19 +67,22 @@ models: description: "Annualized volatility percentage" tests: - value_in_range: - min_value: 0 + arguments: + min_value: 0 - name: win_rate_pct description: "Percentage of days with positive price changes" tests: - value_in_range: - min_value: 0 - max_value: 100 + arguments: + min_value: 0 + max_value: 100 - name: agriculture_commodities_summary description: "Summary statistics for agricultural commodities across different time periods" tests: - unique_combination: - combination_of_columns: ['commodity_name', 'time_period'] + arguments: + combination_of_columns: ['commodity_name', 'time_period'] columns: - name: commodity_name description: "Name of the commodity" @@ -98,13 +105,15 @@ models: description: "Annualized volatility percentage" tests: - value_in_range: - min_value: 0 + arguments: + min_value: 0 - name: win_rate_pct description: "Percentage of days with positive price changes" tests: - value_in_range: - min_value: 0 - max_value: 100 + arguments: + min_value: 0 + max_value: 100 - name: energy_commodities_analysis_return description: > diff --git a/dbt_project/models/markets/schema.yml b/dbt_project/models/markets/schema.yml index 94a64e5..186e049 100644 --- a/dbt_project/models/markets/schema.yml +++ b/dbt_project/models/markets/schema.yml @@ -10,7 +10,8 @@ models: trends and risk characteristics. tests: - unique_combination: - combination_of_columns: ['symbol', 'time_period'] + arguments: + combination_of_columns: ['symbol', 'time_period'] columns: - name: symbol tests: @@ -24,12 +25,14 @@ models: - name: volatility_pct tests: - value_in_range: - min_value: 0 + arguments: + min_value: 0 - name: win_rate_pct tests: - value_in_range: - min_value: 0 - max_value: 100 + arguments: + min_value: 0 + max_value: 100 - name: global_markets_summary description: > @@ -40,7 +43,8 @@ models: of global market performance and risk profiles. tests: - unique_combination: - combination_of_columns: ['symbol', 'time_period'] + arguments: + combination_of_columns: ['symbol', 'time_period'] columns: - name: symbol tests: @@ -54,12 +58,14 @@ models: - name: volatility_pct tests: - value_in_range: - min_value: 0 + arguments: + min_value: 0 - name: win_rate_pct tests: - value_in_range: - min_value: 0 - max_value: 100 + arguments: + min_value: 0 + max_value: 100 - name: major_indicies_summary description: > @@ -70,7 +76,8 @@ models: and 5 year periods. Provides comprehensive market performance insights. tests: - unique_combination: - combination_of_columns: ['symbol', 'time_period'] + arguments: + combination_of_columns: ['symbol', 'time_period'] columns: - name: symbol tests: @@ -84,12 +91,14 @@ models: - name: volatility_pct tests: - value_in_range: - min_value: 0 + arguments: + min_value: 0 - name: win_rate_pct tests: - value_in_range: - min_value: 0 - max_value: 100 + arguments: + min_value: 0 + max_value: 100 - name: us_sector_summary description: > @@ -100,7 +109,8 @@ models: analysis and relative performance comparison across different market sectors. tests: - unique_combination: - combination_of_columns: ['symbol', 'time_period'] + arguments: + combination_of_columns: ['symbol', 'time_period'] columns: - name: symbol tests: @@ -114,12 +124,14 @@ models: - name: volatility_pct tests: - value_in_range: - min_value: 0 + arguments: + min_value: 0 - name: win_rate_pct tests: - value_in_range: - min_value: 0 - max_value: 100 + arguments: + min_value: 0 + max_value: 100 - name: sp500_companies_summary description: > @@ -129,7 +141,8 @@ models: periods for S&P 500 constituents. tests: - unique_combination: - combination_of_columns: ['symbol', 'time_period'] + arguments: + combination_of_columns: ['symbol', 'time_period'] columns: - name: symbol tests: @@ -143,12 +156,14 @@ models: - name: volatility_pct tests: - value_in_range: - min_value: 0 + arguments: + min_value: 0 - name: win_rate_pct tests: - value_in_range: - min_value: 0 - max_value: 100 + arguments: + min_value: 0 + max_value: 100 - name: nasdaq_companies_summary description: > @@ -158,7 +173,8 @@ models: periods for NASDAQ-listed companies. tests: - unique_combination: - combination_of_columns: ['symbol', 'time_period'] + arguments: + combination_of_columns: ['symbol', 'time_period'] columns: - name: symbol tests: @@ -172,12 +188,14 @@ models: - name: volatility_pct tests: - value_in_range: - min_value: 0 + arguments: + min_value: 0 - name: win_rate_pct tests: - value_in_range: - min_value: 0 - max_value: 100 + arguments: + min_value: 0 + max_value: 100 - name: currency_analysis_return description: > diff --git a/dbt_project/models/sources.yml b/dbt_project/models/sources.yml index 265181c..11d09f5 100644 --- a/dbt_project/models/sources.yml +++ b/dbt_project/models/sources.yml @@ -2,15 +2,16 @@ version: 2 sources: - name: staging - database: "{{ env_var('BIGQUERY_PROJECT') }}" + database: "{{ env_var('BIGQUERY_PROJECT', env_var('BIGQUERY_PROJECT_ID', 'econ-data-project-478800')) }}" schema: economics_raw tables: # FRED Data - name: fred_raw - loaded_at_field: date - freshness: - warn_after: {count: 7, period: day} - error_after: {count: 14, period: day} + config: + loaded_at_field: date + freshness: + warn_after: {count: 7, period: day} + error_after: {count: 14, period: day} # Financial Conditions Index - name: financial_conditions_index @@ -41,55 +42,65 @@ sources: # MarketStack Data - name: us_sector_etfs_raw - loaded_at_field: date - freshness: - warn_after: {count: 5, period: day} - error_after: {count: 10, period: day} + config: + loaded_at_field: date + freshness: + warn_after: {count: 5, period: day} + error_after: {count: 10, period: day} - name: currency_etfs_raw - loaded_at_field: date - freshness: - warn_after: {count: 5, period: day} - error_after: {count: 10, period: day} + config: + loaded_at_field: date + freshness: + warn_after: {count: 5, period: day} + error_after: {count: 10, period: day} - name: major_indices_raw - loaded_at_field: date - freshness: - warn_after: {count: 5, period: day} - error_after: {count: 10, period: day} + config: + loaded_at_field: date + freshness: + warn_after: {count: 5, period: day} + error_after: {count: 10, period: day} - name: fixed_income_etfs_raw - loaded_at_field: date - freshness: - warn_after: {count: 5, period: day} - error_after: {count: 10, period: day} + config: + loaded_at_field: date + freshness: + warn_after: {count: 5, period: day} + error_after: {count: 10, period: day} - name: global_markets_raw - loaded_at_field: date - freshness: - warn_after: {count: 5, period: day} - error_after: {count: 10, period: day} + config: + loaded_at_field: date + freshness: + warn_after: {count: 5, period: day} + error_after: {count: 10, period: day} - name: sp500_companies_prices_raw - loaded_at_field: date - freshness: - warn_after: {count: 5, period: day} - error_after: {count: 10, period: day} + config: + loaded_at_field: date + freshness: + warn_after: {count: 5, period: day} + error_after: {count: 10, period: day} - name: nasdaq_companies_prices_raw - loaded_at_field: date - freshness: - warn_after: {count: 5, period: day} - error_after: {count: 10, period: day} + config: + loaded_at_field: date + freshness: + warn_after: {count: 5, period: day} + error_after: {count: 10, period: day} - name: energy_commodities_raw - loaded_at_field: date - freshness: - warn_after: {count: 5, period: day} - error_after: {count: 10, period: day} + config: + loaded_at_field: date + freshness: + warn_after: {count: 5, period: day} + error_after: {count: 10, period: day} - name: input_commodities_raw - loaded_at_field: date - freshness: - warn_after: {count: 5, period: day} - error_after: {count: 10, period: day} + config: + loaded_at_field: date + freshness: + warn_after: {count: 5, period: day} + error_after: {count: 10, period: day} - name: agriculture_commodities_raw - loaded_at_field: date - freshness: - warn_after: {count: 5, period: day} - error_after: {count: 10, period: day} + config: + loaded_at_field: date + freshness: + warn_after: {count: 5, period: day} + error_after: {count: 10, period: day} # News Feed Data - name: reddit_posts_raw diff --git a/dbt_project/models/staging/telemetry/sources.yml b/dbt_project/models/staging/telemetry/sources.yml index bcbb32b..a9072e1 100644 --- a/dbt_project/models/staging/telemetry/sources.yml +++ b/dbt_project/models/staging/telemetry/sources.yml @@ -3,7 +3,7 @@ version: 2 sources: - name: raw_telemetry description: Raw telemetry data - database: "{{ env_var('BIGQUERY_PROJECT') }}" + database: "{{ env_var('BIGQUERY_PROJECT', env_var('BIGQUERY_PROJECT_ID', 'econ-data-project-478800')) }}" schema: economics_raw tables: - name: telemetry_events_raw diff --git a/dbt_project/profiles.yml b/dbt_project/profiles.yml index 7b539f0..1df59cc 100644 --- a/dbt_project/profiles.yml +++ b/dbt_project/profiles.yml @@ -4,7 +4,7 @@ econ_database: dev: type: bigquery method: oauth - project: "{{ env_var('BIGQUERY_PROJECT', 'econ-data-project-478800') }}" + project: "{{ env_var('BIGQUERY_PROJECT', env_var('BIGQUERY_PROJECT_ID', 'econ-data-project-478800')) }}" # Models land in *_dev datasets: economics_staging_dev, economics_marts_dev, etc. # The generate_schema_name macro appends the _dev suffix automatically. dataset: economics_staging_dev @@ -15,7 +15,7 @@ econ_database: staging: type: bigquery method: oauth - project: "{{ env_var('BIGQUERY_PROJECT', 'econ-data-project-478800') }}" + project: "{{ env_var('BIGQUERY_PROJECT', env_var('BIGQUERY_PROJECT_ID', 'econ-data-project-478800')) }}" # Models land in *_staging datasets for pre-prod validation. dataset: economics_staging_staging threads: 6 @@ -28,7 +28,7 @@ econ_database: # and bigquery.jobUser on the project (provisioned in Terraform Phase 0). type: bigquery method: oauth - project: "{{ env_var('BIGQUERY_PROJECT', 'econ-data-project-478800') }}" + project: "{{ env_var('BIGQUERY_PROJECT', env_var('BIGQUERY_PROJECT_ID', 'econ-data-project-478800')) }}" # Models land in canonical datasets: economics_staging, economics_marts, etc. dataset: economics_staging threads: 8 diff --git a/docs/README.md b/docs/README.md index bd14e7d..43f0853 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,7 @@ A data platform that ingests, transforms, and analyzes U.S. economic and financi | [Data Platform Overview](./architecture/data-platform-overview.md) | High-level architecture | | [Dagster Pipeline](./macro_agents/README.md) | Data orchestration documentation | | [dbt Models](./dbt_project/README.md) | SQL transformation documentation | +| [dbt Platform + Fusion Onboarding](./dbt_project/dbt-platform-fusion-onboarding.md) | Issue #82 setup runbook | | [GCP Deployment](./GCP_DEPLOYMENT_GUIDE.md) | Cloud deployment guide | ## System Architecture diff --git a/docs/dbt_project/README.md b/docs/dbt_project/README.md index 8a6ad1d..e8689fb 100644 --- a/docs/dbt_project/README.md +++ b/docs/dbt_project/README.md @@ -2,6 +2,12 @@ The `dbt_project/` module contains SQL-based data transformations that process raw data into analytical models. +## dbt Platform + Fusion + +The project is migrating to dbt Platform with the Fusion engine on BigQuery. +Use the [dbt Platform + Fusion onboarding runbook](./dbt-platform-fusion-onboarding.md) +for issue #82 setup, environment configuration, CI jobs, and Dagster cutover notes. + ## Architecture ```mermaid diff --git a/docs/dbt_project/dbt-platform-fusion-onboarding.md b/docs/dbt_project/dbt-platform-fusion-onboarding.md new file mode 100644 index 0000000..7de9bbd --- /dev/null +++ b/docs/dbt_project/dbt-platform-fusion-onboarding.md @@ -0,0 +1,163 @@ +# dbt Platform + Fusion Onboarding + +Issue: [#82](https://github.com/C00ldudeNoonan/economic-data-project/issues/82) + +This runbook captures the repo-side and dbt Platform setup required to run the +BigQuery dbt project on the Fusion engine. + +## Current State + +- Warehouse: BigQuery. +- dbt engine: Fusion-compatible project metadata parses locally. +- Project path: `dbt_project/`. +- Profile: `econ_database`. +- Model count validated by Wizard index: 132 models, 2 seeds, 41 sources, 385 tests. +- Iceberg support: `catalogs.yml` defines the `iceberg_catalog` BigLake write integration. + +## dbt Platform Project + +Create one dbt Platform project connected to this GitHub repository: + +| Setting | Value | +| --- | --- | +| Repository | `C00ldudeNoonan/economic-data-project` | +| Project subdirectory | `dbt_project` | +| Default branch | `main` | +| Engine | Fusion | +| Adapter | BigQuery | + +## BigQuery Connection + +Configure a BigQuery connection with a service account that can create jobs and +write to the economics datasets. + +Required permissions: + +- `roles/bigquery.jobUser` on the project. +- `roles/bigquery.dataEditor` on target datasets. +- BigLake/Iceberg permissions for the GCS bucket used by `ICEBERG_BUCKET_NAME`. + +Required environment variables: + +| Variable | Purpose | +| --- | --- | +| `BIGQUERY_PROJECT` | BigQuery project id. Falls back to `BIGQUERY_PROJECT_ID` locally. | +| `BIGQUERY_LOCATION` | BigQuery location, default `US`. | +| `ICEBERG_BUCKET_NAME` | GCS bucket backing BigLake Iceberg tables. | +| `DBT_TARGET` | `dev`, `staging`, or `prod`. | + +## Environments + +Configure three environments: + +| Environment | Target | Purpose | +| --- | --- | --- | +| Development | `dev` | Developer validation with `_dev` dataset suffixes. | +| Staging | `staging` | Pre-production validation with `_staging` dataset suffixes. | +| Production | `prod` | Scheduled production builds in canonical datasets. | + +## Jobs + +Recommended dbt Platform jobs: + +| Job | Environment | Trigger | Command | +| --- | --- | --- | --- | +| PR Check | Development | Pull request | `dbt parse && dbt build --select state:modified+ --defer --state prod` | +| Production Build | Production | Schedule | `dbt build` | +| Source Freshness | Production | Schedule before build | `dbt source freshness` | + +The PR job should use deferral against the latest production state so pull +requests only build modified nodes and their downstream dependents. + +## CI/CD Ownership + +dbt Platform owns dbt-specific CI/CD. GitHub Actions should validate Python, +Dagster code loading, and unit tests, but should not be the primary place where +dbt builds run. + +| System | Responsibility | +| --- | --- | +| dbt Platform CI | Pull-request dbt validation with Fusion, deferral, and BigQuery credentials. | +| dbt Platform scheduled jobs | Production `dbt build` and `dbt source freshness`. | +| GitHub Actions | Ruff, typecheck, pytest, and Dagster definition loading. | +| Dagster | Asset orchestration and optional dbt Platform ad hoc run triggering/observation. | + +GitHub branch protection should require: + +- dbt Platform PR check for modified dbt assets. +- `Dagster CI / Test Dagster`. + +GitHub Actions should have these repository secrets so `dg check defs` can load +dbt asset specs from dbt Platform instead of compiling the local dbt project: + +| GitHub secret/variable | Purpose | +| --- | --- | +| `DBT_PLATFORM_ACCOUNT_ID` | dbt Platform account id. | +| `DBT_PLATFORM_PROJECT_ID` | dbt Platform project id. | +| `DBT_PLATFORM_ENVIRONMENT_ID` | dbt Platform environment id for CI/Dagster ad hoc runs. | +| `DBT_PLATFORM_TOKEN` | dbt Platform service token. | +| `DBT_PLATFORM_ACCESS_URL` | Optional repository variable; defaults to `https://cloud.getdbt.com`. | +| `DBT_PLATFORM_ADHOC_JOB_NAME` | Optional repository variable; defaults to `economic-data-dagster-ci`. | + +If `DBT_PLATFORM_TOKEN` is not available, the GitHub workflow falls back to +`DBT_ORCHESTRATION_MODE=local` so forks and local smoke checks can still validate +Dagster definitions without dbt Platform credentials. + +## Fusion Compatibility Results + +Local Wizard validation after issue setup: + +- `dbt parse` succeeds with only `dbt1700: --use-colors is no longer supported`. +- `dbt compile` reaches BigQuery adapter execution, then stops without local ADC credentials. +- YAML schema blockers were fixed by: + - moving source freshness settings under `config:`; + - moving generic test parameters under `arguments:`; + - updating `catalogs.yml` to the Fusion BigLake catalog schema. + +## Dagster Orchestration Decision + +Keep Dagster as the asset orchestrator. The Dagster integration supports two +execution modes: + +| Mode | `DBT_ORCHESTRATION_MODE` | Behavior | +| --- | --- | --- | +| Local CLI | `local` | Uses `DbtCliResource` and the bundled `dbt_project/`. | +| dbt Platform | `dbt_platform` | Uses `DbtCloudWorkspace` to load dbt Platform artifacts and launch ad hoc dbt Platform runs. | + +Configure these Dagster environment variables after the dbt Platform project and +environment exist: + +| Variable | Purpose | +| --- | --- | +| `DBT_ORCHESTRATION_MODE` | Set to `dbt_platform` to switch Dagster from local dbt CLI execution to dbt Platform. | +| `DBT_PLATFORM_ACCOUNT_ID` | dbt Platform account id. | +| `DBT_PLATFORM_PROJECT_ID` | dbt Platform project id. | +| `DBT_PLATFORM_ENVIRONMENT_ID` | dbt Platform environment id Dagster should use for ad hoc runs. | +| `DBT_PLATFORM_TOKEN` | dbt Platform service token. | +| `DBT_PLATFORM_ACCESS_URL` | dbt Platform URL, default `https://cloud.getdbt.com`. | +| `DBT_PLATFORM_ADHOC_JOB_NAME` | Optional ad hoc job name Dagster creates/uses in dbt Platform. | +| `DBT_PLATFORM_POLL_INTERVAL_SECONDS` | Optional polling interval for external dbt Platform run observations. | + +Cutover path: + +1. Keep current `DbtCliResource` orchestration for local and deployment continuity. +2. Create dbt Platform jobs and confirm production artifacts are available. +3. Set `DBT_ORCHESTRATION_MODE=dbt_platform` and configure the `DBT_PLATFORM_*` variables in Dagster. +4. Validate Dagster code location load; Dagster should fetch the manifest from dbt Platform. +5. Materialize a small dbt asset selection from Dagster to confirm ad hoc dbt Platform runs work. +6. Remove local dbt project bundling and `dbt deps` fallback after the API path is stable. + +## Remaining Manual Checklist + +- [ ] Create dbt Platform project. +- [ ] Connect GitHub repository. +- [ ] Configure BigQuery service account connection. +- [ ] Create development, staging, and production environments. +- [ ] Add `BIGQUERY_PROJECT`, `BIGQUERY_LOCATION`, `ICEBERG_BUCKET_NAME`, and `DBT_TARGET` variables. +- [ ] Add `DBT_ORCHESTRATION_MODE=dbt_platform` and the required `DBT_PLATFORM_*` variables to Dagster. +- [ ] Import and validate `catalogs.yml`. +- [ ] Configure PR, production build, and source freshness jobs. +- [ ] Add dbt Platform PR check and `Dagster CI / Test Dagster` to GitHub branch protection. +- [ ] Add dbt Platform GitHub repository secrets/variables for Dagster CI. +- [ ] Decide final Dagster orchestration cutover date. +- [ ] Validate Dagster can load dbt Platform asset specs and trigger an ad hoc run. diff --git a/macro_agents/src/macro_agents/defs/transformation/__init__.py b/macro_agents/src/macro_agents/defs/transformation/__init__.py index 2b278ab..ddec9ba 100644 --- a/macro_agents/src/macro_agents/defs/transformation/__init__.py +++ b/macro_agents/src/macro_agents/defs/transformation/__init__.py @@ -1,6 +1,10 @@ import dagster as dg -from macro_agents.defs.transformation.dbt import dbt_cli_resource, full_dbt_assets +from macro_agents.defs.transformation.dbt import ( + dbt_cloud_polling_sensor, + dbt_resource, + full_dbt_assets, +) from macro_agents.defs.transformation.checks import transformation_checks from macro_agents.defs.transformation.financial_condition_index import ( fci_weights_config, @@ -100,6 +104,11 @@ ) +transformation_sensors = [] +if dbt_cloud_polling_sensor is not None: + transformation_sensors.append(dbt_cloud_polling_sensor) + + defs = dg.Definitions( assets=[full_dbt_assets, financial_conditions_index, fci_weights_config], asset_checks=transformation_checks, @@ -116,5 +125,6 @@ dbt_data_quality_models_job, dbt_backtesting_models_job, ], - resources={"dbt": dbt_cli_resource}, + resources={"dbt": dbt_resource}, + sensors=transformation_sensors, ) diff --git a/macro_agents/src/macro_agents/defs/transformation/dbt.py b/macro_agents/src/macro_agents/defs/transformation/dbt.py index ec82a33..756399f 100644 --- a/macro_agents/src/macro_agents/defs/transformation/dbt.py +++ b/macro_agents/src/macro_agents/defs/transformation/dbt.py @@ -10,9 +10,19 @@ from typing import Any import dagster as dg -from dagster_dbt import DagsterDbtTranslator, DbtCliResource, DbtProject, dbt_assets +from dagster_dbt import ( + DagsterDbtTranslator, + DbtCliResource, + DbtCloudCredentials, + DbtCloudWorkspace, + DbtProject, + build_dbt_cloud_polling_sensor, + dbt_assets, + dbt_cloud_assets, +) logging.getLogger("dagster_dbt").setLevel(logging.DEBUG) +logger = logging.getLogger("dagster_dbt") class CustomizedDagsterDbtTranslator(DagsterDbtTranslator): @@ -37,164 +47,204 @@ def get_automation_condition( environment = os.getenv("DBT_TARGET", "dev") +orchestration_mode = os.getenv("DBT_ORCHESTRATION_MODE", "local").lower() -dbt_project_dir = os.getenv("DBT_PROJECT_DIR") -if dbt_project_dir: - dbt_project_dir = Path(dbt_project_dir).resolve() - if not dbt_project_dir.exists(): - raise FileNotFoundError( - "DBT_PROJECT_DIR environment variable points to a non-existent path." - ) -else: - current_file = Path(__file__).resolve() - # Try to find macro_agents package root (where dbt_project is copied during deployment) - # Go up from: macro_agents/defs/transformation/dbt.py -> macro_agents/defs/transformation -> macro_agents/defs -> macro_agents - macro_agents_root = current_file.parent.parent.parent - repo_root = current_file.parent.parent.parent.parent.parent - cwd = Path.cwd() - - # In Dagster Cloud, the working_directory is set to ./macro_agents - # The dbt_project should be copied there during deployment - # The working directory at runtime is typically working_directory/root - # So dbt_project should be at working_directory/root/dbt_project (i.e., cwd/dbt_project) - possible_dbt_project_paths = [ - # First priority: current working directory (Dagster Cloud runtime: working_directory/root) - # This is where the working_directory contents are extracted - cwd / "dbt_project", - # Second: if we're in a "root" subdirectory, check parent - cwd.parent / "dbt_project" if cwd.name == "root" else None, - # Third: check if dbt_project is in the parent of root (working_directory level) - cwd.parent.parent / "dbt_project" if cwd.parent.name == "root" else None, - # Fourth: check relative to macro_agents package location (if bundled in wheel) - macro_agents_root / "dbt_project", - # Fifth: check if dbt_project is alongside the package - macro_agents_root.parent / "dbt_project", - # Sixth: check in macro_agents subdirectory (if working directory is repo root) - cwd / "macro_agents" / "dbt_project", - # Seventh: check in repo root (fallback) - repo_root / "dbt_project", - # Eighth: check parent of working directory - cwd.parent / "dbt_project", - ] - - # Filter out None values - possible_dbt_project_paths = [ - p for p in possible_dbt_project_paths if p is not None - ] - - dbt_project_dir = None - for path in possible_dbt_project_paths: - try: - abs_path = path.resolve() - if abs_path.exists() and abs_path.is_dir(): - # Verify it's actually a dbt project by checking for dbt_project.yml - if (abs_path / "dbt_project.yml").exists() or ( - abs_path / "dbt_project.yaml" - ).exists(): - dbt_project_dir = abs_path - break - except (OSError, RuntimeError): - # Skip paths that can't be resolved (e.g., broken symlinks) - continue - if dbt_project_dir is None: - tried_paths = [] - for p in possible_dbt_project_paths: +def _first_env(*names: str) -> str | None: + for name in names: + value = os.getenv(name) + if value: + return value + return None + + +def _required_int_env(*names: str) -> int: + value = _first_env(*names) + if value is None: + joined_names = " or ".join(names) + raise RuntimeError(f"Set {joined_names} to enable dbt Platform orchestration.") + try: + return int(value) + except ValueError as error: + joined_names = " or ".join(names) + raise RuntimeError(f"{joined_names} must be an integer.") from error + + +def _required_env_var(*names: str) -> dg.EnvVar: + for name in names: + if os.getenv(name): + return dg.EnvVar(name) + else: + joined_names = " or ".join(names) + raise RuntimeError(f"Set {joined_names} to enable dbt Platform orchestration.") + +if orchestration_mode in {"dbt_platform", "dbt_cloud", "platform", "cloud"}: + dbt_cloud_credentials = DbtCloudCredentials( + account_id=_required_int_env( + "DBT_PLATFORM_ACCOUNT_ID", "DBT_CLOUD_ACCOUNT_ID" + ), + token=_required_env_var("DBT_PLATFORM_TOKEN", "DBT_CLOUD_API_TOKEN"), + access_url=_first_env("DBT_PLATFORM_ACCESS_URL", "DBT_CLOUD_ACCESS_URL") + or "https://cloud.getdbt.com", + ) + + dbt_resource = DbtCloudWorkspace( + credentials=dbt_cloud_credentials, + project_id=_required_int_env( + "DBT_PLATFORM_PROJECT_ID", "DBT_CLOUD_PROJECT_ID" + ), + environment_id=_required_int_env( + "DBT_PLATFORM_ENVIRONMENT_ID", "DBT_CLOUD_ENVIRONMENT_ID" + ), + adhoc_job_name=_first_env( + "DBT_PLATFORM_ADHOC_JOB_NAME", "DBT_CLOUD_ADHOC_JOB_NAME" + ), + ) + + @dbt_cloud_assets( + workspace=dbt_resource, + dagster_dbt_translator=CustomizedDagsterDbtTranslator(), + ) + def full_dbt_assets(context: dg.AssetExecutionContext, dbt: DbtCloudWorkspace): + yield from dbt.cli( + ["build"], + dagster_dbt_translator=CustomizedDagsterDbtTranslator(), + context=context, + ).wait() + + dbt_cloud_polling_sensor = build_dbt_cloud_polling_sensor( + workspace=dbt_resource, + dagster_dbt_translator=CustomizedDagsterDbtTranslator(), + minimum_interval_seconds=int( + os.getenv("DBT_PLATFORM_POLL_INTERVAL_SECONDS", "60") + ), + ) +else: + dbt_project_dir = os.getenv("DBT_PROJECT_DIR") + if dbt_project_dir: + dbt_project_dir = Path(dbt_project_dir).resolve() + if not dbt_project_dir.exists(): + raise FileNotFoundError( + "DBT_PROJECT_DIR environment variable points to a non-existent path." + ) + else: + current_file = Path(__file__).resolve() + macro_agents_root = current_file.parent.parent.parent + repo_root = current_file.parent.parent.parent.parent.parent + cwd = Path.cwd() + + possible_dbt_project_paths = [ + cwd / "dbt_project", + cwd.parent / "dbt_project" if cwd.name == "root" else None, + cwd.parent.parent / "dbt_project" if cwd.parent.name == "root" else None, + macro_agents_root / "dbt_project", + macro_agents_root.parent / "dbt_project", + cwd / "macro_agents" / "dbt_project", + repo_root / "dbt_project", + cwd.parent / "dbt_project", + ] + + possible_dbt_project_paths = [ + path for path in possible_dbt_project_paths if path is not None + ] + + dbt_project_dir = None + for path in possible_dbt_project_paths: try: - tried_paths.append(str(p.resolve())) + abs_path = path.resolve() + if abs_path.exists() and abs_path.is_dir(): + if (abs_path / "dbt_project.yml").exists() or ( + abs_path / "dbt_project.yaml" + ).exists(): + dbt_project_dir = abs_path + break except (OSError, RuntimeError): - tried_paths.append(str(p)) + continue - # Also list what actually exists in the working directory for debugging - cwd_contents = [] - try: - cwd_contents = [str(p) for p in cwd.iterdir()] if cwd.exists() else [] - except (OSError, PermissionError): - pass - - raise FileNotFoundError( - "Could not find dbt_project directory. " - "Please ensure it exists relative to the repository root, or set " - "DBT_PROJECT_DIR." - ) + if dbt_project_dir is None: + raise FileNotFoundError( + "Could not find dbt_project directory. " + "Please ensure it exists relative to the repository root, or set " + "DBT_PROJECT_DIR." + ) -if dbt_project_dir is None: - raise RuntimeError("dbt_project_dir was not resolved") + if dbt_project_dir is None: + raise RuntimeError("dbt_project_dir was not resolved") -dbt_project_dir_path = dbt_project_dir + dbt_project_dir_path = dbt_project_dir -dbt_packages_dir = dbt_project_dir_path / "dbt_packages" -dbt_utils_dir = dbt_packages_dir / "dbt_utils" if dbt_packages_dir.exists() else None + dbt_packages_dir = dbt_project_dir_path / "dbt_packages" + dbt_utils_dir = dbt_packages_dir / "dbt_utils" if dbt_packages_dir.exists() else None -if not dbt_packages_dir.exists() or not dbt_utils_dir or not dbt_utils_dir.exists(): - import subprocess + if not dbt_packages_dir.exists() or not dbt_utils_dir or not dbt_utils_dir.exists(): + import subprocess - logger = logging.getLogger("dagster_dbt") - logger.warning("dbt packages not found. Running 'dbt deps' (15s timeout)...") - try: - result = subprocess.run( - ["dbt", "deps", "--target", environment], - cwd=dbt_project_dir_path, - capture_output=True, - text=True, - timeout=15, - ) - if result.returncode != 0: + logger.warning("dbt packages not found. Running 'dbt deps' (15s timeout)...") + try: + result = subprocess.run( + ["dbt", "deps", "--target", environment], + cwd=dbt_project_dir_path, + capture_output=True, + text=True, + timeout=15, + ) + if result.returncode != 0: + logger.warning( + f"dbt deps failed (non-fatal): {result.stderr or result.stdout}" + ) + else: + logger.info("dbt packages installed successfully") + except subprocess.TimeoutExpired: logger.warning( - f"dbt deps failed (non-fatal): {result.stderr or result.stdout}" + "dbt deps timed out (network may be unavailable), continuing without packages" ) - else: - logger.info("dbt packages installed successfully") - except subprocess.TimeoutExpired: - logger.warning( - "dbt deps timed out (network may be unavailable), continuing without packages" - ) - except Exception as e: - logger.warning(f"Could not install dbt packages (non-fatal): {e}") - -dbt_project = DbtProject( - project_dir=dbt_project_dir_path, - target=environment, -) - -dbt_project.prepare_if_dev() + except Exception as e: + logger.warning(f"Could not install dbt packages (non-fatal): {e}") -manifest_path = dbt_project.manifest_path -if manifest_path and manifest_path.exists(): - import logging + dbt_project = DbtProject( + project_dir=dbt_project_dir_path, + target=environment, + ) - logger = logging.getLogger("dagster_dbt") - logger.info("Using dbt manifest") + dbt_project.prepare_if_dev() -dbt_cli_resource = DbtCliResource(project_dir=dbt_project_dir_path) + manifest_path = dbt_project.manifest_path + if manifest_path and manifest_path.exists(): + logger.info("Using dbt manifest") + dbt_resource = DbtCliResource(project_dir=dbt_project_dir_path) + dbt_cloud_polling_sensor = None -@dbt_assets( - manifest=dbt_project.manifest_path, - dagster_dbt_translator=CustomizedDagsterDbtTranslator(), -) -def full_dbt_assets(context: dg.AssetExecutionContext, dbt: DbtCliResource): - dbt_packages_dir = dbt_project_dir_path / "dbt_packages" - dbt_utils_dir = ( - dbt_packages_dir / "dbt_utils" if dbt_packages_dir.exists() else None + @dbt_assets( + manifest=dbt_project.manifest_path, + dagster_dbt_translator=CustomizedDagsterDbtTranslator(), ) + def full_dbt_assets(context: dg.AssetExecutionContext, dbt: DbtCliResource): + dbt_packages_dir = dbt_project_dir_path / "dbt_packages" + dbt_utils_dir = ( + dbt_packages_dir / "dbt_utils" if dbt_packages_dir.exists() else None + ) - if not dbt_packages_dir.exists() or not dbt_utils_dir or not dbt_utils_dir.exists(): - context.log.info("dbt packages not found. Running 'dbt deps' before build...") - try: - deps_result = dbt.cli(["deps"], context=context).wait() - return_code = getattr(deps_result, "return_code", None) - if return_code not in (None, 0): - context.log.error("Failed to install dbt packages.") + if ( + not dbt_packages_dir.exists() + or not dbt_utils_dir + or not dbt_utils_dir.exists() + ): + context.log.info("dbt packages not found. Running 'dbt deps' before build...") + try: + deps_result = dbt.cli(["deps"], context=context).wait() + return_code = getattr(deps_result, "return_code", None) + if return_code not in (None, 0): + context.log.error("Failed to install dbt packages.") + raise RuntimeError( + "dbt packages installation failed. Please run 'dbt deps' manually" + ) + else: + context.log.info("dbt packages installed successfully") + except Exception as e: + context.log.error(f"Could not install dbt packages: {e}") raise RuntimeError( - "dbt packages installation failed. Please run 'dbt deps' manually" - ) - else: - context.log.info("dbt packages installed successfully") - except Exception as e: - context.log.error(f"Could not install dbt packages: {e}") - raise RuntimeError( - "dbt packages are required but could not be installed. Please run 'dbt deps' manually" - ) from e + "dbt packages are required but could not be installed. Please run 'dbt deps' manually" + ) from e - yield from dbt.cli(["build"], context=context).stream() + yield from dbt.cli(["build"], context=context).stream() From 246b9f98c7b5bba7a2e54dca3700e55748255b66 Mon Sep 17 00:00:00 2001 From: Alex Noonan <48368867+C00ldudeNoonan@users.noreply.github.com> Date: Thu, 11 Jun 2026 12:39:29 -0400 Subject: [PATCH 2/4] chore: apply ruff formatting --- .../src/macro_agents/defs/transformation/dbt.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/macro_agents/src/macro_agents/defs/transformation/dbt.py b/macro_agents/src/macro_agents/defs/transformation/dbt.py index 756399f..bec7ff4 100644 --- a/macro_agents/src/macro_agents/defs/transformation/dbt.py +++ b/macro_agents/src/macro_agents/defs/transformation/dbt.py @@ -78,11 +78,10 @@ def _required_env_var(*names: str) -> dg.EnvVar: joined_names = " or ".join(names) raise RuntimeError(f"Set {joined_names} to enable dbt Platform orchestration.") + if orchestration_mode in {"dbt_platform", "dbt_cloud", "platform", "cloud"}: dbt_cloud_credentials = DbtCloudCredentials( - account_id=_required_int_env( - "DBT_PLATFORM_ACCOUNT_ID", "DBT_CLOUD_ACCOUNT_ID" - ), + account_id=_required_int_env("DBT_PLATFORM_ACCOUNT_ID", "DBT_CLOUD_ACCOUNT_ID"), token=_required_env_var("DBT_PLATFORM_TOKEN", "DBT_CLOUD_API_TOKEN"), access_url=_first_env("DBT_PLATFORM_ACCESS_URL", "DBT_CLOUD_ACCESS_URL") or "https://cloud.getdbt.com", @@ -90,9 +89,7 @@ def _required_env_var(*names: str) -> dg.EnvVar: dbt_resource = DbtCloudWorkspace( credentials=dbt_cloud_credentials, - project_id=_required_int_env( - "DBT_PLATFORM_PROJECT_ID", "DBT_CLOUD_PROJECT_ID" - ), + project_id=_required_int_env("DBT_PLATFORM_PROJECT_ID", "DBT_CLOUD_PROJECT_ID"), environment_id=_required_int_env( "DBT_PLATFORM_ENVIRONMENT_ID", "DBT_CLOUD_ENVIRONMENT_ID" ), @@ -174,7 +171,9 @@ def full_dbt_assets(context: dg.AssetExecutionContext, dbt: DbtCloudWorkspace): dbt_project_dir_path = dbt_project_dir dbt_packages_dir = dbt_project_dir_path / "dbt_packages" - dbt_utils_dir = dbt_packages_dir / "dbt_utils" if dbt_packages_dir.exists() else None + dbt_utils_dir = ( + dbt_packages_dir / "dbt_utils" if dbt_packages_dir.exists() else None + ) if not dbt_packages_dir.exists() or not dbt_utils_dir or not dbt_utils_dir.exists(): import subprocess @@ -230,7 +229,9 @@ def full_dbt_assets(context: dg.AssetExecutionContext, dbt: DbtCliResource): or not dbt_utils_dir or not dbt_utils_dir.exists() ): - context.log.info("dbt packages not found. Running 'dbt deps' before build...") + context.log.info( + "dbt packages not found. Running 'dbt deps' before build..." + ) try: deps_result = dbt.cli(["deps"], context=context).wait() return_code = getattr(deps_result, "return_code", None) From a8537698fc872d9ff0512914df6b5fdc06464a84 Mon Sep 17 00:00:00 2001 From: Alex Noonan <48368867+C00ldudeNoonan@users.noreply.github.com> Date: Thu, 11 Jun 2026 16:53:52 -0400 Subject: [PATCH 3/4] fix: make dbt project build with fusion --- .env.example | 20 ++- .../calculate_commodity_analysis_return.sql | 60 +++---- .../macros/calculate_commodity_summary.sql | 12 ++ .../calculate_market_analysis_return.sql | 60 +++---- .../macros/calculate_market_summary.sql | 32 ++-- dbt_project/macros/test_positive_price.sql | 2 +- .../agent_commodity_performance_snapshot.sql | 23 ++- .../agent_market_performance_snapshot.sql | 23 ++- .../agent_treasury_yield_curve_spreads.sql | 29 ++-- .../analysis/commodity_market_signals.sql | 4 +- .../correlation_analysis_enhanced.sql | 30 ++-- .../dispersion/sector_breadth_timeseries.sql | 34 ++-- .../dispersion/sector_dispersion_analysis.sql | 32 ++-- .../economic_regime_classification.sql | 4 +- dbt_project/models/analysis/factor_tilts.sql | 36 ++-- .../analysis/indicator_market_response.sql | 72 ++++---- .../leading_econ_return_indicator.sql | 2 + .../analysis/market_economic_analysis.sql | 14 +- .../analysis/portfolio_macro_factors.sql | 162 ++++++++--------- .../analysis/reddit_sentiment_trends.sql | 26 ++- dbt_project/models/analysis/schema.yml | 10 +- .../analysis/sector_indicator_sensitivity.sql | 42 ++--- .../analysis/sector_regime_performance.sql | 44 ++--- .../analysis/ticker_sector_sensitivity.sql | 32 ++-- .../analytics/telemetry/error_analysis.sql | 80 --------- .../analytics/telemetry/feature_usage.sql | 57 ------ .../telemetry/performance_metrics.sql | 127 -------------- .../models/analytics/telemetry/schema.yml | 107 ------------ .../analytics/telemetry/session_summary.sql | 117 ------------- .../analytics/telemetry/user_journeys.sql | 86 --------- ...riculture_commodities_summary_snapshot.sql | 10 +- .../energy_commodities_summary_snapshot.sql | 10 +- ...fred_series_latest_aggregates_snapshot.sql | 30 ++-- .../input_commodities_summary_snapshot.sql | 26 ++- ...leading_econ_return_indicator_snapshot.sql | 86 ++++----- dbt_project/models/backtesting/schema.yml | 13 +- .../us_sector_summary_snapshot.sql | 28 ++- dbt_project/models/commodities/schema.yml | 7 +- .../models/data_quality/dq_return_spikes.sql | 12 +- .../models/data_quality/dq_stale_prices.sql | 2 +- .../data_quality/dq_zscore_anomalies.sql | 20 +-- .../models/government/fred_quarterly_roc.sql | 38 ++-- .../models/government/fred_series_grain.sql | 4 +- .../fred_series_latest_aggregates.sql | 35 ++-- .../models/government/housing_inventory.sql | 2 +- .../housing_inventory_and_population.sql | 6 +- .../housing_inventory_latest_aggregates.sql | 4 +- .../nasdaq_companies_analysis_return.sql | 60 +++---- .../markets/nasdaq_companies_summary.sql | 10 +- .../sp500_companies_analysis_return.sql | 60 +++---- .../markets/sp500_companies_summary.sql | 42 ++--- .../signals/cross_asset_divergences.sql | 90 +++++----- .../signals/diffusion_index_signals.sql | 10 +- .../signals/economic_acceleration_signals.sql | 12 +- .../models/signals/economic_alert_inputs.sql | 10 +- dbt_project/models/signals/factor_signals.sql | 8 +- .../signals/financial_conditions_signals.sql | 16 +- .../models/signals/housing_signals.sql | 18 +- .../models/signals/inflation_signals.sql | 2 +- dbt_project/models/signals/labor_signals.sql | 30 ++-- .../models/signals/liquidity_signals.sql | 26 +-- .../models/signals/market_breadth_signals.sql | 49 +++--- .../signals/market_volatility_signals.sql | 10 +- .../models/signals/momentum_signals.sql | 6 +- .../models/signals/net_liquidity_signals.sql | 14 +- .../models/signals/sentiment_signals.sql | 30 ++-- .../models/signals/technical_signals.sql | 2 +- dbt_project/models/signals/trade_signals.sql | 4 +- .../models/staging/corporate_actions.sql | 97 ++++++----- dbt_project/models/staging/schema.yml | 4 +- dbt_project/models/staging/stg_currency.sql | 2 +- .../models/staging/stg_earnings_calendar.sql | 43 +++-- .../models/staging/stg_economic_calendar.sql | 50 ++++-- .../models/staging/stg_fixed_income.sql | 2 +- .../staging/stg_fomc_meeting_summaries.sql | 1 + .../staging/stg_fomc_meetings_enhanced.sql | 1 + .../models/staging/stg_fomc_minutes.sql | 15 +- .../models/staging/stg_fomc_transcripts.sql | 21 ++- .../models/staging/stg_fred_series.sql | 4 +- .../models/staging/stg_global_markets.sql | 2 +- .../models/staging/stg_major_indices.sql | 2 +- .../staging/stg_nasdaq_companies_prices.sql | 2 + .../staging/stg_realtor_country_history.sql | 2 + .../staging/stg_realtor_county_history.sql | 2 + .../staging/stg_realtor_metro_history.sql | 2 + .../staging/stg_realtor_state_history.sql | 2 + .../staging/stg_realtor_zip_history.sql | 2 + .../models/staging/stg_reddit_comments.sql | 2 +- .../staging/stg_reddit_post_content.sql | 2 +- .../models/staging/stg_reddit_posts.sql | 10 +- .../staging/stg_reddit_ticker_mentions.sql | 1 + .../staging/stg_sp500_companies_prices.sql | 2 +- .../models/staging/stg_transcript_topics.sql | 1 + dbt_project/models/staging/stg_us_sectors.sql | 2 +- .../models/staging/telemetry/schema.yml | 164 ------------------ .../models/staging/telemetry/sources.yml | 54 ------ .../models/staging/telemetry/stg_sessions.sql | 123 ------------- .../telemetry/stg_telemetry_events.sql | 104 ----------- dbt_project/profiles.yml | 10 +- .../tests/diagnostic_zero_forward_returns.sql | 36 ---- .../test_forward_returns_all_quarters.sql | 39 ----- .../tests/test_forward_returns_not_zero.sql | 36 ---- .../test_forward_returns_same_quarter.sql | 29 ---- .../tests/test_weekly_data_completeness.sql | 64 +++---- .../tests/test_yearly_data_completeness.sql | 12 +- .../dbt-platform-fusion-onboarding.md | 27 +++ 106 files changed, 1115 insertions(+), 2080 deletions(-) delete mode 100644 dbt_project/models/analytics/telemetry/error_analysis.sql delete mode 100644 dbt_project/models/analytics/telemetry/feature_usage.sql delete mode 100644 dbt_project/models/analytics/telemetry/performance_metrics.sql delete mode 100644 dbt_project/models/analytics/telemetry/schema.yml delete mode 100644 dbt_project/models/analytics/telemetry/session_summary.sql delete mode 100644 dbt_project/models/analytics/telemetry/user_journeys.sql delete mode 100644 dbt_project/models/staging/telemetry/schema.yml delete mode 100644 dbt_project/models/staging/telemetry/sources.yml delete mode 100644 dbt_project/models/staging/telemetry/stg_sessions.sql delete mode 100644 dbt_project/models/staging/telemetry/stg_telemetry_events.sql delete mode 100644 dbt_project/tests/diagnostic_zero_forward_returns.sql delete mode 100644 dbt_project/tests/test_forward_returns_all_quarters.sql delete mode 100644 dbt_project/tests/test_forward_returns_not_zero.sql delete mode 100644 dbt_project/tests/test_forward_returns_same_quarter.sql diff --git a/.env.example b/.env.example index 3f8a56e..8e5a54b 100644 --- a/.env.example +++ b/.env.example @@ -62,12 +62,21 @@ GEMINI_API_KEY=your_gemini_api_key_here # ============================================================================= # GOOGLE CLOUD (for GCS storage, Google Docs, and BigQuery) # ============================================================================= -# Service account credentials (JSON file path or JSON string) -# Used for: GCS storage, Google Docs API, and BigQuery +# Use Application Default Credentials (ADC) for Google auth. +# Local setup: +# gcloud auth application-default login +# Service account impersonation, if preferred: +# gcloud auth application-default login --impersonate-service-account service-account@project.iam.gserviceaccount.com +# On GCE/Cloud Run/GitHub WIF, ADC uses the attached or federated service account. +# +# GOOGLE_APPLICATION_CREDENTIALS is intentionally unset for normal ADC auth. +# Set it only if you must use a local service-account keyfile fallback. +# GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json +# +# Used for: GCS storage, Google Docs API, and BigQuery. # Required scopes for Google Docs: # - https://www.googleapis.com/auth/documents # - https://www.googleapis.com/auth/drive -GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json GCS_BUCKET_NAME=your_gcs_bucket_name # Google Docs Integration (optional) @@ -89,9 +98,8 @@ GOOGLE_DRIVE_FOLDER_ID=your_google_drive_folder_id_here # Note: This can be the same as GOOGLE_DRIVE_FOLDER_ID if using one folder REALTOR_GDRIVE_FOLDER_ID=your_realtor_folder_id_here -# BigQuery credentials (path to service account JSON on local dev) -# On GCE the attached VM service account is used automatically via ADC. -# GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json +# BigQuery auth is through ADC. No service-account keyfile is required when +# local ADC, attached service accounts, or workload identity federation are configured. # ============================================================================= # DBT diff --git a/dbt_project/macros/calculate_commodity_analysis_return.sql b/dbt_project/macros/calculate_commodity_analysis_return.sql index 2716e40..40a41c3 100644 --- a/dbt_project/macros/calculate_commodity_analysis_return.sql +++ b/dbt_project/macros/calculate_commodity_analysis_return.sql @@ -68,18 +68,18 @@ rolling_stats AS ( MAX(current_price) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 365 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 365 PRECEDING AND CURRENT ROW ) AS high_1yr, MIN(current_price) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 365 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 365 PRECEDING AND CURRENT ROW ) AS low_1yr, STDDEV(daily_diff) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 365 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 365 PRECEDING AND CURRENT ROW ) AS std_diff_1yr, price_365d_ago AS price_start_1yr, CASE @@ -90,18 +90,18 @@ rolling_stats AS ( MAX(current_price) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 270 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 270 PRECEDING AND CURRENT ROW ) AS high_9mo, MIN(current_price) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 270 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 270 PRECEDING AND CURRENT ROW ) AS low_9mo, STDDEV(daily_diff) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 270 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 270 PRECEDING AND CURRENT ROW ) AS std_diff_9mo, price_270d_ago AS price_start_9mo, CASE @@ -112,18 +112,18 @@ rolling_stats AS ( MAX(current_price) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 180 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 180 PRECEDING AND CURRENT ROW ) AS high_6mo, MIN(current_price) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 180 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 180 PRECEDING AND CURRENT ROW ) AS low_6mo, STDDEV(daily_diff) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 180 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 180 PRECEDING AND CURRENT ROW ) AS std_diff_6mo, price_180d_ago AS price_start_6mo, CASE @@ -134,18 +134,18 @@ rolling_stats AS ( MAX(current_price) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 90 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 90 PRECEDING AND CURRENT ROW ) AS high_3mo, MIN(current_price) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 90 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 90 PRECEDING AND CURRENT ROW ) AS low_3mo, STDDEV(daily_diff) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 90 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 90 PRECEDING AND CURRENT ROW ) AS std_diff_3mo, price_90d_ago AS price_start_3mo, CASE @@ -156,18 +156,18 @@ rolling_stats AS ( MAX(current_price) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 30 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 30 PRECEDING AND CURRENT ROW ) AS high_1mo, MIN(current_price) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 30 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 30 PRECEDING AND CURRENT ROW ) AS low_1mo, STDDEV(daily_diff) OVER ( PARTITION BY commodity_name, commodity_unit - ORDER BY date - RANGE BETWEEN INTERVAL 30 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 30 PRECEDING AND CURRENT ROW ) AS std_diff_1mo, price_30d_ago AS price_start_1mo, CASE diff --git a/dbt_project/macros/calculate_commodity_summary.sql b/dbt_project/macros/calculate_commodity_summary.sql index d6f49f9..aec6292 100644 --- a/dbt_project/macros/calculate_commodity_summary.sql +++ b/dbt_project/macros/calculate_commodity_summary.sql @@ -87,8 +87,13 @@ start_prices AS ( FROM period_boundaries AS pb INNER JOIN filtered_data AS fd ON pb.commodity_name = fd.commodity_name + AND pb.commodity_unit = fd.commodity_unit AND pb.time_period = fd.time_period AND pb.period_start_date = fd.trade_date + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY pb.commodity_name, pb.commodity_unit, pb.time_period + ORDER BY fd.trade_date ASC, fd.price ASC + ) = 1 ), end_prices AS ( @@ -100,8 +105,13 @@ end_prices AS ( FROM period_boundaries AS pb INNER JOIN filtered_data AS fd ON pb.commodity_name = fd.commodity_name + AND pb.commodity_unit = fd.commodity_unit AND pb.time_period = fd.time_period AND pb.period_end_date = fd.trade_date + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY pb.commodity_name, pb.commodity_unit, pb.time_period + ORDER BY fd.trade_date DESC, fd.price DESC + ) = 1 ), -- Main aggregation @@ -140,10 +150,12 @@ combined_results AS ( LEFT JOIN start_prices AS sp ON ar.commodity_name = sp.commodity_name + AND ar.commodity_unit = sp.commodity_unit AND ar.time_period = sp.time_period LEFT JOIN end_prices AS ep ON ar.commodity_name = ep.commodity_name + AND ar.commodity_unit = ep.commodity_unit AND ar.time_period = ep.time_period ), diff --git a/dbt_project/macros/calculate_market_analysis_return.sql b/dbt_project/macros/calculate_market_analysis_return.sql index 8b185e4..ab5ba91 100644 --- a/dbt_project/macros/calculate_market_analysis_return.sql +++ b/dbt_project/macros/calculate_market_analysis_return.sql @@ -88,18 +88,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 365 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 365 PRECEDING AND CURRENT ROW ) AS high_1yr, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 365 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 365 PRECEDING AND CURRENT ROW ) AS low_1yr, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 365 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 365 PRECEDING AND CURRENT ROW ) AS std_diff_1yr, price_365d_ago AS price_start_1yr, CASE @@ -110,18 +110,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 270 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 270 PRECEDING AND CURRENT ROW ) AS high_9mo, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 270 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 270 PRECEDING AND CURRENT ROW ) AS low_9mo, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 270 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 270 PRECEDING AND CURRENT ROW ) AS std_diff_9mo, price_270d_ago AS price_start_9mo, CASE @@ -132,18 +132,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 180 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 180 PRECEDING AND CURRENT ROW ) AS high_6mo, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 180 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 180 PRECEDING AND CURRENT ROW ) AS low_6mo, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 180 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 180 PRECEDING AND CURRENT ROW ) AS std_diff_6mo, price_180d_ago AS price_start_6mo, CASE @@ -154,18 +154,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 90 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 90 PRECEDING AND CURRENT ROW ) AS high_3mo, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 90 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 90 PRECEDING AND CURRENT ROW ) AS low_3mo, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 90 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 90 PRECEDING AND CURRENT ROW ) AS std_diff_3mo, price_90d_ago AS price_start_3mo, CASE @@ -176,18 +176,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 30 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 30 PRECEDING AND CURRENT ROW ) AS high_1mo, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 30 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 30 PRECEDING AND CURRENT ROW ) AS low_1mo, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 30 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 30 PRECEDING AND CURRENT ROW ) AS std_diff_1mo, price_30d_ago AS price_start_1mo, CASE diff --git a/dbt_project/macros/calculate_market_summary.sql b/dbt_project/macros/calculate_market_summary.sql index e22538b..d5bde29 100644 --- a/dbt_project/macros/calculate_market_summary.sql +++ b/dbt_project/macros/calculate_market_summary.sql @@ -53,50 +53,53 @@ filtered_data AS ( period_boundaries AS ( SELECT symbol, - asset_type, time_period, MIN(trade_date) AS period_start_date, MAX(trade_date) AS period_end_date FROM filtered_data WHERE time_period != 'older' - GROUP BY symbol, asset_type, time_period + GROUP BY symbol, time_period ), start_prices AS ( SELECT pb.symbol, - pb.asset_type, pb.time_period, fd.adj_open AS period_start_price FROM period_boundaries AS pb INNER JOIN filtered_data AS fd ON pb.symbol = fd.symbol - AND pb.asset_type = fd.asset_type AND pb.time_period = fd.time_period AND pb.period_start_date = fd.trade_date + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY pb.symbol, pb.time_period + ORDER BY fd.trade_date ASC, fd.adj_open ASC + ) = 1 ), end_prices AS ( SELECT pb.symbol, - pb.asset_type, pb.time_period, fd.adj_close AS period_end_price FROM period_boundaries AS pb INNER JOIN filtered_data AS fd ON pb.symbol = fd.symbol - AND pb.asset_type = fd.asset_type AND pb.time_period = fd.time_period AND pb.period_end_date = fd.trade_date + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY pb.symbol, pb.time_period + ORDER BY fd.trade_date DESC, fd.adj_close DESC + ) = 1 ), aggregated_results AS ( SELECT symbol, - asset_type, + ARRAY_AGG(asset_type ORDER BY trade_date DESC LIMIT 1)[SAFE_OFFSET(0)] AS asset_type, time_period, - exchange, - name, + ARRAY_AGG(exchange ORDER BY trade_date DESC LIMIT 1)[SAFE_OFFSET(0)] AS exchange, + ARRAY_AGG(name ORDER BY trade_date DESC LIMIT 1)[SAFE_OFFSET(0)] AS name, MIN(trade_date) AS period_start_date, MAX(trade_date) AS period_end_date, COUNT(*) AS trading_days, @@ -125,10 +128,7 @@ aggregated_results AS ( WHERE time_period != 'older' GROUP BY symbol, - asset_type, - time_period, - exchange, - name + time_period ), combined_results AS ( @@ -140,11 +140,9 @@ combined_results AS ( LEFT JOIN start_prices AS sp ON ar.symbol = sp.symbol - AND ar.asset_type = sp.asset_type AND ar.time_period = sp.time_period LEFT JOIN end_prices AS ep ON ar.symbol = ep.symbol - AND ar.asset_type = ep.asset_type AND ar.time_period = ep.time_period ), @@ -192,6 +190,10 @@ SELECT ROUND(period_start_price, 2) AS period_start_price, ROUND(period_end_price, 2) AS period_end_price FROM final_metrics +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY symbol, time_period + ORDER BY period_end_date DESC, period_start_date DESC +) = 1 ORDER BY time_period, asset_type, diff --git a/dbt_project/macros/test_positive_price.sql b/dbt_project/macros/test_positive_price.sql index cd11ac0..1484ae1 100644 --- a/dbt_project/macros/test_positive_price.sql +++ b/dbt_project/macros/test_positive_price.sql @@ -4,5 +4,5 @@ select * from {{ model }} where {{ column_name }} is not null - and cast({{ column_name }} as double) <= 0 + and cast({{ column_name }} as float64) <= 0 {% endtest %} diff --git a/dbt_project/models/agents_preprocess/agent_commodity_performance_snapshot.sql b/dbt_project/models/agents_preprocess/agent_commodity_performance_snapshot.sql index 32f3ddc..e6bf89b 100644 --- a/dbt_project/models/agents_preprocess/agent_commodity_performance_snapshot.sql +++ b/dbt_project/models/agents_preprocess/agent_commodity_performance_snapshot.sql @@ -2,7 +2,7 @@ config( materialized='incremental', unique_key=['snapshot_date', 'commodity_category', 'commodity_name', 'commodity_unit', 'time_period'], - incremental_strategy='delete+insert', + incremental_strategy='merge', tags=['agents_preprocess'] ) }} @@ -113,14 +113,23 @@ agriculture_snapshot as ( DATE '1900-01-01' ) - INTERVAL 1 MONTH {% endif %} -) +), -select * from energy_snapshot +combined_snapshots as ( + select * from energy_snapshot -union all + union all -select * from input_snapshot + select * from input_snapshot -union all + union all + + select * from agriculture_snapshot +) -select * from agriculture_snapshot +select * +from combined_snapshots +qualify row_number() over ( + partition by snapshot_date, commodity_category, commodity_name, commodity_unit, time_period + order by period_end_date desc, period_start_date desc +) = 1 diff --git a/dbt_project/models/agents_preprocess/agent_market_performance_snapshot.sql b/dbt_project/models/agents_preprocess/agent_market_performance_snapshot.sql index 51bb388..7d33de9 100644 --- a/dbt_project/models/agents_preprocess/agent_market_performance_snapshot.sql +++ b/dbt_project/models/agents_preprocess/agent_market_performance_snapshot.sql @@ -2,7 +2,7 @@ config( materialized='incremental', unique_key=['snapshot_date', 'market_category', 'symbol', 'asset_type', 'time_period'], - incremental_strategy='delete+insert', + incremental_strategy='merge', tags=['agents_preprocess'] ) }} @@ -73,18 +73,27 @@ major_index_snapshot as ( period_start_price, period_end_price, 'major_index' as market_category, - DATE_TRUNC('month', period_end_date) as snapshot_date + DATE_TRUNC(period_end_date, MONTH) as snapshot_date from {{ ref('major_indicies_summary') }} {% if is_incremental() %} - where DATE_TRUNC('month', period_end_date) >= COALESCE( + where DATE_TRUNC(period_end_date, MONTH) >= COALESCE( (select max(snapshot_date) from {{ this }}), DATE '1900-01-01' ) - INTERVAL 1 MONTH {% endif %} -) +), -select * from sector_snapshot +combined_snapshots as ( + select * from sector_snapshot -union all + union all + + select * from major_index_snapshot +) -select * from major_index_snapshot +select * +from combined_snapshots +qualify row_number() over ( + partition by snapshot_date, market_category, symbol, asset_type, time_period + order by period_end_date desc, period_start_date desc +) = 1 diff --git a/dbt_project/models/agents_preprocess/agent_treasury_yield_curve_spreads.sql b/dbt_project/models/agents_preprocess/agent_treasury_yield_curve_spreads.sql index ecf05cd..6d2ed6a 100644 --- a/dbt_project/models/agents_preprocess/agent_treasury_yield_curve_spreads.sql +++ b/dbt_project/models/agents_preprocess/agent_treasury_yield_curve_spreads.sql @@ -2,34 +2,33 @@ config( materialized='incremental', unique_key='date', - incremental_strategy='delete+insert', + incremental_strategy='merge', tags=['agents_preprocess'] ) }} with pivoted_yields as ( select - date, - max(case when yield_type = 'BC_1MONTH' then value end) as yield_1m, - max(case when yield_type = 'BC_3MONTH' then value end) as yield_3m, - max(case when yield_type = 'BC_6MONTH' then value end) as yield_6m, - max(case when yield_type = 'BC_1YEAR' then value end) as yield_1y, - max(case when yield_type = 'BC_2YEAR' then value end) as yield_2y, - max(case when yield_type = 'BC_3YEAR' then value end) as yield_3y, - max(case when yield_type = 'BC_5YEAR' then value end) as yield_5y, - max(case when yield_type = 'BC_7YEAR' then value end) as yield_7y, - max(case when yield_type = 'BC_10YEAR' then value end) as yield_10y, - max(case when yield_type = 'BC_20YEAR' then value end) as yield_20y, - max(case when yield_type = 'BC_30YEAR' then value end) as yield_30y + safe_cast(date as date) as date, + safe_cast(bc_1month as float64) as yield_1m, + bc_3month as yield_3m, + bc_6month as yield_6m, + bc_1year as yield_1y, + bc_2year as yield_2y, + bc_3year as yield_3y, + bc_5year as yield_5y, + bc_7year as yield_7y, + bc_10year as yield_10y, + safe_cast(bc_20year as float64) as yield_20y, + bc_30year as yield_30y from {{ ref('stg_treasury_yields') }} where date is not null {% if is_incremental() %} - and date >= COALESCE( + and safe_cast(date as date) >= COALESCE( (select max(date) from {{ this }}), DATE '1900-01-01' ) - INTERVAL 7 DAY {% endif %} - group by date ), yield_spreads as ( diff --git a/dbt_project/models/analysis/commodity_market_signals.sql b/dbt_project/models/analysis/commodity_market_signals.sql index 8ef8bf5..61f6658 100644 --- a/dbt_project/models/analysis/commodity_market_signals.sql +++ b/dbt_project/models/analysis/commodity_market_signals.sql @@ -60,7 +60,7 @@ spy_prices AS ( -- Get all unique dates where we have at least gold or SPY data all_dates AS ( SELECT DISTINCT date FROM gold_prices - UNION + UNION DISTINCT SELECT DISTINCT date FROM spy_prices ), @@ -163,5 +163,5 @@ SELECT END AS oil_trend_signal FROM with_calculations -WHERE date >= CURRENT_DATE - INTERVAL 2 YEAR +WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 2 YEAR) ORDER BY date DESC diff --git a/dbt_project/models/analysis/correlation_analysis_enhanced.sql b/dbt_project/models/analysis/correlation_analysis_enhanced.sql index e1fb5e8..8616922 100644 --- a/dbt_project/models/analysis/correlation_analysis_enhanced.sql +++ b/dbt_project/models/analysis/correlation_analysis_enhanced.sql @@ -10,14 +10,14 @@ WITH sector_monthly AS ( SELECT symbol, - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, LAST_VALUE(pct_change_1mo) OVER ( - PARTITION BY symbol, DATE_TRUNC('month', date) + PARTITION BY symbol, DATE_TRUNC(date, MONTH) ORDER BY date ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS monthly_return, ROW_NUMBER() OVER ( - PARTITION BY symbol, DATE_TRUNC('month', date) + PARTITION BY symbol, DATE_TRUNC(date, MONTH) ORDER BY date DESC ) AS rn FROM {{ ref('us_sector_analysis_return') }} @@ -34,7 +34,7 @@ indicator_monthly AS ( SELECT series_code, series_name, - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, value, CASE WHEN LAG(value) OVER (PARTITION BY series_code ORDER BY date) IS NOT NULL @@ -159,6 +159,19 @@ rolling_correlations AS ( ), -- Calculate correlation stability metrics +correlation_with_signs AS ( + SELECT + *, + CASE + WHEN rolling_corr_12m * LAG(rolling_corr_12m) OVER ( + PARTITION BY symbol, series_code ORDER BY month_date + ) < 0 THEN 1 + ELSE 0 + END AS sign_change_flag + FROM rolling_correlations + WHERE rolling_corr_12m IS NOT NULL +), + correlation_stability AS ( SELECT symbol, @@ -170,13 +183,8 @@ correlation_stability AS ( MIN(rolling_corr_12m) AS rolling_corr_min, MAX(rolling_corr_12m) AS rolling_corr_max, -- Count sign changes (correlation instability indicator) - SUM(CASE - WHEN rolling_corr_12m * LAG(rolling_corr_12m) OVER ( - PARTITION BY symbol, series_code ORDER BY month_date - ) < 0 THEN 1 ELSE 0 - END) AS sign_changes - FROM rolling_correlations - WHERE rolling_corr_12m IS NOT NULL + SUM(sign_change_flag) AS sign_changes + FROM correlation_with_signs GROUP BY symbol, series_code, series_name ), diff --git a/dbt_project/models/analysis/dispersion/sector_breadth_timeseries.sql b/dbt_project/models/analysis/dispersion/sector_breadth_timeseries.sql index 2fb684f..6ad106b 100644 --- a/dbt_project/models/analysis/dispersion/sector_breadth_timeseries.sql +++ b/dbt_project/models/analysis/dispersion/sector_breadth_timeseries.sql @@ -5,20 +5,20 @@ }} WITH sector_mapping AS ( - SELECT sector_mapping.* - FROM (VALUES - ('Information Technology', 'XLK', 'Technology'), - ('Communication Services', 'XLC', 'Communication Services'), - ('Consumer Discretionary', 'XLY', 'Consumer Discretionary'), - ('Financials', 'XLF', 'Financial'), - ('Industrials', 'XLI', 'Industrial'), - ('Utilities', 'XLU', 'Utilities'), - ('Consumer Staples', 'XLP', 'Consumer Staples'), - ('Real Estate', 'XLRE', 'Real Estate'), - ('Materials', 'XLB', 'Materials'), - ('Energy', 'XLE', 'Energy'), - ('Health Care', 'XLV', 'Health Care') - ) AS sector_mapping (gics_sector, etf_symbol, sector_display_name) + SELECT * + FROM UNNEST([ + STRUCT('Information Technology' AS gics_sector, 'XLK' AS etf_symbol, 'Technology' AS sector_display_name), + STRUCT('Communication Services', 'XLC', 'Communication Services'), + STRUCT('Consumer Discretionary', 'XLY', 'Consumer Discretionary'), + STRUCT('Financials', 'XLF', 'Financial'), + STRUCT('Industrials', 'XLI', 'Industrial'), + STRUCT('Utilities', 'XLU', 'Utilities'), + STRUCT('Consumer Staples', 'XLP', 'Consumer Staples'), + STRUCT('Real Estate', 'XLRE', 'Real Estate'), + STRUCT('Materials', 'XLB', 'Materials'), + STRUCT('Energy', 'XLE', 'Energy'), + STRUCT('Health Care', 'XLV', 'Health Care') + ]) ), -- Load 4 years of data: 3 years of output + ~1 year lookback for 200-day MA warm-up @@ -30,7 +30,7 @@ stock_prices AS ( FROM {{ ref('stg_sp500_companies_prices') }} WHERE adj_close IS NOT NULL AND adj_close > 0 - AND date >= CURRENT_DATE - INTERVAL 4 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 4 YEAR) ), -- 200-day SMA per stock (computed on the full 4-year range) @@ -60,7 +60,7 @@ stock_ma_flags AS ( CASE WHEN ma_200_days_count >= 200 AND price > sma_200 THEN 1 ELSE 0 END AS above_200_ma, CASE WHEN ma_200_days_count >= 200 THEN 1 ELSE 0 END AS has_valid_ma FROM stock_with_ma - WHERE date >= CURRENT_DATE - INTERVAL 3 YEAR + WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), -- Join with sector metadata @@ -97,7 +97,7 @@ weekly_breadth AS ( SELECT *, ROW_NUMBER() OVER ( - PARTITION BY gics_sector, DATE_TRUNC('week', date) + PARTITION BY gics_sector, DATE_TRUNC(date, WEEK) ORDER BY date DESC ) AS rn FROM sector_daily_breadth diff --git a/dbt_project/models/analysis/dispersion/sector_dispersion_analysis.sql b/dbt_project/models/analysis/dispersion/sector_dispersion_analysis.sql index c799423..6799a6b 100644 --- a/dbt_project/models/analysis/dispersion/sector_dispersion_analysis.sql +++ b/dbt_project/models/analysis/dispersion/sector_dispersion_analysis.sql @@ -5,20 +5,20 @@ }} WITH sector_mapping AS ( - SELECT sector_mapping.* - FROM (VALUES - ('Information Technology', 'XLK', 'Technology'), - ('Communication Services', 'XLC', 'Communication Services'), - ('Consumer Discretionary', 'XLY', 'Consumer Discretionary'), - ('Financials', 'XLF', 'Financial'), - ('Industrials', 'XLI', 'Industrial'), - ('Utilities', 'XLU', 'Utilities'), - ('Consumer Staples', 'XLP', 'Consumer Staples'), - ('Real Estate', 'XLRE', 'Real Estate'), - ('Materials', 'XLB', 'Materials'), - ('Energy', 'XLE', 'Energy'), - ('Health Care', 'XLV', 'Health Care') - ) AS sector_mapping (gics_sector, etf_symbol, sector_display_name) + SELECT * + FROM UNNEST([ + STRUCT('Information Technology' AS gics_sector, 'XLK' AS etf_symbol, 'Technology' AS sector_display_name), + STRUCT('Communication Services', 'XLC', 'Communication Services'), + STRUCT('Consumer Discretionary', 'XLY', 'Consumer Discretionary'), + STRUCT('Financials', 'XLF', 'Financial'), + STRUCT('Industrials', 'XLI', 'Industrial'), + STRUCT('Utilities', 'XLU', 'Utilities'), + STRUCT('Consumer Staples', 'XLP', 'Consumer Staples'), + STRUCT('Real Estate', 'XLRE', 'Real Estate'), + STRUCT('Materials', 'XLB', 'Materials'), + STRUCT('Energy', 'XLE', 'Energy'), + STRUCT('Health Care', 'XLV', 'Health Care') + ]) ), -- Compute true trailing-12M returns directly from price data @@ -34,7 +34,7 @@ trailing_prices AS ( FROM {{ ref('stg_sp500_companies_prices') }} p WHERE p.adj_close IS NOT NULL AND p.adj_close > 0 - AND p.date >= CURRENT_DATE - INTERVAL 1 YEAR + AND p.date >= DATE_SUB(CURRENT_DATE(), INTERVAL 1 YEAR) ), stock_trailing_returns AS ( @@ -75,7 +75,7 @@ sector_stats AS ( ROUND(STDDEV_SAMP(return_1y), 2) AS intra_sector_std_dev, ROUND(MAX(return_1y) - MIN(return_1y), 2) AS best_worst_spread, ROUND(AVG(return_1y), 2) AS avg_return, - ROUND(MEDIAN(return_1y), 2) AS median_return + ROUND(APPROX_QUANTILES(return_1y, 100)[OFFSET(50)], 2) AS median_return FROM company_returns GROUP BY gics_sector, etf_symbol, sector_display_name ), diff --git a/dbt_project/models/analysis/economic_regime_classification.sql b/dbt_project/models/analysis/economic_regime_classification.sql index b60d3ad..63b5652 100644 --- a/dbt_project/models/analysis/economic_regime_classification.sql +++ b/dbt_project/models/analysis/economic_regime_classification.sql @@ -11,7 +11,7 @@ WITH monthly_indicators AS ( -- Pivot key indicators into columns for each month SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(CASE WHEN series_code = 'INDPRO' THEN value END) AS industrial_production, MAX(CASE WHEN series_code = 'UNRATE' THEN value END) AS unemployment_rate, MAX(CASE WHEN series_code = 'PAYEMS' THEN value END) AS nonfarm_payrolls, @@ -31,7 +31,7 @@ WITH monthly_indicators AS ( 'T10Y2Y', 'T10Y3M', 'CFNAIMA3', 'USSLIND', 'ICSA', 'UMCSENT', 'IPMAN', 'NFCI' ) - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), indicator_changes AS ( diff --git a/dbt_project/models/analysis/factor_tilts.sql b/dbt_project/models/analysis/factor_tilts.sql index 8815767..5d89732 100644 --- a/dbt_project/models/analysis/factor_tilts.sql +++ b/dbt_project/models/analysis/factor_tilts.sql @@ -20,27 +20,21 @@ WITH regime_history AS ( ), regime_mapping AS ( - SELECT - t.* - FROM ( - VALUES - ('Expansion', 'Neutral', 'Overweight', 'Neutral', 'Underweight', 'Neutral', - 'Momentum tends to lead in sustained expansions.'), - ('Slowdown', 'Overweight', 'Neutral', 'Overweight', 'Neutral', 'Neutral', - 'Value and financial strength tend to outperform late-cycle.'), - ('Contraction', 'Neutral', 'Underweight', 'Overweight', 'Overweight', 'Underweight', - 'Quality and low volatility typically hold up best in recessions.'), - ('Recovery', 'Overweight', 'Neutral', 'Neutral', 'Underweight', 'Overweight', - 'Early recoveries favor value and size as risk appetite returns.') - ) AS t ( - regime, - value_tilt, - momentum_tilt, - quality_tilt, - low_vol_tilt, - size_tilt, - notes - ) + SELECT * + FROM UNNEST([ + STRUCT( + 'Expansion' AS regime, + 'Neutral' AS value_tilt, + 'Overweight' AS momentum_tilt, + 'Neutral' AS quality_tilt, + 'Underweight' AS low_vol_tilt, + 'Neutral' AS size_tilt, + 'Momentum tends to lead in sustained expansions.' AS notes + ), + STRUCT('Slowdown', 'Overweight', 'Neutral', 'Overweight', 'Neutral', 'Neutral', 'Value and financial strength tend to outperform late-cycle.'), + STRUCT('Contraction', 'Neutral', 'Underweight', 'Overweight', 'Overweight', 'Underweight', 'Quality and low volatility typically hold up best in recessions.'), + STRUCT('Recovery', 'Overweight', 'Neutral', 'Neutral', 'Underweight', 'Overweight', 'Early recoveries favor value and size as risk appetite returns.') + ]) ) SELECT diff --git a/dbt_project/models/analysis/indicator_market_response.sql b/dbt_project/models/analysis/indicator_market_response.sql index 48627dd..68b932e 100644 --- a/dbt_project/models/analysis/indicator_market_response.sql +++ b/dbt_project/models/analysis/indicator_market_response.sql @@ -8,34 +8,34 @@ -- Calculates how sectors respond to economic indicator releases and surprise moves WITH sector_names AS ( - SELECT sector_names.* - FROM (VALUES - ('XLK', 'Technology'), - ('XLC', 'Communication Services'), - ('XLY', 'Consumer Discretionary'), - ('XLF', 'Financial'), - ('XLI', 'Industrial'), - ('XLU', 'Utilities'), - ('XLP', 'Consumer Staples'), - ('XLRE', 'Real Estate'), - ('XLB', 'Materials'), - ('XLE', 'Energy'), - ('XLV', 'Health Care') - ) AS sector_names (symbol, sector_name) + SELECT * + FROM UNNEST([ + STRUCT('XLK' AS symbol, 'Technology' AS sector_name), + STRUCT('XLC' AS symbol, 'Communication Services' AS sector_name), + STRUCT('XLY' AS symbol, 'Consumer Discretionary' AS sector_name), + STRUCT('XLF' AS symbol, 'Financial' AS sector_name), + STRUCT('XLI' AS symbol, 'Industrial' AS sector_name), + STRUCT('XLU' AS symbol, 'Utilities' AS sector_name), + STRUCT('XLP' AS symbol, 'Consumer Staples' AS sector_name), + STRUCT('XLRE' AS symbol, 'Real Estate' AS sector_name), + STRUCT('XLB' AS symbol, 'Materials' AS sector_name), + STRUCT('XLE' AS symbol, 'Energy' AS sector_name), + STRUCT('XLV' AS symbol, 'Health Care' AS sector_name) + ]) ), -- Get monthly sector returns sector_monthly AS ( SELECT symbol, - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, LAST_VALUE(pct_change_1mo) OVER ( - PARTITION BY symbol, DATE_TRUNC('month', date) + PARTITION BY symbol, DATE_TRUNC(date, MONTH) ORDER BY date ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS monthly_return, ROW_NUMBER() OVER ( - PARTITION BY symbol, DATE_TRUNC('month', date) + PARTITION BY symbol, DATE_TRUNC(date, MONTH) ORDER BY date DESC ) AS rn FROM {{ ref('us_sector_analysis_return') }} @@ -48,35 +48,47 @@ sector_returns AS ( WHERE rn = 1 ), --- Get monthly FRED indicator values with changes and rolling stats -indicator_monthly AS ( +-- Get monthly FRED indicator values with month-over-month changes +indicator_values AS ( SELECT series_code, series_name, - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, value, - -- Month-over-month change value - LAG(value) OVER (PARTITION BY series_code ORDER BY date) AS mom_change, - -- Percentage change CASE WHEN LAG(value) OVER (PARTITION BY series_code ORDER BY date) IS NOT NULL AND LAG(value) OVER (PARTITION BY series_code ORDER BY date) != 0 - THEN ((value - LAG(value) OVER (PARTITION BY series_code ORDER BY date)) - / ABS(LAG(value) OVER (PARTITION BY series_code ORDER BY date))) * 100 - END AS mom_pct_change, + THEN SAFE_DIVIDE( + value - LAG(value) OVER (PARTITION BY series_code ORDER BY date), + ABS(LAG(value) OVER (PARTITION BY series_code ORDER BY date)) + ) * 100 + END AS mom_pct_change + FROM {{ ref('fred_monthly_diff') }} +), + +-- Add rolling stats in a second stage; BigQuery disallows nested analytic functions. +indicator_monthly AS ( + SELECT + series_code, + series_name, + month_date, + value, + mom_change, + mom_pct_change, -- Rolling 12-month average change (as trend/expected) - AVG(value - LAG(value) OVER (PARTITION BY series_code ORDER BY date)) OVER ( + AVG(mom_change) OVER ( PARTITION BY series_code - ORDER BY date + ORDER BY month_date ROWS BETWEEN 12 PRECEDING AND 1 PRECEDING ) AS avg_12mo_change, -- Rolling standard deviation of changes - STDDEV(value - LAG(value) OVER (PARTITION BY series_code ORDER BY date)) OVER ( + STDDEV(mom_change) OVER ( PARTITION BY series_code - ORDER BY date + ORDER BY month_date ROWS BETWEEN 12 PRECEDING AND 1 PRECEDING ) AS std_12mo_change - FROM {{ ref('fred_monthly_diff') }} + FROM indicator_values ), -- Calculate indicator surprises (deviation from trend) diff --git a/dbt_project/models/analysis/leading_econ_return_indicator.sql b/dbt_project/models/analysis/leading_econ_return_indicator.sql index ca13cc1..6a0b36c 100644 --- a/dbt_project/models/analysis/leading_econ_return_indicator.sql +++ b/dbt_project/models/analysis/leading_econ_return_indicator.sql @@ -70,6 +70,8 @@ WITH economic_changes AS ( LEFT JOIN {{ ref('fred_series_mapping') }} AS fsm ON bha.series_name = fsm.series_name WHERE bha.value IS NOT NULL + AND bha.series_name IS NOT NULL + AND fsm.category IS NOT NULL ), correlation_analysis AS ( diff --git a/dbt_project/models/analysis/market_economic_analysis.sql b/dbt_project/models/analysis/market_economic_analysis.sql index 23ea0c0..762b224 100644 --- a/dbt_project/models/analysis/market_economic_analysis.sql +++ b/dbt_project/models/analysis/market_economic_analysis.sql @@ -15,7 +15,7 @@ WITH economic_data AS ( data_source AS economic_data_source, -- Create month_date for joining CASE - WHEN year_month ~ '^\d{4}-\d{1,2}$' + WHEN REGEXP_CONTAINS(year_month, r'^\d{4}-\d{1,2}$') THEN DATE( CAST(SPLIT(year_month, '-')[OFFSET(0)] AS INT64), @@ -36,7 +36,7 @@ economic_indicators_pivoted AS ( CASE WHEN series_code LIKE '%GDP%' - OR series_name ILIKE '%gross domestic product%' + OR LOWER(series_name) LIKE '%gross domestic product%' THEN economic_value END ) AS gdp_value, @@ -44,7 +44,7 @@ economic_indicators_pivoted AS ( CASE WHEN series_code LIKE '%GDP%' - OR series_name ILIKE '%gross domestic product%' + OR LOWER(series_name) LIKE '%gross domestic product%' THEN economic_change_pct END ) AS gdp_change_pct, @@ -54,7 +54,7 @@ economic_indicators_pivoted AS ( CASE WHEN series_code LIKE '%CPI%' - OR series_name ILIKE '%consumer price%' + OR LOWER(series_name) LIKE '%consumer price%' THEN economic_value END ) AS cpi_value, @@ -62,7 +62,7 @@ economic_indicators_pivoted AS ( CASE WHEN series_code LIKE '%CPI%' - OR series_name ILIKE '%consumer price%' + OR LOWER(series_name) LIKE '%consumer price%' THEN economic_change_pct END ) AS cpi_change_pct, @@ -71,14 +71,14 @@ economic_indicators_pivoted AS ( MAX( CASE WHEN - series_name ILIKE '%interest%' OR series_name ILIKE '%rate%' + LOWER(series_name) LIKE '%interest%' OR LOWER(series_name) LIKE '%rate%' THEN economic_value END ) AS interest_rate_value, MAX( CASE WHEN - series_name ILIKE '%interest%' OR series_name ILIKE '%rate%' + LOWER(series_name) LIKE '%interest%' OR LOWER(series_name) LIKE '%rate%' THEN economic_change_pct END ) AS interest_rate_change_pct diff --git a/dbt_project/models/analysis/portfolio_macro_factors.sql b/dbt_project/models/analysis/portfolio_macro_factors.sql index dc10dbc..f8d3397 100644 --- a/dbt_project/models/analysis/portfolio_macro_factors.sql +++ b/dbt_project/models/analysis/portfolio_macro_factors.sql @@ -7,100 +7,100 @@ -- Define macro factor groupings based on indicator characteristics -- More granular than the simple Growth/Inflation categories WITH factor_mapping AS ( - SELECT factor_mapping.* - FROM (VALUES + SELECT * + FROM UNNEST([ -- Inflation factors - ('CPIAUCSL', 'Inflation', 'Core Inflation'), - ('CPILFESL', 'Inflation', 'Core Inflation'), - ('CPIAUCNS', 'Inflation', 'Core Inflation'), - ('CPILFENS', 'Inflation', 'Core Inflation'), - ('PCEPI', 'Inflation', 'Core Inflation'), - ('PCEPILFE', 'Inflation', 'Core Inflation'), - ('MEDCPIM158SFRBCLE', 'Inflation', 'Core Inflation'), - ('CORESTICKM159SFRBATL', 'Inflation', 'Sticky Inflation'), - ('STICKCPIM159SFRBATL', 'Inflation', 'Sticky Inflation'), - ('PCETRIM12M159SFRBDAL', 'Inflation', 'Core Inflation'), - ('T10YIE', 'Inflation', 'Inflation Expectations'), - ('T5YIE', 'Inflation', 'Inflation Expectations'), - ('T5YIFR', 'Inflation', 'Inflation Expectations'), - ('PPIACO', 'Inflation', 'Producer Prices'), - ('PPIFIS', 'Inflation', 'Producer Prices'), - ('PPIFID', 'Inflation', 'Producer Prices'), - ('CPIENGSL', 'Inflation', 'Energy Inflation'), + STRUCT('CPIAUCSL' AS series_code, 'Inflation' AS macro_factor, 'Core Inflation' AS sub_factor), + STRUCT('CPILFESL' AS series_code, 'Inflation' AS macro_factor, 'Core Inflation' AS sub_factor), + STRUCT('CPIAUCNS' AS series_code, 'Inflation' AS macro_factor, 'Core Inflation' AS sub_factor), + STRUCT('CPILFENS' AS series_code, 'Inflation' AS macro_factor, 'Core Inflation' AS sub_factor), + STRUCT('PCEPI' AS series_code, 'Inflation' AS macro_factor, 'Core Inflation' AS sub_factor), + STRUCT('PCEPILFE' AS series_code, 'Inflation' AS macro_factor, 'Core Inflation' AS sub_factor), + STRUCT('MEDCPIM158SFRBCLE' AS series_code, 'Inflation' AS macro_factor, 'Core Inflation' AS sub_factor), + STRUCT('CORESTICKM159SFRBATL' AS series_code, 'Inflation' AS macro_factor, 'Sticky Inflation' AS sub_factor), + STRUCT('STICKCPIM159SFRBATL' AS series_code, 'Inflation' AS macro_factor, 'Sticky Inflation' AS sub_factor), + STRUCT('PCETRIM12M159SFRBDAL' AS series_code, 'Inflation' AS macro_factor, 'Core Inflation' AS sub_factor), + STRUCT('T10YIE' AS series_code, 'Inflation' AS macro_factor, 'Inflation Expectations' AS sub_factor), + STRUCT('T5YIE' AS series_code, 'Inflation' AS macro_factor, 'Inflation Expectations' AS sub_factor), + STRUCT('T5YIFR' AS series_code, 'Inflation' AS macro_factor, 'Inflation Expectations' AS sub_factor), + STRUCT('PPIACO' AS series_code, 'Inflation' AS macro_factor, 'Producer Prices' AS sub_factor), + STRUCT('PPIFIS' AS series_code, 'Inflation' AS macro_factor, 'Producer Prices' AS sub_factor), + STRUCT('PPIFID' AS series_code, 'Inflation' AS macro_factor, 'Producer Prices' AS sub_factor), + STRUCT('CPIENGSL' AS series_code, 'Inflation' AS macro_factor, 'Energy Inflation' AS sub_factor), -- Employment factors - ('PAYEMS', 'Employment', 'Jobs'), - ('UNRATE', 'Employment', 'Unemployment'), - ('U6RATE', 'Employment', 'Unemployment'), - ('ICSA', 'Employment', 'Unemployment Claims'), - ('ICSA4WMA', 'Employment', 'Unemployment Claims'), - ('JTSJOL', 'Employment', 'Job Openings'), - ('JTSQUR', 'Employment', 'Job Turnover'), - ('CIVPART', 'Employment', 'Labor Participation'), - ('EMRATIO', 'Employment', 'Labor Participation'), - ('CE16OV', 'Employment', 'Jobs'), - ('MANEMP', 'Employment', 'Manufacturing Jobs'), - ('USCONS', 'Employment', 'Construction Jobs'), - ('AHETPI', 'Employment', 'Wages'), - ('ECIWAG', 'Employment', 'Wages'), + STRUCT('PAYEMS' AS series_code, 'Employment' AS macro_factor, 'Jobs' AS sub_factor), + STRUCT('UNRATE' AS series_code, 'Employment' AS macro_factor, 'Unemployment' AS sub_factor), + STRUCT('U6RATE' AS series_code, 'Employment' AS macro_factor, 'Unemployment' AS sub_factor), + STRUCT('ICSA' AS series_code, 'Employment' AS macro_factor, 'Unemployment Claims' AS sub_factor), + STRUCT('ICSA4WMA' AS series_code, 'Employment' AS macro_factor, 'Unemployment Claims' AS sub_factor), + STRUCT('JTSJOL' AS series_code, 'Employment' AS macro_factor, 'Job Openings' AS sub_factor), + STRUCT('JTSQUR' AS series_code, 'Employment' AS macro_factor, 'Job Turnover' AS sub_factor), + STRUCT('CIVPART' AS series_code, 'Employment' AS macro_factor, 'Labor Participation' AS sub_factor), + STRUCT('EMRATIO' AS series_code, 'Employment' AS macro_factor, 'Labor Participation' AS sub_factor), + STRUCT('CE16OV' AS series_code, 'Employment' AS macro_factor, 'Jobs' AS sub_factor), + STRUCT('MANEMP' AS series_code, 'Employment' AS macro_factor, 'Manufacturing Jobs' AS sub_factor), + STRUCT('USCONS' AS series_code, 'Employment' AS macro_factor, 'Construction Jobs' AS sub_factor), + STRUCT('AHETPI' AS series_code, 'Employment' AS macro_factor, 'Wages' AS sub_factor), + STRUCT('ECIWAG' AS series_code, 'Employment' AS macro_factor, 'Wages' AS sub_factor), -- Growth factors - ('GDP', 'Growth', 'GDP'), - ('GDPC1', 'Growth', 'GDP'), - ('GDPC96', 'Growth', 'GDP'), - ('A191RL1Q225SBEA', 'Growth', 'GDP'), - ('INDPRO', 'Growth', 'Industrial Production'), - ('IPMAN', 'Growth', 'Industrial Production'), - ('TCU', 'Growth', 'Capacity Utilization'), - ('CAPUTLG2211S', 'Growth', 'Capacity Utilization'), - ('RSXFS', 'Growth', 'Retail Sales'), - ('RRSFS', 'Growth', 'Retail Sales'), - ('PCE', 'Growth', 'Consumer Spending'), - ('PCEC96', 'Growth', 'Consumer Spending'), - ('CFNAI', 'Growth', 'Economic Activity'), - ('CFNAIMA3', 'Growth', 'Economic Activity'), - ('USSLIND', 'Growth', 'Leading Indicators'), + STRUCT('GDP' AS series_code, 'Growth' AS macro_factor, 'GDP' AS sub_factor), + STRUCT('GDPC1' AS series_code, 'Growth' AS macro_factor, 'GDP' AS sub_factor), + STRUCT('GDPC96' AS series_code, 'Growth' AS macro_factor, 'GDP' AS sub_factor), + STRUCT('A191RL1Q225SBEA' AS series_code, 'Growth' AS macro_factor, 'GDP' AS sub_factor), + STRUCT('INDPRO' AS series_code, 'Growth' AS macro_factor, 'Industrial Production' AS sub_factor), + STRUCT('IPMAN' AS series_code, 'Growth' AS macro_factor, 'Industrial Production' AS sub_factor), + STRUCT('TCU' AS series_code, 'Growth' AS macro_factor, 'Capacity Utilization' AS sub_factor), + STRUCT('CAPUTLG2211S' AS series_code, 'Growth' AS macro_factor, 'Capacity Utilization' AS sub_factor), + STRUCT('RSXFS' AS series_code, 'Growth' AS macro_factor, 'Retail Sales' AS sub_factor), + STRUCT('RRSFS' AS series_code, 'Growth' AS macro_factor, 'Retail Sales' AS sub_factor), + STRUCT('PCE' AS series_code, 'Growth' AS macro_factor, 'Consumer Spending' AS sub_factor), + STRUCT('PCEC96' AS series_code, 'Growth' AS macro_factor, 'Consumer Spending' AS sub_factor), + STRUCT('CFNAI' AS series_code, 'Growth' AS macro_factor, 'Economic Activity' AS sub_factor), + STRUCT('CFNAIMA3' AS series_code, 'Growth' AS macro_factor, 'Economic Activity' AS sub_factor), + STRUCT('USSLIND' AS series_code, 'Growth' AS macro_factor, 'Leading Indicators' AS sub_factor), -- Housing factors - ('HOUST', 'Housing', 'Housing Starts'), - ('HOUST1F', 'Housing', 'Housing Starts'), - ('PERMIT', 'Housing', 'Building Permits'), - ('NHSDPTS', 'Housing', 'Home Sales'), - ('EXHOSLUSM495S', 'Housing', 'Home Sales'), - ('CSUSHPISA', 'Housing', 'Home Prices'), - ('MSPUS', 'Housing', 'Home Prices'), - ('MORTGAGE30US', 'Housing', 'Mortgage Rates'), - ('MORTGAGE15US', 'Housing', 'Mortgage Rates'), + STRUCT('HOUST' AS series_code, 'Housing' AS macro_factor, 'Housing Starts' AS sub_factor), + STRUCT('HOUST1F' AS series_code, 'Housing' AS macro_factor, 'Housing Starts' AS sub_factor), + STRUCT('PERMIT' AS series_code, 'Housing' AS macro_factor, 'Building Permits' AS sub_factor), + STRUCT('NHSDPTS' AS series_code, 'Housing' AS macro_factor, 'Home Sales' AS sub_factor), + STRUCT('EXHOSLUSM495S' AS series_code, 'Housing' AS macro_factor, 'Home Sales' AS sub_factor), + STRUCT('CSUSHPISA' AS series_code, 'Housing' AS macro_factor, 'Home Prices' AS sub_factor), + STRUCT('MSPUS' AS series_code, 'Housing' AS macro_factor, 'Home Prices' AS sub_factor), + STRUCT('MORTGAGE30US' AS series_code, 'Housing' AS macro_factor, 'Mortgage Rates' AS sub_factor), + STRUCT('MORTGAGE15US' AS series_code, 'Housing' AS macro_factor, 'Mortgage Rates' AS sub_factor), -- Consumer Sentiment factors - ('UMCSENT', 'Consumer', 'Consumer Sentiment'), - ('CSCICP03USM665S', 'Consumer', 'Consumer Confidence'), - ('PSAVERT', 'Consumer', 'Savings Rate'), - ('DSPIC96', 'Consumer', 'Income'), - ('PI', 'Consumer', 'Income'), + STRUCT('UMCSENT' AS series_code, 'Consumer' AS macro_factor, 'Consumer Sentiment' AS sub_factor), + STRUCT('CSCICP03USM665S' AS series_code, 'Consumer' AS macro_factor, 'Consumer Confidence' AS sub_factor), + STRUCT('PSAVERT' AS series_code, 'Consumer' AS macro_factor, 'Savings Rate' AS sub_factor), + STRUCT('DSPIC96' AS series_code, 'Consumer' AS macro_factor, 'Income' AS sub_factor), + STRUCT('PI' AS series_code, 'Consumer' AS macro_factor, 'Income' AS sub_factor), -- Financial Conditions factors - ('DFF', 'Rates', 'Fed Funds'), - ('FEDFUNDS', 'Rates', 'Fed Funds'), - ('DGS10', 'Rates', 'Treasury Yields'), - ('TB10YR', 'Rates', 'Treasury Yields'), - ('TB2YR', 'Rates', 'Treasury Yields'), - ('T10Y2Y', 'Rates', 'Yield Curve'), - ('T10Y3M', 'Rates', 'Yield Curve'), - ('VIXCLS', 'Financial', 'Volatility'), - ('NFCI', 'Financial', 'Financial Conditions'), - ('NFCICREDIT', 'Financial', 'Credit Conditions'), - ('BAMLC0A0CM', 'Financial', 'Credit Spreads'), - ('BAMLH0A0HYM2', 'Financial', 'Credit Spreads'), - ('TEDRATE', 'Financial', 'Credit Spreads'), + STRUCT('DFF' AS series_code, 'Rates' AS macro_factor, 'Fed Funds' AS sub_factor), + STRUCT('FEDFUNDS' AS series_code, 'Rates' AS macro_factor, 'Fed Funds' AS sub_factor), + STRUCT('DGS10' AS series_code, 'Rates' AS macro_factor, 'Treasury Yields' AS sub_factor), + STRUCT('TB10YR' AS series_code, 'Rates' AS macro_factor, 'Treasury Yields' AS sub_factor), + STRUCT('TB2YR' AS series_code, 'Rates' AS macro_factor, 'Treasury Yields' AS sub_factor), + STRUCT('T10Y2Y' AS series_code, 'Rates' AS macro_factor, 'Yield Curve' AS sub_factor), + STRUCT('T10Y3M' AS series_code, 'Rates' AS macro_factor, 'Yield Curve' AS sub_factor), + STRUCT('VIXCLS' AS series_code, 'Financial' AS macro_factor, 'Volatility' AS sub_factor), + STRUCT('NFCI' AS series_code, 'Financial' AS macro_factor, 'Financial Conditions' AS sub_factor), + STRUCT('NFCICREDIT' AS series_code, 'Financial' AS macro_factor, 'Credit Conditions' AS sub_factor), + STRUCT('BAMLC0A0CM' AS series_code, 'Financial' AS macro_factor, 'Credit Spreads' AS sub_factor), + STRUCT('BAMLH0A0HYM2' AS series_code, 'Financial' AS macro_factor, 'Credit Spreads' AS sub_factor), + STRUCT('TEDRATE' AS series_code, 'Financial' AS macro_factor, 'Credit Spreads' AS sub_factor), -- Business Activity factors - ('IPMAN', 'Business', 'Manufacturing Production'), - ('NEWORDER', 'Business', 'Manufacturing Orders'), - ('MANEMP', 'Business', 'Manufacturing Employment'), - ('BPEA', 'Business', 'Business Outlook'), - ('GACDISA066MSFRBNY', 'Business', 'Regional Surveys') - ) AS factor_mapping (series_code, macro_factor, sub_factor) + STRUCT('IPMAN' AS series_code, 'Business' AS macro_factor, 'Manufacturing Production' AS sub_factor), + STRUCT('NEWORDER' AS series_code, 'Business' AS macro_factor, 'Manufacturing Orders' AS sub_factor), + STRUCT('MANEMP' AS series_code, 'Business' AS macro_factor, 'Manufacturing Employment' AS sub_factor), + STRUCT('BPEA' AS series_code, 'Business' AS macro_factor, 'Business Outlook' AS sub_factor), + STRUCT('GACDISA066MSFRBNY' AS series_code, 'Business' AS macro_factor, 'Regional Surveys' AS sub_factor) + ]) ), -- Get sector sensitivity scores with factor mapping diff --git a/dbt_project/models/analysis/reddit_sentiment_trends.sql b/dbt_project/models/analysis/reddit_sentiment_trends.sql index 1be5424..f0c6bcd 100644 --- a/dbt_project/models/analysis/reddit_sentiment_trends.sql +++ b/dbt_project/models/analysis/reddit_sentiment_trends.sql @@ -6,41 +6,39 @@ with daily_engagement as ( select - partition_date, + safe_cast(partition_date as date) as partition_date, subreddit, count(*) as num_posts, avg(score) as avg_score, avg(num_comments) as avg_comments, sum(num_comments) as total_comments, sum(engagement_score) as total_engagement, - approx_percentile(score, 0.5) as median_score, - approx_percentile(score, 0.75) as p75_score, - approx_percentile(score, 0.90) as p90_score, + approx_quantiles(score, 100)[offset(50)] as median_score, + approx_quantiles(score, 100)[offset(75)] as p75_score, + approx_quantiles(score, 100)[offset(90)] as p90_score, max(score) as max_score, max(num_comments) as max_comments, avg(case when is_self_post then 1.0 else 0.0 end) as self_post_ratio, avg(case when is_deleted then 1.0 else 0.0 end) as deleted_post_ratio from {{ ref('stg_reddit_posts') }} - group by partition_date, subreddit + group by safe_cast(partition_date as date), subreddit ), daily_sentiment as ( select - partition_date, + safe_cast(partition_date as date) as partition_date, subreddit, count(*) as total_scored, avg(compound_score) as avg_compound, avg(case when content_type like 'post%' then compound_score end) as avg_post_sentiment, avg(case when content_type = 'comment' then compound_score end) as avg_comment_sentiment, - sum(case when sentiment_label = 'positive' then 1 else 0 end) / 1.0 - / nullif(count(*), 0) * 100 as pct_positive, - sum(case when sentiment_label = 'negative' then 1 else 0 end) / 1.0 - / nullif(count(*), 0) * 100 as pct_negative, + safe_divide(sum(case when sentiment_label = 'positive' then 1 else 0 end), count(*)) * 100 as pct_positive, + safe_divide(sum(case when sentiment_label = 'negative' then 1 else 0 end), count(*)) * 100 as pct_negative, avg(sentiment_intensity) as avg_intensity, sum(case when sentiment_strength = 'very_positive' then 1 else 0 end) as very_positive_count, sum(case when sentiment_strength = 'very_negative' then 1 else 0 end) as very_negative_count from {{ ref('stg_reddit_sentiment') }} - group by partition_date, subreddit + group by safe_cast(partition_date as date), subreddit ), combined as ( @@ -94,17 +92,17 @@ with_momentum as ( *, case when weekly_avg_score > 0 - then ((avg_score - weekly_avg_score) / weekly_avg_score) * 100 + then safe_divide(avg_score - weekly_avg_score, weekly_avg_score) * 100 else 0 end as score_momentum_pct, case when weekly_avg_comments > 0 - then ((avg_comments - weekly_avg_comments) / weekly_avg_comments) * 100 + then safe_divide(avg_comments - weekly_avg_comments, weekly_avg_comments) * 100 else 0 end as comments_momentum_pct, case when weekly_avg_posts > 0 - then ((num_posts - weekly_avg_posts) / weekly_avg_posts) * 100 + then safe_divide(num_posts - weekly_avg_posts, weekly_avg_posts) * 100 else 0 end as activity_momentum_pct, -- Sentiment momentum diff --git a/dbt_project/models/analysis/schema.yml b/dbt_project/models/analysis/schema.yml index a103457..0d02877 100644 --- a/dbt_project/models/analysis/schema.yml +++ b/dbt_project/models/analysis/schema.yml @@ -100,11 +100,11 @@ models: - name: score_momentum_pct description: "Percentage change from 7-day average score" - name: sentiment_trend - description: "Overall sentiment direction based on score and comment trends" + description: "Overall sentiment classification based on score and compound sentiment trends" tests: - accepted_values: arguments: - values: ['increasing', 'decreasing', 'stable'] + values: ['bullish', 'bearish', 'positive', 'negative', 'neutral'] - name: sector_indicator_sensitivity description: > @@ -244,7 +244,7 @@ models: macro sensitivity data at the individual stock level. Enables watchlist impact analysis by linking stocks to their sector's economic indicator sensitivity scores. columns: - - name: symbol + - name: ticker description: "Stock ticker symbol" tests: - not_null @@ -276,11 +276,11 @@ models: - name: top_sensitive_indicators description: "Top 5 economic indicators the sector is most sensitive to" - name: sector_type - description: "Whether the sector is Cyclical or Defensive based on regime performance" + description: "Whether the sector is Cyclical, Defensive, or Interest-Sensitive based on regime performance" tests: - accepted_values: arguments: - values: ['Cyclical', 'Defensive'] + values: ['Cyclical', 'Defensive', 'Interest-Sensitive'] - name: portfolio_macro_factors description: > diff --git a/dbt_project/models/analysis/sector_indicator_sensitivity.sql b/dbt_project/models/analysis/sector_indicator_sensitivity.sql index 8c24e2c..657ed53 100644 --- a/dbt_project/models/analysis/sector_indicator_sensitivity.sql +++ b/dbt_project/models/analysis/sector_indicator_sensitivity.sql @@ -6,50 +6,50 @@ -- Sector-to-name mapping for readable output WITH sector_names AS ( - SELECT sector_names.* - FROM (VALUES - ('XLK', 'Technology'), - ('XLC', 'Communication Services'), - ('XLY', 'Consumer Discretionary'), - ('XLF', 'Financial'), - ('XLI', 'Industrial'), - ('XLU', 'Utilities'), - ('XLP', 'Consumer Staples'), - ('XLRE', 'Real Estate'), - ('XLB', 'Materials'), - ('XLE', 'Energy'), - ('XLV', 'Health Care') - ) AS sector_names (symbol, sector_name) + SELECT * + FROM UNNEST([ + STRUCT('XLK' AS symbol, 'Technology' AS sector_name), + STRUCT('XLC' AS symbol, 'Communication Services' AS sector_name), + STRUCT('XLY' AS symbol, 'Consumer Discretionary' AS sector_name), + STRUCT('XLF' AS symbol, 'Financial' AS sector_name), + STRUCT('XLI' AS symbol, 'Industrial' AS sector_name), + STRUCT('XLU' AS symbol, 'Utilities' AS sector_name), + STRUCT('XLP' AS symbol, 'Consumer Staples' AS sector_name), + STRUCT('XLRE' AS symbol, 'Real Estate' AS sector_name), + STRUCT('XLB' AS symbol, 'Materials' AS sector_name), + STRUCT('XLE' AS symbol, 'Energy' AS sector_name), + STRUCT('XLV' AS symbol, 'Health Care' AS sector_name) + ]) ), -- Get monthly sector returns (last trading day of each month) sector_monthly AS ( SELECT symbol, - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, -- Use the last available data point for each month LAST_VALUE(pct_change_1mo) OVER ( - PARTITION BY symbol, DATE_TRUNC('month', date) + PARTITION BY symbol, DATE_TRUNC(date, MONTH) ORDER BY date ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS return_1mo, LAST_VALUE(pct_change_3mo) OVER ( - PARTITION BY symbol, DATE_TRUNC('month', date) + PARTITION BY symbol, DATE_TRUNC(date, MONTH) ORDER BY date ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS return_3mo, LAST_VALUE(pct_change_6mo) OVER ( - PARTITION BY symbol, DATE_TRUNC('month', date) + PARTITION BY symbol, DATE_TRUNC(date, MONTH) ORDER BY date ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS return_6mo, LAST_VALUE(pct_change_1yr) OVER ( - PARTITION BY symbol, DATE_TRUNC('month', date) + PARTITION BY symbol, DATE_TRUNC(date, MONTH) ORDER BY date ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS return_12mo, ROW_NUMBER() OVER ( - PARTITION BY symbol, DATE_TRUNC('month', date) + PARTITION BY symbol, DATE_TRUNC(date, MONTH) ORDER BY date DESC ) AS rn FROM {{ ref('us_sector_analysis_return') }} @@ -73,7 +73,7 @@ indicator_monthly AS ( SELECT series_code, series_name, - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, value, -- Calculate month-over-month percentage change CASE diff --git a/dbt_project/models/analysis/sector_regime_performance.sql b/dbt_project/models/analysis/sector_regime_performance.sql index dd4c53e..4608dc4 100644 --- a/dbt_project/models/analysis/sector_regime_performance.sql +++ b/dbt_project/models/analysis/sector_regime_performance.sql @@ -11,20 +11,20 @@ WITH sector_monthly AS ( -- Get monthly sector returns SELECT symbol, - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, -- Use the last day's data for each month LAST_VALUE(pct_change_1mo) OVER ( - PARTITION BY symbol, DATE_TRUNC('month', date) + PARTITION BY symbol, DATE_TRUNC(date, MONTH) ORDER BY date ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS monthly_return, LAST_VALUE(pct_change_3mo) OVER ( - PARTITION BY symbol, DATE_TRUNC('month', date) + PARTITION BY symbol, DATE_TRUNC(date, MONTH) ORDER BY date ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS return_3mo, ROW_NUMBER() OVER ( - PARTITION BY symbol, DATE_TRUNC('month', date) + PARTITION BY symbol, DATE_TRUNC(date, MONTH) ORDER BY date DESC ) AS rn FROM {{ ref('us_sector_analysis_return') }} @@ -43,20 +43,20 @@ sector_returns AS ( -- Sector name mapping sector_names AS ( - SELECT sector_names.* - FROM (VALUES - ('XLK', 'Technology', 'Cyclical'), - ('XLC', 'Communication Services', 'Cyclical'), - ('XLY', 'Consumer Discretionary', 'Cyclical'), - ('XLF', 'Financial', 'Cyclical'), - ('XLI', 'Industrial', 'Cyclical'), - ('XLU', 'Utilities', 'Defensive'), - ('XLP', 'Consumer Staples', 'Defensive'), - ('XLRE', 'Real Estate', 'Interest-Sensitive'), - ('XLB', 'Materials', 'Cyclical'), - ('XLE', 'Energy', 'Cyclical'), - ('XLV', 'Health Care', 'Defensive') - ) AS sector_names (symbol, sector_name, sector_type) + SELECT * + FROM UNNEST([ + STRUCT('XLK' AS symbol, 'Technology' AS sector_name, 'Cyclical' AS sector_type), + STRUCT('XLC' AS symbol, 'Communication Services' AS sector_name, 'Cyclical' AS sector_type), + STRUCT('XLY' AS symbol, 'Consumer Discretionary' AS sector_name, 'Cyclical' AS sector_type), + STRUCT('XLF' AS symbol, 'Financial' AS sector_name, 'Cyclical' AS sector_type), + STRUCT('XLI' AS symbol, 'Industrial' AS sector_name, 'Cyclical' AS sector_type), + STRUCT('XLU' AS symbol, 'Utilities' AS sector_name, 'Defensive' AS sector_type), + STRUCT('XLP' AS symbol, 'Consumer Staples' AS sector_name, 'Defensive' AS sector_type), + STRUCT('XLRE' AS symbol, 'Real Estate' AS sector_name, 'Interest-Sensitive' AS sector_type), + STRUCT('XLB' AS symbol, 'Materials' AS sector_name, 'Cyclical' AS sector_type), + STRUCT('XLE' AS symbol, 'Energy' AS sector_name, 'Cyclical' AS sector_type), + STRUCT('XLV' AS symbol, 'Health Care' AS sector_name, 'Defensive' AS sector_type) + ]) ), -- Join sector returns with regime classification @@ -89,11 +89,11 @@ regime_performance AS ( ROUND(AVG(monthly_return), 2) AS avg_monthly_return, ROUND(STDDEV(monthly_return), 2) AS return_volatility, ROUND(AVG(monthly_return) / NULLIF(STDDEV(monthly_return), 0), 2) AS sharpe_proxy, - ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY monthly_return), 2) AS median_return, + ROUND(APPROX_QUANTILES(monthly_return, 100)[OFFSET(50)], 2) AS median_return, ROUND(MIN(monthly_return), 2) AS worst_month, ROUND(MAX(monthly_return), 2) AS best_month, SUM(CASE WHEN monthly_return > 0 THEN 1 ELSE 0 END) AS positive_months, - ROUND(SUM(CASE WHEN monthly_return > 0 THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS win_rate + ROUND(SAFE_DIVIDE(SUM(CASE WHEN monthly_return > 0 THEN 1 ELSE 0 END) * 100.0, COUNT(*)), 1) AS win_rate FROM sector_regime_data GROUP BY symbol, sector_name, sector_type, regime ), @@ -109,11 +109,11 @@ overall_performance AS ( ROUND(AVG(monthly_return), 2) AS avg_monthly_return, ROUND(STDDEV(monthly_return), 2) AS return_volatility, ROUND(AVG(monthly_return) / NULLIF(STDDEV(monthly_return), 0), 2) AS sharpe_proxy, - ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY monthly_return), 2) AS median_return, + ROUND(APPROX_QUANTILES(monthly_return, 100)[OFFSET(50)], 2) AS median_return, ROUND(MIN(monthly_return), 2) AS worst_month, ROUND(MAX(monthly_return), 2) AS best_month, SUM(CASE WHEN monthly_return > 0 THEN 1 ELSE 0 END) AS positive_months, - ROUND(SUM(CASE WHEN monthly_return > 0 THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS win_rate + ROUND(SAFE_DIVIDE(SUM(CASE WHEN monthly_return > 0 THEN 1 ELSE 0 END) * 100.0, COUNT(*)), 1) AS win_rate FROM sector_regime_data GROUP BY symbol, sector_name, sector_type ), diff --git a/dbt_project/models/analysis/ticker_sector_sensitivity.sql b/dbt_project/models/analysis/ticker_sector_sensitivity.sql index 318c88a..90101f6 100644 --- a/dbt_project/models/analysis/ticker_sector_sensitivity.sql +++ b/dbt_project/models/analysis/ticker_sector_sensitivity.sql @@ -9,20 +9,20 @@ -- Mapping from GICS sector names to sector ETF symbols WITH sector_etf_mapping AS ( - SELECT sector_etf_mapping.* - FROM (VALUES - ('Information Technology', 'XLK', 'Technology'), - ('Communication Services', 'XLC', 'Communication Services'), - ('Consumer Discretionary', 'XLY', 'Consumer Discretionary'), - ('Financials', 'XLF', 'Financial'), - ('Industrials', 'XLI', 'Industrial'), - ('Utilities', 'XLU', 'Utilities'), - ('Consumer Staples', 'XLP', 'Consumer Staples'), - ('Real Estate', 'XLRE', 'Real Estate'), - ('Materials', 'XLB', 'Materials'), - ('Energy', 'XLE', 'Energy'), - ('Health Care', 'XLV', 'Health Care') - ) AS sector_etf_mapping (gics_sector, etf_symbol, sector_display_name) + SELECT * + FROM UNNEST([ + STRUCT('Information Technology' AS gics_sector, 'XLK' AS etf_symbol, 'Technology' AS sector_display_name), + STRUCT('Communication Services', 'XLC', 'Communication Services'), + STRUCT('Consumer Discretionary', 'XLY', 'Consumer Discretionary'), + STRUCT('Financials', 'XLF', 'Financial'), + STRUCT('Industrials', 'XLI', 'Industrial'), + STRUCT('Utilities', 'XLU', 'Utilities'), + STRUCT('Consumer Staples', 'XLP', 'Consumer Staples'), + STRUCT('Real Estate', 'XLRE', 'Real Estate'), + STRUCT('Materials', 'XLB', 'Materials'), + STRUCT('Energy', 'XLE', 'Energy'), + STRUCT('Health Care', 'XLV', 'Health Care') + ]) ), -- Get S&P 500 company metadata with sector info @@ -86,8 +86,8 @@ sector_sensitivity_agg AS ( COUNT(*) AS n_sensitive_indicators, AVG(sensitivity_score) AS avg_sensitivity_score, MAX(sensitivity_score) AS max_sensitivity_score, - STRING_AGG(DISTINCT series_code, ', ' ORDER BY sensitivity_score DESC) AS top_indicator_codes, - STRING_AGG(DISTINCT series_name, '; ' ORDER BY sensitivity_score DESC) AS top_indicator_names + STRING_AGG(series_code, ', ' ORDER BY sensitivity_score DESC) AS top_indicator_codes, + STRING_AGG(series_name, '; ' ORDER BY sensitivity_score DESC) AS top_indicator_names FROM sector_top_indicators GROUP BY sector_etf ) diff --git a/dbt_project/models/analytics/telemetry/error_analysis.sql b/dbt_project/models/analytics/telemetry/error_analysis.sql deleted file mode 100644 index c1b1aef..0000000 --- a/dbt_project/models/analytics/telemetry/error_analysis.sql +++ /dev/null @@ -1,80 +0,0 @@ -with error_events as ( - select * from {{ ref('stg_telemetry_events') }} - where is_error_event -), - -daily_error_summary as ( - select - event_date as date, - event_type, - coalesce(error_message, 'unknown_error') as error_message, - coalesce(error_code, 'no_code') as error_code, - coalesce(page_path, 'unknown_page') as page_path, - - count(*) as error_count, - count(distinct session_id) as affected_sessions, - count(distinct if(user_id is not null, user_id, null)) as affected_users, - - min(event_timestamp) as first_occurrence, - max(event_timestamp) as last_occurrence - - from error_events - group by - event_date, - event_type, - error_message, - error_code, - page_path -), - -session_context as ( - select - session_date as date, - count(distinct session_id) as total_sessions - from {{ ref('stg_sessions') }} - group by session_date -) - -select - es.date, - es.event_type, - es.error_message, - es.error_code, - es.page_path, - es.error_count, - es.affected_sessions, - es.affected_users, - es.first_occurrence, - es.last_occurrence, - - timestamp_diff(es.last_occurrence, es.first_occurrence, second) / 3600.0 - as hours_between_first_and_last, - - round( - cast(es.affected_sessions as float64) / nullif(sc.total_sessions, 0) * 100, - 2 - ) as session_error_rate_pct, - - round( - cast(es.error_count as float64) / nullif(es.affected_sessions, 0), - 2 - ) as avg_errors_per_affected_session, - - case - when es.error_count > 100 then 'critical' - when es.error_count > 50 then 'high' - when es.error_count > 10 then 'medium' - else 'low' - end as severity, - - case - when es.error_count > 100 and es.affected_sessions > 20 then 'high_impact' - when es.error_count > 50 or es.affected_sessions > 10 then 'medium_impact' - else 'low_impact' - end as impact_level - -from daily_error_summary as es -left join session_context as sc - on es.date = sc.date - -order by es.date desc, es.error_count desc diff --git a/dbt_project/models/analytics/telemetry/feature_usage.sql b/dbt_project/models/analytics/telemetry/feature_usage.sql deleted file mode 100644 index 19a9443..0000000 --- a/dbt_project/models/analytics/telemetry/feature_usage.sql +++ /dev/null @@ -1,57 +0,0 @@ -with events as ( - select * from {{ ref('stg_telemetry_events') }} - where is_feature_usage_event -), - -daily_feature_usage as ( - select - event_date as date, - event_type, - coalesce(feature_name, 'unspecified') as feature, - - count(*) as event_count, - count(distinct session_id) as unique_sessions, - count(distinct if(user_id is not null, user_id, null)) as unique_users, - - count(distinct if(chart_type is not null, chart_type, null)) - as unique_chart_types_used, - countif(export_format is not null) as exports_with_format, - - avg(if(query_text is not null, length(query_text), null)) as avg_query_length, - countif(query_text is not null) as queries_with_text - - from events - group by event_date, event_type, feature -) - -select - date, - event_type, - feature, - event_count, - unique_sessions, - unique_users, - - unique_chart_types_used, - exports_with_format, - - queries_with_text, - round(cast(event_count as float64) / nullif(unique_sessions, 0), 2) - as events_per_session, - - round(cast(event_count as float64) / nullif(unique_users, 0), 2) - as events_per_user, - round(avg_query_length, 0) as avg_query_length_chars, - - round( - cast(unique_sessions as float64) - / nullif(( - select count(distinct session_id) from {{ ref('stg_sessions') }} - where session_date = daily_feature_usage.date - ), 0) - * 100, - 2 - ) as session_penetration_pct - -from daily_feature_usage -order by date desc, event_count desc diff --git a/dbt_project/models/analytics/telemetry/performance_metrics.sql b/dbt_project/models/analytics/telemetry/performance_metrics.sql deleted file mode 100644 index 19ddee4..0000000 --- a/dbt_project/models/analytics/telemetry/performance_metrics.sql +++ /dev/null @@ -1,127 +0,0 @@ -with web_vitals_events as ( - select - event_date, - event_timestamp, - page_path, - session_id, - user_id, - metric_name, - metric_value, - metric_rating - from {{ ref('stg_telemetry_events') }} - where - is_performance_event - and metric_name is not null - and metric_value is not null -), - -daily_metrics as ( - select - event_date as date, - metric_name, - coalesce(page_path, 'unknown_page') as page, - - count(*) as sample_size, - avg(metric_value) as avg_value, - approx_quantiles(metric_value, 100)[offset(50)] as p50_value, - approx_quantiles(metric_value, 100)[offset(75)] as p75_value, - approx_quantiles(metric_value, 100)[offset(95)] as p95_value, - approx_quantiles(metric_value, 100)[offset(99)] as p99_value, - min(metric_value) as min_value, - max(metric_value) as max_value, - - countif(metric_rating = 'good') as good_count, - countif(metric_rating = 'needs-improvement') as needs_improvement_count, - countif(metric_rating = 'poor') as poor_count, - - count(distinct session_id) as unique_sessions, - count(distinct if(user_id is not null, user_id, null)) as unique_users - - from web_vitals_events - group by event_date, page, metric_name -) - -select - date, - page, - metric_name, - sample_size, - - good_count, - needs_improvement_count, - poor_count, - unique_sessions, - unique_users, - round(avg_value, 2) as avg_value, - round(p50_value, 2) as p50_value, - - round(p75_value, 2) as p75_value, - round(p95_value, 2) as p95_value, - round(p99_value, 2) as p99_value, - - round(min_value, 2) as min_value, - round(max_value, 2) as max_value, - round(cast(good_count as float64) / nullif(sample_size, 0) * 100, 2) - as good_pct, - - round(cast(needs_improvement_count as float64) / nullif(sample_size, 0) * 100, 2) - as needs_improvement_pct, - round(cast(poor_count as float64) / nullif(sample_size, 0) * 100, 2) - as poor_pct, - - case - when metric_name = 'LCP' - then - case - when p75_value <= 2500 then 'good' - when p75_value <= 4000 then 'needs-improvement' - else 'poor' - end - when metric_name = 'FID' - then - case - when p75_value <= 100 then 'good' - when p75_value <= 300 then 'needs-improvement' - else 'poor' - end - when metric_name = 'CLS' - then - case - when p75_value <= 0.1 then 'good' - when p75_value <= 0.25 then 'needs-improvement' - else 'poor' - end - when metric_name = 'FCP' - then - case - when p75_value <= 1800 then 'good' - when p75_value <= 3000 then 'needs-improvement' - else 'poor' - end - when metric_name = 'TTFB' - then - case - when p75_value <= 800 then 'good' - when p75_value <= 1800 then 'needs-improvement' - else 'poor' - end - when metric_name = 'INP' - then - case - when p75_value <= 200 then 'good' - when p75_value <= 500 then 'needs-improvement' - else 'poor' - end - else 'unknown' - end as overall_rating, - - case - when p95_value > (avg_value * 2) then 'high_variability' - when p95_value > (avg_value * 1.5) then 'moderate_variability' - else 'low_variability' - end as variability_level - -from daily_metrics -where sample_size >= 5 - -order by date desc, metric_name asc, page asc diff --git a/dbt_project/models/analytics/telemetry/schema.yml b/dbt_project/models/analytics/telemetry/schema.yml deleted file mode 100644 index 6049d93..0000000 --- a/dbt_project/models/analytics/telemetry/schema.yml +++ /dev/null @@ -1,107 +0,0 @@ -version: 2 - -models: - - name: session_summary - description: Daily aggregated session metrics for tracking engagement and usage patterns - columns: - - name: date - description: Date of the session metrics - tests: - - not_null - - unique - - name: total_sessions - description: Total number of sessions on this date - - name: bounce_rate_pct - description: Percentage of sessions with only one page view - - name: avg_engagement_score - description: Average engagement score (0-7 scale) - - name: chat_usage_rate_pct - description: Percentage of sessions that used chat feature - - - name: feature_usage - description: Daily feature usage metrics showing adoption and engagement per feature - columns: - - name: date - description: Date of the feature usage - tests: - - not_null - - name: event_type - description: Type of feature event - tests: - - not_null - - name: feature - description: Name of the feature used - - name: event_count - description: Number of times feature was used - - name: unique_sessions - description: Number of unique sessions using the feature - - name: session_penetration_pct - description: Percentage of total sessions that used this feature - - - name: user_journeys - description: Page transition analysis showing common user paths and drop-off points - columns: - - name: current_page - description: Starting page of the transition - tests: - - not_null - - name: next_page - description: Destination page of the transition - tests: - - not_null - - name: transition_count - description: Number of times this transition occurred - - name: transition_rate_pct - description: Percentage of current page visitors who made this transition - - name: dropoff_category - description: Drop-off severity category (high/medium/low) - - - name: error_analysis - description: Error tracking and analysis for identifying and prioritizing issues - columns: - - name: date - description: Date of the error occurrence - tests: - - not_null - - name: event_type - description: Type of error (api_error or app_error) - tests: - - not_null - - name: error_message - description: Error message text - - name: error_code - description: Error code if available - - name: page_path - description: Page where error occurred - - name: error_count - description: Number of times this error occurred - - name: affected_sessions - description: Number of sessions affected by this error - - name: severity - description: Error severity level (critical/high/medium/low) - - name: impact_level - description: Impact level based on frequency and reach (high/medium/low) - - - name: performance_metrics - description: Web Vitals and performance metrics analysis by page - columns: - - name: date - description: Date of the performance measurements - tests: - - not_null - - name: page - description: Page path being measured - tests: - - not_null - - name: metric_name - description: Name of the performance metric (LCP, FID, CLS, etc.) - tests: - - not_null - - name: p75_value - description: 75th percentile value (key metric for Web Vitals) - - name: overall_rating - description: Overall performance rating based on Web Vitals thresholds (good/needs-improvement/poor) - - name: sample_size - description: Number of measurements - - name: variability_level - description: Performance consistency indicator (high/moderate/low variability) diff --git a/dbt_project/models/analytics/telemetry/session_summary.sql b/dbt_project/models/analytics/telemetry/session_summary.sql deleted file mode 100644 index a1e6628..0000000 --- a/dbt_project/models/analytics/telemetry/session_summary.sql +++ /dev/null @@ -1,117 +0,0 @@ -with sessions as ( - select * from {{ ref('stg_sessions') }} -), - -daily_summary as ( - select - session_date as date, - - count(distinct session_id) as total_sessions, - count(distinct if(user_id is not null, user_id, null)) - as unique_authenticated_users, - count(distinct if(is_authenticated, session_id, null)) - as authenticated_sessions, - count(distinct if(not is_authenticated, session_id, null)) - as anonymous_sessions, - - avg(session_duration_seconds) as avg_duration_seconds, - avg(session_duration_minutes) as avg_duration_minutes, - approx_quantiles(session_duration_seconds, 100)[offset(50)] - as median_duration_seconds, - approx_quantiles(session_duration_seconds, 100)[offset(75)] - as p75_duration_seconds, - approx_quantiles(session_duration_seconds, 100)[offset(95)] - as p95_duration_seconds, - - avg(total_events) as avg_events_per_session, - avg(page_views) as avg_page_views_per_session, - avg(feature_interactions) as avg_feature_interactions_per_session, - avg(unique_pages_visited) as avg_unique_pages_per_session, - avg(engagement_score) as avg_engagement_score, - - countif(is_bounce) / cast(nullif(count(*), 0) as float64) as bounce_rate, - countif(has_errors) / cast(nullif(count(*), 0) as float64) as error_rate, - countif(used_chat) / cast(nullif(count(*), 0) as float64) as chat_usage_rate, - countif(exported_chart) / cast(nullif(count(*), 0) as float64) as export_rate, - countif(is_authenticated) / cast(nullif(count(*), 0) as float64) - as authenticated_session_rate, - - countif(is_signup_session) as signup_sessions, - countif(is_signin_session) as signin_sessions, - - sum(page_view_count) as total_page_views, - sum(chat_query_count) as total_chat_queries, - sum(chart_export_count) as total_chart_exports, - sum(filter_applied_count) as total_filter_applications, - sum(api_error_count) as total_api_errors, - sum(app_error_count) as total_app_errors, - - countif(engagement_score >= 5) as high_engagement_sessions, - countif(engagement_score >= 3 and engagement_score < 5) - as medium_engagement_sessions, - countif(engagement_score < 3) as low_engagement_sessions - - from sessions - group by session_date -) - -select - date, - - total_sessions, - unique_authenticated_users, - authenticated_sessions, - anonymous_sessions, - - signup_sessions, - signin_sessions, - total_page_views, - total_chat_queries, - total_chart_exports, - total_filter_applications, - - total_api_errors, - total_app_errors, - high_engagement_sessions, - medium_engagement_sessions, - low_engagement_sessions, - - case - when unique_authenticated_users > 0 - then cast(total_sessions as float64) / unique_authenticated_users - end as avg_sessions_per_user, - round(avg_duration_seconds, 2) as avg_duration_seconds, - round(avg_duration_minutes, 2) as avg_duration_minutes, - round(median_duration_seconds, 2) as median_duration_seconds, - round(p75_duration_seconds, 2) as p75_duration_seconds, - - round(p95_duration_seconds, 2) as p95_duration_seconds, - round(avg_events_per_session, 2) as avg_events_per_session, - - round(avg_page_views_per_session, 2) as avg_page_views_per_session, - round(avg_feature_interactions_per_session, 2) - as avg_feature_interactions_per_session, - round(avg_unique_pages_per_session, 2) as avg_unique_pages_per_session, - round(avg_engagement_score, 2) as avg_engagement_score, - round(bounce_rate * 100, 2) as bounce_rate_pct, - round(error_rate * 100, 2) as error_rate_pct, - - round(chat_usage_rate * 100, 2) as chat_usage_rate_pct, - round(export_rate * 100, 2) as export_rate_pct, - round(authenticated_session_rate * 100, 2) as authenticated_session_rate_pct, - - round( - cast(high_engagement_sessions as float64) / nullif(total_sessions, 0) * 100, - 2 - ) as high_engagement_pct, - round( - cast(medium_engagement_sessions as float64) / nullif(total_sessions, 0) * 100, - 2 - ) as medium_engagement_pct, - round( - cast(low_engagement_sessions as float64) / nullif(total_sessions, 0) * 100, - 2 - ) as low_engagement_pct - -from daily_summary -order by date desc diff --git a/dbt_project/models/analytics/telemetry/user_journeys.sql b/dbt_project/models/analytics/telemetry/user_journeys.sql deleted file mode 100644 index 452f49b..0000000 --- a/dbt_project/models/analytics/telemetry/user_journeys.sql +++ /dev/null @@ -1,86 +0,0 @@ -with page_views as ( - select - session_id, - event_timestamp, - page_path, - row_number() over (partition by session_id order by event_timestamp) as step_number - from {{ ref('stg_telemetry_events') }} - where is_navigation_event and page_path is not null -), - -page_transitions as ( - select - pv1.session_id, - pv1.page_path as current_page, - pv2.page_path as next_page, - pv1.step_number, - timestamp_diff(pv2.event_timestamp, pv1.event_timestamp, second) - as time_to_next_seconds - from page_views as pv1 - left join page_views as pv2 - on - pv1.session_id = pv2.session_id - and pv2.step_number = pv1.step_number + 1 -), - -transition_aggregates as ( - select - current_page, - next_page, - count(*) as transition_count, - count(distinct session_id) as unique_sessions, - avg(time_to_next_seconds) as avg_time_to_next_seconds, - approx_quantiles(time_to_next_seconds, 100)[offset(50)] - as median_time_to_next_seconds, - - avg(step_number) as avg_step_in_journey - - from page_transitions - where next_page is not null - group by current_page, next_page -), - -page_metrics as ( - select - page_path, - count(*) as total_views, - count(distinct session_id) as unique_sessions_viewing - from page_views - group by page_path -) - -select - ta.current_page, - ta.next_page, - ta.transition_count, - ta.unique_sessions, - - pm_current.total_views as current_page_total_views, - pm_next.total_views as next_page_total_views, - round(ta.avg_time_to_next_seconds, 2) as avg_time_to_next_seconds, - - round(ta.median_time_to_next_seconds, 2) as median_time_to_next_seconds, - - round(ta.avg_step_in_journey, 1) as avg_step_in_journey, - round( - cast(ta.transition_count as float64) / nullif(pm_current.total_views, 0) * 100, - 2 - ) as transition_rate_pct, - - case - when cast(ta.transition_count as float64) - / nullif(pm_current.total_views, 0) * 100 < 20 then 'high_dropoff' - when cast(ta.transition_count as float64) - / nullif(pm_current.total_views, 0) * 100 < 50 then 'medium_dropoff' - else 'low_dropoff' - end as dropoff_category - -from transition_aggregates as ta -left join page_metrics as pm_current - on ta.current_page = pm_current.page_path -left join page_metrics as pm_next - on ta.next_page = pm_next.page_path - -where ta.transition_count >= 10 - -order by ta.transition_count desc diff --git a/dbt_project/models/backtesting/agriculture_commodities_summary_snapshot.sql b/dbt_project/models/backtesting/agriculture_commodities_summary_snapshot.sql index 9b128d2..7d18622 100644 --- a/dbt_project/models/backtesting/agriculture_commodities_summary_snapshot.sql +++ b/dbt_project/models/backtesting/agriculture_commodities_summary_snapshot.sql @@ -1,14 +1,14 @@ {{ config( unique_key=['snapshot_date', 'commodity_name', 'commodity_unit', 'time_period'], - incremental_strategy='delete+insert' + incremental_strategy='merge' ) }} WITH snapshot_dates AS ( - SELECT DISTINCT DATE_TRUNC('month', date) AS snapshot_date + SELECT DISTINCT DATE_TRUNC(date, MONTH) AS snapshot_date FROM {{ ref('stg_agriculture_commodities') }} WHERE date >= '2020-01-01' {% if is_incremental() %} - AND DATE_TRUNC('month', date) >= COALESCE( + AND DATE_TRUNC(date, MONTH) >= COALESCE( (SELECT MAX(snapshot_date) FROM {{ this }}), DATE '1900-01-01' ) - INTERVAL 1 MONTH @@ -49,8 +49,8 @@ base_data AS ( price IS NOT NULL AND date IS NOT NULL AND price > 0 - AND trade_date <= sd.snapshot_date - AND trade_date >= sd.snapshot_date - INTERVAL 5 YEAR + AND CAST(date AS DATE) <= sd.snapshot_date + AND CAST(date AS DATE) >= sd.snapshot_date - INTERVAL 5 YEAR ), date_boundaries AS ( diff --git a/dbt_project/models/backtesting/energy_commodities_summary_snapshot.sql b/dbt_project/models/backtesting/energy_commodities_summary_snapshot.sql index 6b48784..67c9d1a 100644 --- a/dbt_project/models/backtesting/energy_commodities_summary_snapshot.sql +++ b/dbt_project/models/backtesting/energy_commodities_summary_snapshot.sql @@ -1,14 +1,14 @@ {{ config( unique_key=['snapshot_date', 'commodity_name', 'commodity_unit', 'time_period'], - incremental_strategy='delete+insert' + incremental_strategy='merge' ) }} WITH snapshot_dates AS ( - SELECT DISTINCT DATE_TRUNC('month', date) AS snapshot_date + SELECT DISTINCT DATE_TRUNC(date, MONTH) AS snapshot_date FROM {{ ref('stg_energy_commodities') }} WHERE date >= '2020-01-01' {% if is_incremental() %} - AND DATE_TRUNC('month', date) >= COALESCE( + AND DATE_TRUNC(date, MONTH) >= COALESCE( (SELECT MAX(snapshot_date) FROM {{ this }}), DATE '1900-01-01' ) - INTERVAL 1 MONTH @@ -49,8 +49,8 @@ base_data AS ( price IS NOT NULL AND date IS NOT NULL AND price > 0 - AND trade_date <= sd.snapshot_date - AND trade_date >= sd.snapshot_date - INTERVAL 5 YEAR + AND CAST(date AS DATE) <= sd.snapshot_date + AND CAST(date AS DATE) >= sd.snapshot_date - INTERVAL 5 YEAR ), date_boundaries AS ( diff --git a/dbt_project/models/backtesting/fred_series_latest_aggregates_snapshot.sql b/dbt_project/models/backtesting/fred_series_latest_aggregates_snapshot.sql index ca77815..62f3564 100644 --- a/dbt_project/models/backtesting/fred_series_latest_aggregates_snapshot.sql +++ b/dbt_project/models/backtesting/fred_series_latest_aggregates_snapshot.sql @@ -1,15 +1,15 @@ {{ config( unique_key=['snapshot_date', 'series_code', 'month'], - incremental_strategy='delete+insert' + incremental_strategy='merge' ) }} WITH snapshot_dates AS ( -- Generate snapshot dates (first day of each month) from available data - SELECT DISTINCT DATE_TRUNC('month', date) AS snapshot_date + SELECT DISTINCT DATE_TRUNC(date, MONTH) AS snapshot_date FROM {{ ref('stg_fred_series') }} WHERE date >= '2020-01-01' -- Adjust based on your data availability {% if is_incremental() %} - AND DATE_TRUNC('month', date) >= COALESCE( + AND DATE_TRUNC(date, MONTH) >= COALESCE( (SELECT MAX(snapshot_date) FROM {{ this }}), DATE '1900-01-01' ) - INTERVAL 1 MONTH @@ -29,10 +29,10 @@ series_dates AS ( db.snapshot_date, fred_data.series_code, fred_data.series_name, - LAG(CAST(NULLIF(fred_data.value, '.') AS FLOAT), -2) + LAG(fred_data.value, -2) OVER (PARTITION BY db.snapshot_date, fred_data.series_code ORDER BY fred_data.date DESC) AS previous_date, - LAG(CAST(NULLIF(fred_data.value, '.') AS FLOAT), -3) + LAG(fred_data.value, -3) OVER (PARTITION BY db.snapshot_date, fred_data.series_code ORDER BY fred_data.date DESC) AS two_events_ago FROM {{ ref('stg_fred_series') }} AS fred_data @@ -40,7 +40,7 @@ series_dates AS ( WHERE fred_data.date >= db.start_date AND fred_data.date <= db.end_date ), -date_grain AS ( +series_grain AS ( SELECT s.snapshot_date, s.series_code, @@ -63,24 +63,24 @@ aggregates AS ( db.snapshot_date, fred_data.series_code, fred_data.series_name, - date_grain.date_grain, - DATE_TRUNC('month', fred_data.date) AS month, - ROUND(AVG(CAST(NULLIF(fred_data.value, '.') AS FLOAT)), 4) AS clean_value + series_grain.date_grain, + DATE_TRUNC(fred_data.date, MONTH) AS month, + ROUND(AVG(fred_data.value), 4) AS clean_value FROM {{ ref('stg_fred_series') }} AS fred_data CROSS JOIN date_bounds AS db - LEFT JOIN date_grain + LEFT JOIN series_grain ON - db.snapshot_date = date_grain.snapshot_date - AND fred_data.series_code = date_grain.series_code + db.snapshot_date = series_grain.snapshot_date + AND fred_data.series_code = series_grain.series_code WHERE fred_data.date >= db.start_date AND fred_data.date <= db.end_date - AND date_grain.date_grain IN ('Daily', 'Monthly', 'Quarterly', 'Weekly') + AND series_grain.date_grain IN ('Daily', 'Monthly', 'Quarterly', 'Weekly') GROUP BY db.snapshot_date, - DATE_TRUNC('month', fred_data.date), + DATE_TRUNC(fred_data.date, MONTH), fred_data.series_code, - date_grain.date_grain, + series_grain.date_grain, fred_data.series_name ), diff --git a/dbt_project/models/backtesting/input_commodities_summary_snapshot.sql b/dbt_project/models/backtesting/input_commodities_summary_snapshot.sql index f0dc4d3..49c8733 100644 --- a/dbt_project/models/backtesting/input_commodities_summary_snapshot.sql +++ b/dbt_project/models/backtesting/input_commodities_summary_snapshot.sql @@ -1,14 +1,14 @@ {{ config( unique_key=['snapshot_date', 'commodity_name', 'commodity_unit', 'time_period'], - incremental_strategy='delete+insert' + incremental_strategy='merge' ) }} WITH snapshot_dates AS ( - SELECT DISTINCT DATE_TRUNC('month', date) AS snapshot_date + SELECT DISTINCT DATE_TRUNC(date, MONTH) AS snapshot_date FROM {{ ref('stg_input_commodities') }} WHERE date >= '2020-01-01' {% if is_incremental() %} - AND DATE_TRUNC('month', date) >= COALESCE( + AND DATE_TRUNC(date, MONTH) >= COALESCE( (SELECT MAX(snapshot_date) FROM {{ this }}), DATE '1900-01-01' ) - INTERVAL 1 MONTH @@ -49,8 +49,8 @@ base_data AS ( price IS NOT NULL AND date IS NOT NULL AND price > 0 - AND trade_date <= sd.snapshot_date - AND trade_date >= sd.snapshot_date - INTERVAL 5 YEAR + AND CAST(date AS DATE) <= sd.snapshot_date + AND CAST(date AS DATE) >= sd.snapshot_date - INTERVAL 5 YEAR ), date_boundaries AS ( @@ -106,8 +106,13 @@ start_prices AS ( INNER JOIN filtered_data AS fd ON pb.snapshot_date = fd.snapshot_date AND pb.commodity_name = fd.commodity_name + AND pb.commodity_unit = fd.commodity_unit AND pb.time_period = fd.time_period AND pb.period_start_date = fd.trade_date + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY pb.snapshot_date, pb.commodity_name, pb.commodity_unit, pb.time_period + ORDER BY fd.trade_date ASC, fd.price ASC + ) = 1 ), end_prices AS ( @@ -121,8 +126,13 @@ end_prices AS ( INNER JOIN filtered_data AS fd ON pb.snapshot_date = fd.snapshot_date AND pb.commodity_name = fd.commodity_name + AND pb.commodity_unit = fd.commodity_unit AND pb.time_period = fd.time_period AND pb.period_end_date = fd.trade_date + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY pb.snapshot_date, pb.commodity_name, pb.commodity_unit, pb.time_period + ORDER BY fd.trade_date DESC, fd.price DESC + ) = 1 ), aggregated_results AS ( @@ -161,11 +171,13 @@ combined_results AS ( ON ar.snapshot_date = sp.snapshot_date AND ar.commodity_name = sp.commodity_name + AND ar.commodity_unit = sp.commodity_unit AND ar.time_period = sp.time_period LEFT JOIN end_prices AS ep ON ar.snapshot_date = ep.snapshot_date AND ar.commodity_name = ep.commodity_name + AND ar.commodity_unit = ep.commodity_unit AND ar.time_period = ep.time_period ), @@ -205,4 +217,8 @@ SELECT ROUND(period_start_price, 2) AS period_start_price, ROUND(period_end_price, 2) AS period_end_price FROM final_metrics +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY snapshot_date, commodity_name, commodity_unit, time_period + ORDER BY period_end_date DESC, period_start_date DESC +) = 1 ORDER BY snapshot_date DESC, time_period ASC, commodity_name ASC diff --git a/dbt_project/models/backtesting/leading_econ_return_indicator_snapshot.sql b/dbt_project/models/backtesting/leading_econ_return_indicator_snapshot.sql index 40a8dc8..3a8515e 100644 --- a/dbt_project/models/backtesting/leading_econ_return_indicator_snapshot.sql +++ b/dbt_project/models/backtesting/leading_econ_return_indicator_snapshot.sql @@ -1,14 +1,14 @@ {{ config( unique_key=['snapshot_date', 'symbol', 'series_name', 'category', 'economic_category'], - incremental_strategy='delete+insert' + incremental_strategy='merge' ) }} WITH snapshot_dates AS ( - SELECT DISTINCT DATE_TRUNC('month', month_date) AS snapshot_date + SELECT DISTINCT DATE_TRUNC(date, MONTH) AS snapshot_date FROM {{ ref('base_historical_analysis') }} - WHERE month_date >= '2020-01-01' + WHERE date >= '2020-01-01' {% if is_incremental() %} - AND DATE_TRUNC('month', month_date) >= COALESCE( + AND DATE_TRUNC(date, MONTH) >= COALESCE( (SELECT MAX(snapshot_date) FROM {{ this }}), DATE '1900-01-01' ) - INTERVAL 1 MONTH @@ -19,61 +19,63 @@ snapshot_base_historical AS ( SELECT bha.*, sd.snapshot_date - FROM {{ ref('base_historical_analysis') }} AS bha - CROSS JOIN snapshot_dates AS sd - WHERE bha.month_date <= sd.snapshot_date + FROM {{ ref('base_historical_analysis') }} AS bha + CROSS JOIN snapshot_dates AS sd + WHERE bha.date <= sd.snapshot_date ), economic_changes AS ( SELECT - snapshot_date, - symbol, - month_date, - bha.series_name, - bha.category, - fsm.category AS economic_category, - value AS current_econ_value, - monthly_avg_close, - pct_change_q1_forward, - pct_change_q2_forward, - pct_change_q3_forward, + snapshot_date, + symbol, + date, + bha.series_name, + bha.category, + fsm.category AS economic_category, + value AS current_econ_value, + current_price, + pct_change_3mo AS pct_change_q1, + pct_change_6mo AS pct_change_q2, + pct_change_9mo AS pct_change_q3, LAG(value, 1) OVER ( - PARTITION BY snapshot_date, symbol, bha.series_name - ORDER BY month_date + PARTITION BY snapshot_date, symbol, bha.series_name + ORDER BY date ) AS prev_econ_value, CASE WHEN LAG(value, 1) OVER ( - PARTITION BY snapshot_date, symbol, bha.series_name - ORDER BY month_date + PARTITION BY snapshot_date, symbol, bha.series_name + ORDER BY date ) IS NOT NULL AND LAG(value, 1) OVER ( - PARTITION BY snapshot_date, symbol, bha.series_name - ORDER BY month_date + PARTITION BY snapshot_date, symbol, bha.series_name + ORDER BY date ) != 0 THEN ( ( value - LAG(value, 1) OVER ( - PARTITION BY snapshot_date, symbol, bha.series_name - ORDER BY month_date + PARTITION BY snapshot_date, symbol, bha.series_name + ORDER BY date ) ) / LAG(value, 1) OVER ( - PARTITION BY snapshot_date, symbol, bha.series_name - ORDER BY month_date + PARTITION BY snapshot_date, symbol, bha.series_name + ORDER BY date ) ) * 100 END AS econ_mom_change_pct, AVG(value) OVER ( - PARTITION BY snapshot_date, symbol, bha.series_name - ORDER BY month_date + PARTITION BY snapshot_date, symbol, bha.series_name + ORDER BY date ROWS BETWEEN 2 PRECEDING AND CURRENT ROW ) AS econ_3mo_avg FROM snapshot_base_historical AS bha LEFT JOIN {{ ref('fred_series_mapping') }} AS fsm ON bha.series_name = fsm.series_name WHERE bha.value IS NOT NULL + AND bha.series_name IS NOT NULL + AND fsm.category IS NOT NULL ), correlation_analysis AS ( @@ -84,18 +86,18 @@ correlation_analysis AS ( category, economic_category, COUNT(*) AS observation_count, - CORR(econ_mom_change_pct, pct_change_q1_forward) - AS corr_econ_q1_returns, - CORR(econ_mom_change_pct, pct_change_q2_forward) - AS corr_econ_q2_returns, - CORR(econ_mom_change_pct, pct_change_q3_forward) - AS corr_econ_q3_returns, - AVG(CASE WHEN econ_mom_change_pct > 0 THEN pct_change_q1_forward END) - AS avg_q1_return_when_econ_growing, - AVG(CASE WHEN econ_mom_change_pct < 0 THEN pct_change_q1_forward END) - AS avg_q1_return_when_econ_declining, - STDDEV(econ_mom_change_pct) AS econ_change_volatility, - STDDEV(pct_change_q1_forward) AS q1_return_volatility, + CORR(econ_mom_change_pct, pct_change_q1) + AS corr_econ_q1_returns, + CORR(econ_mom_change_pct, pct_change_q2) + AS corr_econ_q2_returns, + CORR(econ_mom_change_pct, pct_change_q3) + AS corr_econ_q3_returns, + AVG(CASE WHEN econ_mom_change_pct > 0 THEN pct_change_q1 END) + AS avg_q1_return_when_econ_growing, + AVG(CASE WHEN econ_mom_change_pct < 0 THEN pct_change_q1 END) + AS avg_q1_return_when_econ_declining, + STDDEV(econ_mom_change_pct) AS econ_change_volatility, + STDDEV(pct_change_q1) AS q1_return_volatility, AVG(econ_mom_change_pct) AS avg_econ_change_pct, MIN(econ_mom_change_pct) AS min_econ_change_pct, MAX(econ_mom_change_pct) AS max_econ_change_pct diff --git a/dbt_project/models/backtesting/schema.yml b/dbt_project/models/backtesting/schema.yml index f93f221..f5ac72e 100644 --- a/dbt_project/models/backtesting/schema.yml +++ b/dbt_project/models/backtesting/schema.yml @@ -12,7 +12,7 @@ models: config: materialized: incremental unique_key: ['snapshot_date', 'symbol', 'asset_type', 'time_period'] - incremental_strategy: delete+insert + incremental_strategy: merge columns: - name: snapshot_date tests: @@ -31,7 +31,7 @@ models: config: materialized: incremental unique_key: ['snapshot_date', 'symbol', 'series_name', 'category', 'economic_category'] - incremental_strategy: delete+insert + incremental_strategy: merge columns: - name: snapshot_date tests: @@ -50,7 +50,7 @@ models: config: materialized: incremental unique_key: ['snapshot_date', 'series_code', 'month'] - incremental_strategy: delete+insert + incremental_strategy: merge columns: - name: snapshot_date tests: @@ -70,7 +70,7 @@ models: config: materialized: incremental unique_key: ['snapshot_date', 'commodity_name', 'commodity_unit', 'time_period'] - incremental_strategy: delete+insert + incremental_strategy: merge columns: - name: snapshot_date tests: @@ -90,7 +90,7 @@ models: config: materialized: incremental unique_key: ['snapshot_date', 'commodity_name', 'commodity_unit', 'time_period'] - incremental_strategy: delete+insert + incremental_strategy: merge columns: - name: snapshot_date tests: @@ -110,7 +110,7 @@ models: config: materialized: incremental unique_key: ['snapshot_date', 'commodity_name', 'commodity_unit', 'time_period'] - incremental_strategy: delete+insert + incremental_strategy: merge columns: - name: snapshot_date tests: @@ -118,4 +118,3 @@ models: - name: commodity_name tests: - not_null - diff --git a/dbt_project/models/backtesting/us_sector_summary_snapshot.sql b/dbt_project/models/backtesting/us_sector_summary_snapshot.sql index 30821d6..dcf657e 100644 --- a/dbt_project/models/backtesting/us_sector_summary_snapshot.sql +++ b/dbt_project/models/backtesting/us_sector_summary_snapshot.sql @@ -1,15 +1,15 @@ {{ config( unique_key=['snapshot_date', 'symbol', 'asset_type', 'time_period'], - incremental_strategy='delete+insert' + incremental_strategy='merge' ) }} WITH snapshot_dates AS ( -- Generate snapshot dates (first day of each month) from available data - SELECT DISTINCT DATE_TRUNC('month', date) AS snapshot_date + SELECT DISTINCT DATE_TRUNC(date, MONTH) AS snapshot_date FROM {{ ref('stg_us_sectors') }} WHERE date >= '2020-01-01' -- Adjust based on your data availability {% if is_incremental() %} - AND DATE_TRUNC('month', date) >= COALESCE( + AND DATE_TRUNC(date, MONTH) >= COALESCE( (SELECT MAX(snapshot_date) FROM {{ this }}), DATE '1900-01-01' ) - INTERVAL 1 MONTH @@ -45,8 +45,8 @@ base_data AS ( AND open IS NOT NULL AND close IS NOT NULL AND open > 0 - AND trade_date <= sd.snapshot_date - AND trade_date >= sd.snapshot_date - INTERVAL 5 YEAR + AND CAST(date AS DATE) <= sd.snapshot_date + AND CAST(date AS DATE) >= sd.snapshot_date - INTERVAL 5 YEAR ), -- Define date boundaries for different periods relative to snapshot_date @@ -107,6 +107,10 @@ start_prices AS ( AND pb.asset_type = fd.asset_type AND pb.time_period = fd.time_period AND pb.period_start_date = fd.trade_date + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY pb.snapshot_date, pb.symbol, pb.asset_type, pb.time_period + ORDER BY fd.trade_date ASC, fd.adj_open ASC + ) = 1 ), end_prices AS ( @@ -123,6 +127,10 @@ end_prices AS ( AND pb.asset_type = fd.asset_type AND pb.time_period = fd.time_period AND pb.period_end_date = fd.trade_date + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY pb.snapshot_date, pb.symbol, pb.asset_type, pb.time_period + ORDER BY fd.trade_date DESC, fd.adj_close DESC + ) = 1 ), -- Main aggregation @@ -132,8 +140,8 @@ aggregated_results AS ( symbol, asset_type, time_period, - exchange, - name, + ANY_VALUE(exchange) AS exchange, + ANY_VALUE(name) AS name, MIN(trade_date) AS period_start_date, MAX(trade_date) AS period_end_date, COUNT(*) AS trading_days, @@ -160,7 +168,7 @@ aggregated_results AS ( SUM(CASE WHEN price_change_adj = 0 THEN 1 ELSE 0 END) AS neutral_days FROM filtered_data WHERE time_period != 'older' - GROUP BY snapshot_date, symbol, asset_type, time_period, exchange, name + GROUP BY snapshot_date, symbol, asset_type, time_period ), -- Combine aggregated results with period boundary prices @@ -224,4 +232,8 @@ SELECT ROUND(period_start_price, 2) AS period_start_price, ROUND(period_end_price, 2) AS period_end_price FROM final_metrics +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY snapshot_date, symbol, asset_type, time_period + ORDER BY period_end_date DESC, period_start_date DESC +) = 1 ORDER BY snapshot_date DESC, time_period ASC, asset_type ASC, symbol ASC diff --git a/dbt_project/models/commodities/schema.yml b/dbt_project/models/commodities/schema.yml index a40d519..2910fde 100644 --- a/dbt_project/models/commodities/schema.yml +++ b/dbt_project/models/commodities/schema.yml @@ -6,7 +6,7 @@ models: tests: - unique_combination: arguments: - combination_of_columns: ['commodity_name', 'time_period'] + combination_of_columns: ['commodity_name', 'commodity_unit', 'time_period'] columns: - name: commodity_name description: "Name of the commodity" @@ -44,7 +44,7 @@ models: tests: - unique_combination: arguments: - combination_of_columns: ['commodity_name', 'time_period'] + combination_of_columns: ['commodity_name', 'commodity_unit', 'time_period'] columns: - name: commodity_name description: "Name of the commodity" @@ -82,7 +82,7 @@ models: tests: - unique_combination: arguments: - combination_of_columns: ['commodity_name', 'time_period'] + combination_of_columns: ['commodity_name', 'commodity_unit', 'time_period'] columns: - name: commodity_name description: "Name of the commodity" @@ -294,4 +294,3 @@ models: description: Standard deviation of daily price differences over the past 1 month - name: pct_change_1mo description: Percentage change from 1 month ago to current price - diff --git a/dbt_project/models/data_quality/dq_return_spikes.sql b/dbt_project/models/data_quality/dq_return_spikes.sql index 8b2318f..38c1ac0 100644 --- a/dbt_project/models/data_quality/dq_return_spikes.sql +++ b/dbt_project/models/data_quality/dq_return_spikes.sql @@ -9,12 +9,12 @@ {% for table_name in tables %} ( -with {{ table_name }}_returns as ( - select - symbol, - cast(date as date) as date, - close, - open, + with {{ table_name }}_returns as ( + select + symbol, + safe_cast(substr(cast(date as string), 1, 10) as date) as date, + close, + open, high, low, lag(close) over (partition by symbol order by date) as prev_close, diff --git a/dbt_project/models/data_quality/dq_stale_prices.sql b/dbt_project/models/data_quality/dq_stale_prices.sql index f696578..26c20ba 100644 --- a/dbt_project/models/data_quality/dq_stale_prices.sql +++ b/dbt_project/models/data_quality/dq_stale_prices.sql @@ -12,7 +12,7 @@ with {{ table_name }}_consecutive as ( select symbol, - cast(date as date) as date, + SAFE_CAST(SUBSTR(CAST(date AS STRING), 1, 10) AS DATE) AS date, close, open, high, diff --git a/dbt_project/models/data_quality/dq_zscore_anomalies.sql b/dbt_project/models/data_quality/dq_zscore_anomalies.sql index abe8104..1b4b271 100644 --- a/dbt_project/models/data_quality/dq_zscore_anomalies.sql +++ b/dbt_project/models/data_quality/dq_zscore_anomalies.sql @@ -10,13 +10,13 @@ {% for table_name in tables %} ( -with {{ table_name }}_rolling as ( - select - symbol, - cast(date as date) as date, - close, - open, - high, + with {{ table_name }}_rolling as ( + select + symbol, + safe_cast(substr(cast(date as string), 1, 10) as date) as date, + close, + open, + high, low, avg(close) over w as rolling_avg_close, stddev(close) over w as rolling_std_close, @@ -63,8 +63,8 @@ where where ca.source_table = '{{ table_name }}' and ca.symbol = {{ table_name }}_rolling.symbol and ca.action_type = 'split' - and {{ table_name }}_rolling.date between ca.date - interval '2' day and ca.date + interval '2' day - ) -) + and {{ table_name }}_rolling.date between ca.date - interval 2 day and ca.date + interval 2 day + ) + ) {% if not loop.last %}union all{% endif %} {% endfor %} diff --git a/dbt_project/models/government/fred_quarterly_roc.sql b/dbt_project/models/government/fred_quarterly_roc.sql index a1f2fb7..a81de0c 100644 --- a/dbt_project/models/government/fred_quarterly_roc.sql +++ b/dbt_project/models/government/fred_quarterly_roc.sql @@ -4,21 +4,32 @@ ) }} -WITH quarterly_data AS ( +WITH fred_monthly AS ( SELECT series_code, series_name, - CONCAT(EXTRACT(YEAR FROM date), '-', EXTRACT(MONTH FROM date)) - AS year_month, EXTRACT(YEAR FROM date) AS year_val, EXTRACT(MONTH FROM date) AS month_val, DATE(EXTRACT(YEAR FROM date), EXTRACT(MONTH FROM date), 1) AS month_date, - AVG(literal) AS avg_value + literal FROM {{ ref('stg_fred_series') }} +), + +quarterly_data AS ( + SELECT + series_code, + series_name, + CONCAT(year_val, '-', month_val) AS year_month, + year_val, + month_val, + month_date, + AVG(literal) AS avg_value + FROM fred_monthly GROUP BY - EXTRACT(YEAR FROM date), - EXTRACT(MONTH FROM date), + year_val, + month_val, + month_date, series_code, series_name ), @@ -110,12 +121,15 @@ SELECT month_date, ROUND(avg_value, 2) AS avg_value, ROUND( - (avg_value - LAG(avg_value) OVER ( - PARTITION BY series_code - ORDER BY month_date - )) / LAG(avg_value) OVER ( - PARTITION BY series_code - ORDER BY month_date + SAFE_DIVIDE( + avg_value - LAG(avg_value) OVER ( + PARTITION BY series_code + ORDER BY month_date + ), + LAG(avg_value) OVER ( + PARTITION BY series_code + ORDER BY month_date + ) ) * 100, 2 ) AS pct_change_period diff --git a/dbt_project/models/government/fred_series_grain.sql b/dbt_project/models/government/fred_series_grain.sql index b56f49c..dcd67b8 100644 --- a/dbt_project/models/government/fred_series_grain.sql +++ b/dbt_project/models/government/fred_series_grain.sql @@ -1,7 +1,7 @@ WITH date_bounds AS ( SELECT - CURRENT_DATE AS end_date, - CURRENT_DATE - INTERVAL 12 MONTH AS start_date + CURRENT_DATE() AS end_date, + DATE_SUB(CURRENT_DATE(), INTERVAL 12 MONTH) AS start_date ), series_dates AS ( diff --git a/dbt_project/models/government/fred_series_latest_aggregates.sql b/dbt_project/models/government/fred_series_latest_aggregates.sql index 4760645..715c1e5 100644 --- a/dbt_project/models/government/fred_series_latest_aggregates.sql +++ b/dbt_project/models/government/fred_series_latest_aggregates.sql @@ -1,24 +1,24 @@ WITH date_bounds AS ( SELECT - CURRENT_DATE AS end_date, - CURRENT_DATE - INTERVAL 12 MONTH AS start_date + CURRENT_DATE() AS end_date, + DATE_SUB(CURRENT_DATE(), INTERVAL 12 MONTH) AS start_date ), series_dates AS ( SELECT series_code, series_name, - LAG(CAST(NULLIF(value, '.') AS FLOAT), -2) + LAG(fred_data.value, -2) OVER (PARTITION BY series_code ORDER BY date DESC) AS previous_date, - LAG(CAST(NULLIF(value, '.') AS FLOAT), -3) + LAG(fred_data.value, -3) OVER (PARTITION BY series_code ORDER BY date DESC) AS two_events_ago FROM {{ ref('stg_fred_series') }} AS fred_data, date_bounds AS d WHERE fred_data.date >= d.start_date AND fred_data.date <= d.end_date ), -date_grain AS ( +series_grain AS ( SELECT s.series_code, s.series_name, @@ -44,20 +44,20 @@ aggregates AS ( SELECT fred_data.series_code, fred_data.series_name, - date_grain.date_grain, - DATE_TRUNC('month', fred_data.date) AS month, - ROUND(AVG(CAST(NULLIF(fred_data.value, '.') AS FLOAT)), 4) + series_grain.date_grain, + DATE_TRUNC(fred_data.date, MONTH) AS month, + ROUND(AVG(fred_data.value), 4) AS clean_value FROM {{ ref('stg_fred_series') }} AS fred_data - LEFT JOIN date_grain - ON fred_data.series_code = date_grain.series_code - WHERE date_grain.date_grain IN ('Daily', 'Monthly', 'Quarterly', 'Weekly') + LEFT JOIN series_grain + ON fred_data.series_code = series_grain.series_code + WHERE series_grain.date_grain IN ('Daily', 'Monthly', 'Quarterly', 'Weekly') GROUP BY - DATE_TRUNC('month', fred_data.date), + DATE_TRUNC(fred_data.date, MONTH), fred_data.series_code, - date_grain.date_grain, + series_grain.date_grain, fred_data.series_name - ORDER BY DATE_TRUNC('month', fred_data.date) DESC + ORDER BY DATE_TRUNC(fred_data.date, MONTH) DESC ), date_ranges AS ( @@ -114,7 +114,7 @@ max_date_view AS ( FROM calc_view GROUP BY - series_code, + series_code ), final AS ( @@ -126,15 +126,12 @@ final AS ( calc_view.pct_change_3m, calc_view.pct_change_6m, calc_view.pct_change_1y, - date_grain.date_grain + calc_view.date_grain FROM calc_view INNER JOIN max_date_view ON calc_view.series_code = max_date_view.series_code AND calc_view.month = max_date_view.month - LEFT JOIN date_grain - ON calc_view.series_code = date_grain.series_code - ) SELECT * FROM final diff --git a/dbt_project/models/government/housing_inventory.sql b/dbt_project/models/government/housing_inventory.sql index aeca423..4fd0da2 100644 --- a/dbt_project/models/government/housing_inventory.sql +++ b/dbt_project/models/government/housing_inventory.sql @@ -2,7 +2,7 @@ select data_type_code as data_code, seasonally_adj, category_code, - cast(cell_value as float) as series_value, + cast(cell_value as float64) as series_value, error_data, time, series_name, diff --git a/dbt_project/models/government/housing_inventory_and_population.sql b/dbt_project/models/government/housing_inventory_and_population.sql index 6fef623..bb0e880 100644 --- a/dbt_project/models/government/housing_inventory_and_population.sql +++ b/dbt_project/models/government/housing_inventory_and_population.sql @@ -1,17 +1,17 @@ With hs As ( Select time_date, - cast(series_value As float) As number_of_households, + cast(series_value As float64) As number_of_households, extract(Year From time_date) As year From {{ ref('housing_inventory') }} Where category_code = 'TTLHH' - And series_value <> '.' + And series_value is not null ) Select series_name, - cast(series_value As float) As series_value, + cast(series_value As float64) As series_value, cast((Case When right(housing_inventory.time, 2) = 'Q1' diff --git a/dbt_project/models/government/housing_inventory_latest_aggregates.sql b/dbt_project/models/government/housing_inventory_latest_aggregates.sql index 70921b6..aa138b7 100644 --- a/dbt_project/models/government/housing_inventory_latest_aggregates.sql +++ b/dbt_project/models/government/housing_inventory_latest_aggregates.sql @@ -3,7 +3,7 @@ With inventory As ( data_type_code As series_code, seasonally_adj, category_code, - cast(cell_value As float) As clean_value, + cast(cell_value As float64) As clean_value, error_data, time, series_name, @@ -81,7 +81,7 @@ max_date_view As ( From calc_view Group By - series_code, + series_code ), final As ( diff --git a/dbt_project/models/markets/nasdaq_companies_analysis_return.sql b/dbt_project/models/markets/nasdaq_companies_analysis_return.sql index cbc108f..ba15268 100644 --- a/dbt_project/models/markets/nasdaq_companies_analysis_return.sql +++ b/dbt_project/models/markets/nasdaq_companies_analysis_return.sql @@ -88,18 +88,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 365 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 365 PRECEDING AND CURRENT ROW ) AS high_1yr, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 365 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 365 PRECEDING AND CURRENT ROW ) AS low_1yr, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 365 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 365 PRECEDING AND CURRENT ROW ) AS std_diff_1yr, price_365d_ago AS price_start_1yr, CASE @@ -110,18 +110,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 270 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 270 PRECEDING AND CURRENT ROW ) AS high_9mo, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 270 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 270 PRECEDING AND CURRENT ROW ) AS low_9mo, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 270 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 270 PRECEDING AND CURRENT ROW ) AS std_diff_9mo, price_270d_ago AS price_start_9mo, CASE @@ -132,18 +132,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 180 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 180 PRECEDING AND CURRENT ROW ) AS high_6mo, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 180 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 180 PRECEDING AND CURRENT ROW ) AS low_6mo, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 180 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 180 PRECEDING AND CURRENT ROW ) AS std_diff_6mo, price_180d_ago AS price_start_6mo, CASE @@ -154,18 +154,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 90 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 90 PRECEDING AND CURRENT ROW ) AS high_3mo, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 90 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 90 PRECEDING AND CURRENT ROW ) AS low_3mo, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 90 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 90 PRECEDING AND CURRENT ROW ) AS std_diff_3mo, price_90d_ago AS price_start_3mo, CASE @@ -176,18 +176,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 30 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 30 PRECEDING AND CURRENT ROW ) AS high_1mo, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 30 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 30 PRECEDING AND CURRENT ROW ) AS low_1mo, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 30 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 30 PRECEDING AND CURRENT ROW ) AS std_diff_1mo, price_30d_ago AS price_start_1mo, CASE diff --git a/dbt_project/models/markets/nasdaq_companies_summary.sql b/dbt_project/models/markets/nasdaq_companies_summary.sql index a2a8ffb..5a94497 100644 --- a/dbt_project/models/markets/nasdaq_companies_summary.sql +++ b/dbt_project/models/markets/nasdaq_companies_summary.sql @@ -32,11 +32,11 @@ WITH base_data AS ( -- Define date boundaries for different periods date_boundaries AS ( SELECT - CURRENT_DATE AS today, - CURRENT_DATE - INTERVAL 12 WEEK AS twelve_weeks_ago, - CURRENT_DATE - INTERVAL 6 MONTH AS six_months_ago, - CURRENT_DATE - INTERVAL 1 YEAR AS one_year_ago, - CURRENT_DATE - INTERVAL 5 YEAR AS five_years_ago + CURRENT_DATE() AS today, + DATE_SUB(CURRENT_DATE(), INTERVAL 12 WEEK) AS twelve_weeks_ago, + DATE_SUB(CURRENT_DATE(), INTERVAL 6 MONTH) AS six_months_ago, + DATE_SUB(CURRENT_DATE(), INTERVAL 1 YEAR) AS one_year_ago, + DATE_SUB(CURRENT_DATE(), INTERVAL 5 YEAR) AS five_years_ago ), -- Filter data for each time period diff --git a/dbt_project/models/markets/sp500_companies_analysis_return.sql b/dbt_project/models/markets/sp500_companies_analysis_return.sql index 64c98ee..d1dec93 100644 --- a/dbt_project/models/markets/sp500_companies_analysis_return.sql +++ b/dbt_project/models/markets/sp500_companies_analysis_return.sql @@ -88,18 +88,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 365 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 365 PRECEDING AND CURRENT ROW ) AS high_1yr, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 365 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 365 PRECEDING AND CURRENT ROW ) AS low_1yr, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 365 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 365 PRECEDING AND CURRENT ROW ) AS std_diff_1yr, price_365d_ago AS price_start_1yr, CASE @@ -110,18 +110,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 270 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 270 PRECEDING AND CURRENT ROW ) AS high_9mo, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 270 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 270 PRECEDING AND CURRENT ROW ) AS low_9mo, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 270 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 270 PRECEDING AND CURRENT ROW ) AS std_diff_9mo, price_270d_ago AS price_start_9mo, CASE @@ -132,18 +132,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 180 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 180 PRECEDING AND CURRENT ROW ) AS high_6mo, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 180 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 180 PRECEDING AND CURRENT ROW ) AS low_6mo, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 180 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 180 PRECEDING AND CURRENT ROW ) AS std_diff_6mo, price_180d_ago AS price_start_6mo, CASE @@ -154,18 +154,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 90 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 90 PRECEDING AND CURRENT ROW ) AS high_3mo, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 90 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 90 PRECEDING AND CURRENT ROW ) AS low_3mo, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 90 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 90 PRECEDING AND CURRENT ROW ) AS std_diff_3mo, price_90d_ago AS price_start_3mo, CASE @@ -176,18 +176,18 @@ rolling_stats AS ( MAX(current_high) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 30 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 30 PRECEDING AND CURRENT ROW ) AS high_1mo, MIN(current_low) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 30 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 30 PRECEDING AND CURRENT ROW ) AS low_1mo, STDDEV(daily_diff) OVER ( PARTITION BY symbol, exchange - ORDER BY date - RANGE BETWEEN INTERVAL 30 DAY PRECEDING AND CURRENT ROW + ORDER BY UNIX_DATE(date) + RANGE BETWEEN 30 PRECEDING AND CURRENT ROW ) AS std_diff_1mo, price_30d_ago AS price_start_1mo, CASE diff --git a/dbt_project/models/markets/sp500_companies_summary.sql b/dbt_project/models/markets/sp500_companies_summary.sql index 7e2d89b..08fbc79 100644 --- a/dbt_project/models/markets/sp500_companies_summary.sql +++ b/dbt_project/models/markets/sp500_companies_summary.sql @@ -32,11 +32,11 @@ WITH base_data AS ( -- Define date boundaries for different periods date_boundaries AS ( SELECT - CURRENT_DATE AS today, - CURRENT_DATE - INTERVAL 12 WEEK AS twelve_weeks_ago, - CURRENT_DATE - INTERVAL 6 MONTH AS six_months_ago, - CURRENT_DATE - INTERVAL 1 YEAR AS one_year_ago, - CURRENT_DATE - INTERVAL 5 YEAR AS five_years_ago + CURRENT_DATE() AS today, + DATE_SUB(CURRENT_DATE(), INTERVAL 12 WEEK) AS twelve_weeks_ago, + DATE_SUB(CURRENT_DATE(), INTERVAL 6 MONTH) AS six_months_ago, + DATE_SUB(CURRENT_DATE(), INTERVAL 1 YEAR) AS one_year_ago, + DATE_SUB(CURRENT_DATE(), INTERVAL 5 YEAR) AS five_years_ago ), -- Filter data for each time period @@ -60,52 +60,55 @@ filtered_data AS ( period_boundaries AS ( SELECT symbol, - asset_type, time_period, MIN(trade_date) AS period_start_date, MAX(trade_date) AS period_end_date FROM filtered_data WHERE time_period != 'older' - GROUP BY symbol, asset_type, time_period + GROUP BY symbol, time_period ), -- Get start and end prices start_prices AS ( SELECT pb.symbol, - pb.asset_type, pb.time_period, fd.adj_open AS period_start_price FROM period_boundaries AS pb INNER JOIN filtered_data AS fd ON pb.symbol = fd.symbol - AND pb.asset_type = fd.asset_type AND pb.time_period = fd.time_period AND pb.period_start_date = fd.trade_date + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY pb.symbol, pb.time_period + ORDER BY fd.trade_date ASC, fd.adj_open ASC + ) = 1 ), end_prices AS ( SELECT pb.symbol, - pb.asset_type, pb.time_period, fd.adj_close AS period_end_price FROM period_boundaries AS pb INNER JOIN filtered_data AS fd ON pb.symbol = fd.symbol - AND pb.asset_type = fd.asset_type AND pb.time_period = fd.time_period AND pb.period_end_date = fd.trade_date + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY pb.symbol, pb.time_period + ORDER BY fd.trade_date DESC, fd.adj_close DESC + ) = 1 ), -- Main aggregation without window functions aggregated_results AS ( SELECT symbol, - asset_type, + ARRAY_AGG(asset_type ORDER BY trade_date DESC LIMIT 1)[SAFE_OFFSET(0)] AS asset_type, time_period, - exchange, - name, + ARRAY_AGG(exchange ORDER BY trade_date DESC LIMIT 1)[SAFE_OFFSET(0)] AS exchange, + ARRAY_AGG(name ORDER BY trade_date DESC LIMIT 1)[SAFE_OFFSET(0)] AS name, -- Date range info MIN(trade_date) AS period_start_date, @@ -147,10 +150,7 @@ aggregated_results AS ( WHERE time_period != 'older' GROUP BY symbol, - asset_type, - time_period, - exchange, - name + time_period ), -- Combine aggregated results with period boundary prices @@ -163,11 +163,9 @@ combined_results AS ( LEFT JOIN start_prices AS sp ON ar.symbol = sp.symbol - AND ar.asset_type = sp.asset_type AND ar.time_period = sp.time_period LEFT JOIN end_prices AS ep ON ar.symbol = ep.symbol - AND ar.asset_type = ep.asset_type AND ar.time_period = ep.time_period ), @@ -232,6 +230,10 @@ SELECT ROUND(period_end_price, 2) AS period_end_price FROM final_metrics +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY symbol, time_period + ORDER BY period_end_date DESC, period_start_date DESC +) = 1 ORDER BY time_period, asset_type, diff --git a/dbt_project/models/signals/cross_asset_divergences.sql b/dbt_project/models/signals/cross_asset_divergences.sql index d42ae06..58b60de 100644 --- a/dbt_project/models/signals/cross_asset_divergences.sql +++ b/dbt_project/models/signals/cross_asset_divergences.sql @@ -11,7 +11,7 @@ WITH spy_prices AS ( FROM {{ ref('stg_major_indices') }} WHERE symbol = 'SPY' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), spy_indicators AS ( @@ -40,7 +40,7 @@ hyg_prices AS ( FROM {{ ref('stg_fixed_income') }} WHERE symbol = 'HYG' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), hyg_indicators AS ( @@ -54,14 +54,14 @@ hyg_indicators AS ( FROM hyg_prices ), -hy_spread AS ( +hy_spread_data AS ( SELECT date, value AS hy_spread FROM {{ ref('stg_fred_series') }} WHERE series_code = 'BAMLH0A0HYM2' AND value IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), hy_spread_indicators AS ( @@ -69,7 +69,7 @@ hy_spread_indicators AS ( date, hy_spread, hy_spread - LAG(hy_spread, 20) OVER (ORDER BY date) AS hy_spread_20d_change - FROM hy_spread + FROM hy_spread_data ), hy_equity_divergence AS ( @@ -102,7 +102,7 @@ spy_returns AS ( SELECT date, spy_close, - (spy_close / LAG(spy_close) OVER (ORDER BY date) - 1.0) AS spy_return + (SAFE_DIVIDE(spy_close, LAG(spy_close) OVER (ORDER BY date)) - 1.0) AS spy_return FROM spy_prices ), @@ -113,14 +113,14 @@ govt_prices AS ( FROM {{ ref('stg_fixed_income') }} WHERE symbol = 'GOVT' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), govt_returns AS ( SELECT date, govt_close, - (govt_close / LAG(govt_close) OVER (ORDER BY date) - 1.0) AS govt_return + (SAFE_DIVIDE(govt_close, LAG(govt_close) OVER (ORDER BY date)) - 1.0) AS govt_return FROM govt_prices ), @@ -145,7 +145,7 @@ xlp_prices AS ( FROM {{ ref('stg_us_sectors') }} WHERE symbol = 'XLP' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), xly_prices AS ( @@ -155,16 +155,16 @@ xly_prices AS ( FROM {{ ref('stg_us_sectors') }} WHERE symbol = 'XLY' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), -xlp_xly_ratio AS ( +xlp_xly_ratio_data AS ( SELECT xlp.date, xlp.xlp_close, xly.xly_close, CASE - WHEN xly.xly_close > 0 THEN xlp.xlp_close / xly.xly_close + WHEN xly.xly_close > 0 THEN SAFE_DIVIDE(xlp.xlp_close, xly.xly_close) END AS xlp_xly_ratio FROM xlp_prices xlp INNER JOIN xly_prices xly @@ -183,7 +183,7 @@ xlp_xly_indicators AS ( ORDER BY date ROWS BETWEEN 199 PRECEDING AND CURRENT ROW ) AS xlp_xly_sma_200 - FROM xlp_xly_ratio + FROM xlp_xly_ratio_data ), gold_prices AS ( @@ -194,7 +194,7 @@ gold_prices AS ( WHERE commodity_name = 'gold' AND price IS NOT NULL AND price > 0 - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), copper_prices AS ( @@ -205,7 +205,7 @@ copper_prices AS ( WHERE commodity_name = 'copper' AND price IS NOT NULL AND price > 0 - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), real_yields AS ( @@ -215,7 +215,7 @@ real_yields AS ( FROM {{ ref('stg_fred_series') }} WHERE series_code = 'DFII10' AND value IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), gold_real_base AS ( @@ -251,15 +251,19 @@ gold_real_residual AS ( real_yield_10y, CASE WHEN (avg_x2 - (avg_real_yield * avg_real_yield)) <> 0 THEN - (avg_xy - (avg_real_yield * avg_gold_price)) - / (avg_x2 - (avg_real_yield * avg_real_yield)) + SAFE_DIVIDE( + avg_xy - (avg_real_yield * avg_gold_price), + avg_x2 - (avg_real_yield * avg_real_yield) + ) END AS beta, CASE WHEN (avg_x2 - (avg_real_yield * avg_real_yield)) <> 0 THEN avg_gold_price - ( - (avg_xy - (avg_real_yield * avg_gold_price)) - / (avg_x2 - (avg_real_yield * avg_real_yield)) + SAFE_DIVIDE( + avg_xy - (avg_real_yield * avg_gold_price), + avg_x2 - (avg_real_yield * avg_real_yield) + ) ) * avg_real_yield END AS alpha FROM gold_real_regression @@ -300,7 +304,7 @@ iwm_prices AS ( FROM {{ ref('stg_major_indices') }} WHERE symbol = 'IWM' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), rsp_prices AS ( @@ -310,14 +314,14 @@ rsp_prices AS ( FROM {{ ref('stg_major_indices') }} WHERE symbol = 'RSP' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), -iwm_spy_ratio AS ( +iwm_spy_ratio_data AS ( SELECT s.date, CASE - WHEN s.spy_close > 0 THEN i.iwm_close / s.spy_close + WHEN s.spy_close > 0 THEN SAFE_DIVIDE(i.iwm_close, s.spy_close) END AS iwm_spy_ratio FROM spy_prices s INNER JOIN iwm_prices i @@ -336,14 +340,14 @@ iwm_spy_indicators AS ( ORDER BY date ROWS BETWEEN 199 PRECEDING AND CURRENT ROW ) AS iwm_spy_sma_200 - FROM iwm_spy_ratio + FROM iwm_spy_ratio_data ), -rsp_spy_ratio AS ( +rsp_spy_ratio_data AS ( SELECT s.date, CASE - WHEN s.spy_close > 0 THEN r.rsp_close / s.spy_close + WHEN s.spy_close > 0 THEN SAFE_DIVIDE(r.rsp_close, s.spy_close) END AS rsp_spy_ratio FROM spy_prices s INNER JOIN rsp_prices r @@ -362,7 +366,7 @@ rsp_spy_indicators AS ( ORDER BY date ROWS BETWEEN 199 PRECEDING AND CURRENT ROW ) AS rsp_spy_sma_200 - FROM rsp_spy_ratio + FROM rsp_spy_ratio_data ), copper_gold_base AS ( @@ -371,7 +375,7 @@ copper_gold_base AS ( g.gold_price, c.copper_price, CASE - WHEN g.gold_price > 0 THEN (c.copper_price / g.gold_price) * 1000 + WHEN g.gold_price > 0 THEN SAFE_DIVIDE(c.copper_price, g.gold_price) * 1000 END AS copper_gold_ratio FROM gold_prices g INNER JOIN copper_prices c @@ -380,11 +384,11 @@ copper_gold_base AS ( treasury_yields AS ( SELECT - date, + SAFE_CAST(date AS DATE) AS date, bc_10year AS treasury_10y_yield FROM {{ ref('stg_treasury_yields') }} WHERE bc_10year IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND SAFE_CAST(date AS DATE) >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), copper_gold_yield_corr AS ( @@ -408,14 +412,14 @@ fxa_prices AS ( FROM {{ ref('stg_currency') }} WHERE symbol = 'FXA' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), -fxa_spy_ratio AS ( +fxa_spy_ratio_data AS ( SELECT s.date, CASE - WHEN s.spy_close > 0 THEN f.fxa_close / s.spy_close + WHEN s.spy_close > 0 THEN SAFE_DIVIDE(f.fxa_close, s.spy_close) END AS fxa_spy_ratio FROM spy_prices s INNER JOIN fxa_prices f @@ -430,7 +434,7 @@ fxa_spy_indicators AS ( ORDER BY date ROWS BETWEEN 49 PRECEDING AND CURRENT ROW ) AS fxa_spy_sma_50 - FROM fxa_spy_ratio + FROM fxa_spy_ratio_data ), dia_prices AS ( @@ -440,7 +444,7 @@ dia_prices AS ( FROM {{ ref('stg_major_indices') }} WHERE symbol = 'DIA' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), iyt_prices AS ( @@ -450,7 +454,7 @@ iyt_prices AS ( FROM {{ ref('stg_major_indices') }} WHERE symbol = 'IYT' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), dow_theory AS ( @@ -478,14 +482,14 @@ soxx_prices AS ( FROM {{ ref('stg_major_indices') }} WHERE symbol = 'SOXX' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), -soxx_spy_ratio AS ( +soxx_spy_ratio_data AS ( SELECT s.date, CASE - WHEN s.spy_close > 0 THEN x.soxx_close / s.spy_close + WHEN s.spy_close > 0 THEN SAFE_DIVIDE(x.soxx_close, s.spy_close) END AS soxx_spy_ratio FROM spy_prices s INNER JOIN soxx_prices x @@ -500,7 +504,7 @@ soxx_spy_indicators AS ( ORDER BY date ROWS BETWEEN 199 PRECEDING AND CURRENT ROW ) AS soxx_spy_sma_200 - FROM soxx_spy_ratio + FROM soxx_spy_ratio_data ) SELECT @@ -536,7 +540,7 @@ SELECT gr.real_yield_10y, gr.gold_real_residual, CASE - WHEN gr.residual_std > 0 THEN (gr.gold_real_residual - gr.residual_avg) / gr.residual_std + WHEN gr.residual_std > 0 THEN SAFE_DIVIDE(gr.gold_real_residual - gr.residual_avg, gr.residual_std) END AS gold_real_residual_zscore, iwm.iwm_spy_ratio, @@ -588,5 +592,5 @@ LEFT JOIN fxa_spy_indicators fxa ON h.date = fxa.date LEFT JOIN dow_theory dow ON h.date = dow.date LEFT JOIN soxx_spy_indicators soxx ON h.date = soxx.date -WHERE h.date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE h.date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY h.date DESC diff --git a/dbt_project/models/signals/diffusion_index_signals.sql b/dbt_project/models/signals/diffusion_index_signals.sql index c490d23..754e91f 100644 --- a/dbt_project/models/signals/diffusion_index_signals.sql +++ b/dbt_project/models/signals/diffusion_index_signals.sql @@ -19,7 +19,7 @@ WITH series_monthly AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, series_code, MAX(value) AS val FROM {{ ref('stg_fred_series') }} @@ -52,7 +52,7 @@ WITH series_monthly AS ( 'PI' -- Personal income ) AND value IS NOT NULL - GROUP BY DATE_TRUNC('month', date), series_code + GROUP BY DATE_TRUNC(date, MONTH), series_code ), with_changes AS ( @@ -87,10 +87,10 @@ scored AS ( monthly_diffusion AS ( SELECT month_date AS date, - COUNT(*) FILTER (WHERE is_improving IS NOT NULL) AS total_count, + COUNTIF(is_improving IS NOT NULL) AS total_count, COALESCE(SUM(is_improving), 0) AS improving_count, ROUND( - COALESCE(SUM(is_improving), 0) * 100.0 / NULLIF(COUNT(*) FILTER (WHERE is_improving IS NOT NULL), 0), + COALESCE(SUM(is_improving), 0) * 100.0 / NULLIF(COUNTIF(is_improving IS NOT NULL), 0), 1 ) AS diffusion_pct FROM scored @@ -143,5 +143,5 @@ SELECT END AS diffusion_status FROM with_stats -WHERE date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY date DESC diff --git a/dbt_project/models/signals/economic_acceleration_signals.sql b/dbt_project/models/signals/economic_acceleration_signals.sql index bf1fe73..7031baf 100644 --- a/dbt_project/models/signals/economic_acceleration_signals.sql +++ b/dbt_project/models/signals/economic_acceleration_signals.sql @@ -17,12 +17,12 @@ WITH payems_raw AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(value) AS payems FROM {{ ref('stg_fred_series') }} WHERE series_code = 'PAYEMS' AND value IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), payems_derivatives AS ( @@ -66,12 +66,12 @@ payems_consecutive AS ( -- CPI acceleration (monthly) cpi_raw AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(value) AS cpi FROM {{ ref('stg_fred_series') }} WHERE series_code = 'CPIAUCSL' AND value IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), cpi_derivatives AS ( @@ -134,7 +134,7 @@ combined AS ( g.gdp_acceleration FROM payems_consecutive p LEFT JOIN cpi_accel c ON p.month_date = c.month_date - LEFT JOIN gdp_accel g ON DATE_TRUNC('quarter', p.month_date) = g.quarter_date + LEFT JOIN gdp_accel g ON DATE_TRUNC(p.month_date, QUARTER) = g.quarter_date ), with_stats AS ( @@ -195,5 +195,5 @@ SELECT END AS gdp_accel_status FROM with_stats -WHERE date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY date DESC diff --git a/dbt_project/models/signals/economic_alert_inputs.sql b/dbt_project/models/signals/economic_alert_inputs.sql index de89d79..688d7a4 100644 --- a/dbt_project/models/signals/economic_alert_inputs.sql +++ b/dbt_project/models/signals/economic_alert_inputs.sql @@ -87,13 +87,13 @@ hy_oas AS ( all_dates AS ( SELECT date FROM cpi_yoy - UNION + UNION DISTINCT SELECT date FROM t10y2y - UNION + UNION DISTINCT SELECT date FROM unrate_delta - UNION + UNION DISTINCT SELECT date FROM fedfunds_delta - UNION + UNION DISTINCT SELECT date FROM hy_oas ) @@ -110,5 +110,5 @@ LEFT JOIN t10y2y AS t ON d.date = t.date LEFT JOIN unrate_delta AS u ON d.date = u.date LEFT JOIN fedfunds_delta AS f ON d.date = f.date LEFT JOIN hy_oas AS h ON d.date = h.date -WHERE d.date >= CURRENT_DATE - INTERVAL 2 YEAR +WHERE d.date >= DATE_SUB(CURRENT_DATE(), INTERVAL 2 YEAR) ORDER BY d.date DESC diff --git a/dbt_project/models/signals/factor_signals.sql b/dbt_project/models/signals/factor_signals.sql index f9e7cfa..63af0dd 100644 --- a/dbt_project/models/signals/factor_signals.sql +++ b/dbt_project/models/signals/factor_signals.sql @@ -19,7 +19,7 @@ WITH iwd_prices AS ( FROM {{ ref('stg_major_indices') }} WHERE symbol = 'IWD' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), iwf_prices AS ( @@ -29,7 +29,7 @@ iwf_prices AS ( FROM {{ ref('stg_major_indices') }} WHERE symbol = 'IWF' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), iwm_prices AS ( @@ -39,7 +39,7 @@ iwm_prices AS ( FROM {{ ref('stg_major_indices') }} WHERE symbol = 'IWM' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), spy_prices AS ( @@ -49,7 +49,7 @@ spy_prices AS ( FROM {{ ref('stg_major_indices') }} WHERE symbol = 'SPY' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), value_growth_ratio AS ( diff --git a/dbt_project/models/signals/financial_conditions_signals.sql b/dbt_project/models/signals/financial_conditions_signals.sql index b20e8f7..f737650 100644 --- a/dbt_project/models/signals/financial_conditions_signals.sql +++ b/dbt_project/models/signals/financial_conditions_signals.sql @@ -65,36 +65,36 @@ nfci_leverage AS ( cc_delinquency AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS cc_delinquency_rate FROM {{ ref('stg_fred_series') }} WHERE series_code = 'DRCCLACBS' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), -- Bank lending standards (large and small firms) lending_large AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS lending_standards_large FROM {{ ref('stg_fred_series') }} WHERE series_code = 'DRTSCILM' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), lending_small AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS lending_standards_small FROM {{ ref('stg_fred_series') }} WHERE series_code = 'DRTSCIS' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), -- Combine NFCI components (weekly data) @@ -159,7 +159,7 @@ final AS ( FROM nfci_combined AS nc FULL OUTER JOIN lending_combined AS lc ON nc.date = lc.date FULL OUTER JOIN cc_delinquency AS cd - ON DATE_TRUNC('month', COALESCE(nc.date, lc.date)) = cd.month_date + ON DATE_TRUNC(COALESCE(nc.date, lc.date), MONTH) = cd.month_date ) SELECT @@ -204,5 +204,5 @@ SELECT END AS lending_status FROM final -WHERE date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY date DESC diff --git a/dbt_project/models/signals/housing_signals.sql b/dbt_project/models/signals/housing_signals.sql index 033016f..ed99422 100644 --- a/dbt_project/models/signals/housing_signals.sql +++ b/dbt_project/models/signals/housing_signals.sql @@ -16,24 +16,24 @@ WITH housing_starts AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS starts FROM {{ ref('stg_fred_series') }} WHERE series_code = 'HOUST' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), building_permits AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS permits FROM {{ ref('stg_fred_series') }} WHERE series_code = 'PERMIT' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), mortgage_rate AS ( @@ -48,22 +48,22 @@ mortgage_rate AS ( mortgage_monthly AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, AVG(rate_30y) AS avg_mortgage_rate, MAX(rate_30y) AS max_mortgage_rate FROM mortgage_rate - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), months_supply AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS months_of_supply FROM {{ ref('stg_fred_series') }} WHERE series_code = 'MSACSR' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), combined AS ( @@ -150,5 +150,5 @@ SELECT END AS supply_status FROM with_trends -WHERE date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY date DESC diff --git a/dbt_project/models/signals/inflation_signals.sql b/dbt_project/models/signals/inflation_signals.sql index 51ea4a6..31df0d8 100644 --- a/dbt_project/models/signals/inflation_signals.sql +++ b/dbt_project/models/signals/inflation_signals.sql @@ -156,5 +156,5 @@ SELECT END AS breakeven_status FROM combined -WHERE date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY date DESC diff --git a/dbt_project/models/signals/labor_signals.sql b/dbt_project/models/signals/labor_signals.sql index cb791ce..bcb6c39 100644 --- a/dbt_project/models/signals/labor_signals.sql +++ b/dbt_project/models/signals/labor_signals.sql @@ -18,46 +18,46 @@ WITH job_openings AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS job_openings FROM {{ ref('stg_fred_series') }} WHERE series_code = 'JTSJOL' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), unemployed AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS unemployed_count FROM {{ ref('stg_fred_series') }} WHERE series_code = 'UNEMPLOY' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), unemployment_rate AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS unrate FROM {{ ref('stg_fred_series') }} WHERE series_code = 'UNRATE' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), sahm_rule AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS sahm_rule FROM {{ ref('stg_fred_series') }} WHERE series_code = 'SAHMCURRENT' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), initial_claims AS ( @@ -72,34 +72,34 @@ initial_claims AS ( claims_monthly AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, AVG(claims) AS avg_monthly_claims, MAX(claims) AS max_monthly_claims, MIN(claims) AS min_monthly_claims FROM initial_claims - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), emp_pop_ratio AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS emratio FROM {{ ref('stg_fred_series') }} WHERE series_code = 'EMRATIO' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), quits_rate AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS quits_rate FROM {{ ref('stg_fred_series') }} WHERE series_code = 'JTSQUR' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), combined AS ( @@ -182,5 +182,5 @@ SELECT END AS quits_trend_status FROM with_trends -WHERE date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY date DESC diff --git a/dbt_project/models/signals/liquidity_signals.sql b/dbt_project/models/signals/liquidity_signals.sql index 3e1a6e6..0617fe3 100644 --- a/dbt_project/models/signals/liquidity_signals.sql +++ b/dbt_project/models/signals/liquidity_signals.sql @@ -16,46 +16,46 @@ WITH m2_data AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS m2_level FROM {{ ref('stg_fred_series') }} WHERE series_code = 'M2SL' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), m1_data AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS m1_level FROM {{ ref('stg_fred_series') }} WHERE series_code = 'M1SL' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), business_loans AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS busloans FROM {{ ref('stg_fred_series') }} WHERE series_code = 'BUSLOANS' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), total_credit AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS total_consumer_credit FROM {{ ref('stg_fred_series') }} WHERE series_code = 'TOTALSL' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), -- M2 Velocity (quarterly data) @@ -86,13 +86,13 @@ velocity_with_trend AS ( -- Fed Balance Sheet (weekly -> monthly) walcl_data AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, AVG(literal) AS walcl_avg FROM {{ ref('stg_fred_series') }} WHERE series_code = 'WALCL' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), walcl_with_changes AS ( @@ -107,13 +107,13 @@ walcl_with_changes AS ( -- Reverse Repo (daily -> monthly) rrp_data AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, AVG(literal) AS rrp_avg FROM {{ ref('stg_fred_series') }} WHERE series_code = 'RRPONTSYD' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), rrp_with_changes AS ( @@ -227,5 +227,5 @@ SELECT FROM with_growth wg CROSS JOIN velocity_with_trend vt -WHERE wg.date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE wg.date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY wg.date DESC diff --git a/dbt_project/models/signals/market_breadth_signals.sql b/dbt_project/models/signals/market_breadth_signals.sql index 543be66..c3ff240 100644 --- a/dbt_project/models/signals/market_breadth_signals.sql +++ b/dbt_project/models/signals/market_breadth_signals.sql @@ -23,7 +23,7 @@ WITH RECURSIVE stock_prices AS ( WHERE adj_close IS NOT NULL AND adj_close > 0 - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), -- Calculate moving averages for each stock @@ -100,14 +100,14 @@ daily_breadth AS ( SUM(advancing_volume) AS total_advancing_volume, SUM(declining_volume) AS total_declining_volume, -- Percentage metrics - ROUND(100.0 * SUM(above_200_ma) / NULLIF(COUNT(DISTINCT symbol), 0), 2) AS pct_above_200_ma, - ROUND(100.0 * SUM(above_50_ma) / NULLIF(COUNT(DISTINCT symbol), 0), 2) AS pct_above_50_ma, + ROUND(SAFE_DIVIDE(100.0 * SUM(above_200_ma), COUNT(DISTINCT symbol)), 2) AS pct_above_200_ma, + ROUND(SAFE_DIVIDE(100.0 * SUM(above_50_ma), COUNT(DISTINCT symbol)), 2) AS pct_above_50_ma, -- Advance/Decline ratio - ROUND(1.0 * SUM(is_advancing) / NULLIF(SUM(is_declining), 0), 3) AS ad_ratio, + ROUND(SAFE_DIVIDE(1.0 * SUM(is_advancing), SUM(is_declining)), 3) AS ad_ratio, -- Advance/Decline line contribution (advances - declines) SUM(is_advancing) - SUM(is_declining) AS ad_line_delta FROM stock_signals - WHERE date >= CURRENT_DATE - INTERVAL 2 YEAR + WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 2 YEAR) GROUP BY date HAVING COUNT(DISTINCT symbol) >= 400 -- Ensure we have most S&P 500 stocks ), @@ -116,21 +116,14 @@ breadth_base AS ( SELECT *, (advancing_stocks - declining_stocks) AS net_advances, - CASE - WHEN advancing_stocks + declining_stocks > 0 THEN ROUND( - (advancing_stocks - declining_stocks) * 1000.0 - / (advancing_stocks + declining_stocks), - 2 - ) - ELSE 0 - END AS rana, - CASE - WHEN advancing_stocks + declining_stocks > 0 THEN ROUND( - 1.0 * advancing_stocks / (advancing_stocks + declining_stocks), - 6 - ) - ELSE 0.5 - END AS adv_ratio + COALESCE( + ROUND(SAFE_DIVIDE((advancing_stocks - declining_stocks) * 1000.0, advancing_stocks + declining_stocks), 2), + 0 + ) AS rana, + COALESCE( + ROUND(SAFE_DIVIDE(1.0 * advancing_stocks, advancing_stocks + declining_stocks), 6), + 0.5 + ) AS adv_ratio FROM daily_breadth ), @@ -157,7 +150,7 @@ breadth_with_cum AS ( pct_above_200_ma - LAG(pct_above_200_ma, 5) OVER (ORDER BY date) AS breadth_5d_change, pct_above_200_ma - LAG(pct_above_200_ma, 20) OVER (ORDER BY date) AS breadth_20d_change, -- % advancing for thrust calculations - ROUND(100.0 * advancing_stocks / NULLIF(advancing_stocks + declining_stocks, 0), 2) AS pct_advancing + ROUND(SAFE_DIVIDE(100.0 * advancing_stocks, advancing_stocks + declining_stocks), 2) AS pct_advancing FROM breadth_base ), @@ -230,7 +223,7 @@ spy_prices AS ( FROM {{ ref('stg_major_indices') }} WHERE symbol = 'SPY' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), spy_with_highs AS ( @@ -249,7 +242,7 @@ sector_prices AS ( adj_close AS price FROM {{ ref('stg_us_sectors') }} WHERE adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), sector_with_sma AS ( @@ -287,7 +280,7 @@ internals_prices AS ( adj_close AS price FROM {{ ref('stg_us_sectors') }} WHERE adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) UNION ALL @@ -298,7 +291,7 @@ internals_prices AS ( FROM {{ ref('stg_major_indices') }} WHERE symbol IN ('SPY', 'QQQ') AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL 3 YEAR + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ), internals_returns AS ( @@ -306,7 +299,7 @@ internals_returns AS ( symbol, date, price, - (price / LAG(price) OVER (PARTITION BY symbol ORDER BY date) - 1.0) AS daily_return + (SAFE_DIVIDE(price, LAG(price) OVER (PARTITION BY symbol ORDER BY date)) - 1.0) AS daily_return FROM internals_prices ), @@ -407,7 +400,7 @@ SELECT ROUND(b.breadth_5d_change, 2) AS breadth_5d_change, ROUND(b.breadth_20d_change, 2) AS breadth_20d_change, -- Volume A/D ratio - ROUND(1.0 * b.total_advancing_volume / NULLIF(b.total_declining_volume, 0), 3) AS volume_ad_ratio, + ROUND(SAFE_DIVIDE(1.0 * b.total_advancing_volume, b.total_declining_volume), 3) AS volume_ad_ratio, -- McClellan + Zweig b.net_advances, b.rana AS ratio_adjusted_net_advances, @@ -431,7 +424,7 @@ SELECT -- Sector participation sp.sector_participation_count, sp.sector_total, - ROUND(100.0 * sp.sector_participation_count / NULLIF(sp.sector_total, 0), 2) AS sector_participation_pct, + ROUND(SAFE_DIVIDE(100.0 * sp.sector_participation_count, sp.sector_total), 2) AS sector_participation_pct, -- Internals correlation/dispersion ROUND(ic.avg_pair_correlation_63d, 4) AS avg_pair_correlation_63d, ROUND(id.return_dispersion, 4) AS return_dispersion, diff --git a/dbt_project/models/signals/market_volatility_signals.sql b/dbt_project/models/signals/market_volatility_signals.sql index 53849da..8c4014c 100644 --- a/dbt_project/models/signals/market_volatility_signals.sql +++ b/dbt_project/models/signals/market_volatility_signals.sql @@ -47,7 +47,7 @@ returns AS ( adj_high, adj_low, adj_close, - (adj_close / LAG(adj_close) OVER (PARTITION BY symbol ORDER BY date) - 1.0) AS daily_return + (SAFE_DIVIDE(adj_close, LAG(adj_close) OVER (PARTITION BY symbol ORDER BY date)) - 1.0) AS daily_return FROM price_base ), @@ -66,8 +66,8 @@ vol_inputs AS ( ORDER BY date ROWS BETWEEN 29 PRECEDING AND CURRENT ROW ) * SQRT(252) * 100 AS realized_vol_30d, - LN(adj_high / adj_low) AS log_hl, - LN(adj_close / NULLIF(adj_open, 0)) AS log_co + LN(SAFE_DIVIDE(adj_high, adj_low)) AS log_hl, + LN(SAFE_DIVIDE(adj_close, adj_open)) AS log_co FROM returns WHERE adj_high > 0 AND adj_low > 0 @@ -142,7 +142,7 @@ SELECT v.vix_prev_close, v.vix_close - v.vix_prev_close AS vix_daily_change, CASE - WHEN v.vix_prev_close > 0 THEN ((v.vix_close - v.vix_prev_close) / v.vix_prev_close) * 100 + WHEN v.vix_prev_close > 0 THEN SAFE_DIVIDE(v.vix_close - v.vix_prev_close, v.vix_prev_close) * 100 ELSE 0 END AS vix_daily_change_pct, spy.realized_vol_20d AS spy_realized_vol_20d, @@ -164,5 +164,5 @@ SELECT FROM vix_stats v LEFT JOIN spy ON v.date = spy.date LEFT JOIN qqq ON v.date = qqq.date -WHERE v.date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE v.date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY v.date DESC diff --git a/dbt_project/models/signals/momentum_signals.sql b/dbt_project/models/signals/momentum_signals.sql index cbbb06b..4360aac 100644 --- a/dbt_project/models/signals/momentum_signals.sql +++ b/dbt_project/models/signals/momentum_signals.sql @@ -57,8 +57,8 @@ sector_stats AS ( SELECT date, MAX(avg_momentum) - MIN(avg_momentum) AS dispersion, - ARG_MAX(symbol, avg_momentum) AS top_sector, - ARG_MIN(symbol, avg_momentum) AS bottom_sector + ARRAY_AGG(symbol ORDER BY avg_momentum DESC LIMIT 1)[OFFSET(0)] AS top_sector, + ARRAY_AGG(symbol ORDER BY avg_momentum ASC LIMIT 1)[OFFSET(0)] AS bottom_sector FROM sector_returns GROUP BY date ), @@ -185,5 +185,5 @@ final AS ( SELECT * FROM final -WHERE date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY date DESC diff --git a/dbt_project/models/signals/net_liquidity_signals.sql b/dbt_project/models/signals/net_liquidity_signals.sql index 54d91a7..614ecde 100644 --- a/dbt_project/models/signals/net_liquidity_signals.sql +++ b/dbt_project/models/signals/net_liquidity_signals.sql @@ -20,32 +20,32 @@ WITH walcl_weekly AS ( SELECT - DATE_TRUNC('week', date) AS week_date, + DATE_TRUNC(date, WEEK) AS week_date, AVG(literal) AS walcl FROM {{ ref('stg_fred_series') }} WHERE series_code = 'WALCL' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('week', date) + GROUP BY DATE_TRUNC(date, WEEK) ), wtregen_weekly AS ( SELECT - DATE_TRUNC('week', date) AS week_date, + DATE_TRUNC(date, WEEK) AS week_date, AVG(literal) AS wtregen FROM {{ ref('stg_fred_series') }} WHERE series_code = 'WTREGEN' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('week', date) + GROUP BY DATE_TRUNC(date, WEEK) ), rrp_weekly AS ( SELECT - DATE_TRUNC('week', date) AS week_date, + DATE_TRUNC(date, WEEK) AS week_date, AVG(literal) AS rrpontsyd FROM {{ ref('stg_fred_series') }} WHERE series_code = 'RRPONTSYD' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('week', date) + GROUP BY DATE_TRUNC(date, WEEK) ), combined AS ( @@ -120,5 +120,5 @@ SELECT END AS rrp_depletion_status FROM with_trends -WHERE date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY date DESC diff --git a/dbt_project/models/signals/sentiment_signals.sql b/dbt_project/models/signals/sentiment_signals.sql index 8855e28..ecd5319 100644 --- a/dbt_project/models/signals/sentiment_signals.sql +++ b/dbt_project/models/signals/sentiment_signals.sql @@ -16,79 +16,79 @@ WITH consumer_sentiment AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS umcsent FROM {{ ref('stg_fred_series') }} WHERE series_code = 'UMCSENT' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), consumer_confidence AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS confidence FROM {{ ref('stg_fred_series') }} WHERE series_code = 'CSCICP03USM665S' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), mfg_production AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS ipman FROM {{ ref('stg_fred_series') }} WHERE series_code = 'IPMAN' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), mfg_new_orders AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS new_orders FROM {{ ref('stg_fred_series') }} WHERE series_code = 'NEWORDER' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), mfg_prices AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS prices FROM {{ ref('stg_fred_series') }} WHERE series_code = 'PCUOMFG' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), mfg_employment AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS employment FROM {{ ref('stg_fred_series') }} WHERE series_code = 'MANEMP' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), mfg_inventories AS ( SELECT - DATE_TRUNC('month', date) AS month_date, + DATE_TRUNC(date, MONTH) AS month_date, MAX(literal) AS inventories FROM {{ ref('stg_fred_series') }} WHERE series_code = 'MNFCTRMPCIMSA' AND literal IS NOT NULL - GROUP BY DATE_TRUNC('month', date) + GROUP BY DATE_TRUNC(date, MONTH) ), combined AS ( @@ -210,5 +210,5 @@ SELECT END AS orders_inventories_status FROM with_yoy -WHERE date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY date DESC diff --git a/dbt_project/models/signals/technical_signals.sql b/dbt_project/models/signals/technical_signals.sql index a1170f0..23c3995 100644 --- a/dbt_project/models/signals/technical_signals.sql +++ b/dbt_project/models/signals/technical_signals.sql @@ -214,5 +214,5 @@ SELECT END AS vix_mean_reversion_status FROM combined -WHERE date >= CURRENT_DATE - INTERVAL 3 YEAR +WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 3 YEAR) ORDER BY date DESC diff --git a/dbt_project/models/signals/trade_signals.sql b/dbt_project/models/signals/trade_signals.sql index 4677ef7..15ada62 100644 --- a/dbt_project/models/signals/trade_signals.sql +++ b/dbt_project/models/signals/trade_signals.sql @@ -3,7 +3,7 @@ with dollar_broad as ( select - date_trunc('month', date) as month, + DATE_TRUNC(date, MONTH) as month, avg(value) as dollar_broad_avg from {{ ref('stg_fred_series') }} where series_code = 'DTWEXBGS' @@ -13,7 +13,7 @@ with dollar_broad as ( dollar_em as ( select - date_trunc('month', date) as month, + DATE_TRUNC(date, MONTH) as month, avg(value) as em_dollar_avg from {{ ref('stg_fred_series') }} where series_code = 'DTWEXEMEGS' diff --git a/dbt_project/models/staging/corporate_actions.sql b/dbt_project/models/staging/corporate_actions.sql index 14a6a2a..a25a3b4 100644 --- a/dbt_project/models/staging/corporate_actions.sql +++ b/dbt_project/models/staging/corporate_actions.sql @@ -40,39 +40,46 @@ WITH splits_api AS ( -- factors (e.g. 1.01) where normal daily volatility could match. splits_api_adjusted AS ( SELECT - sa.source_table, - sa.symbol, + source_table, + symbol, CASE - WHEN sa.split_factor >= 1.2 - AND p.prev_close IS NOT NULL - AND p.prev_close > 0 - AND p.open > 0 + WHEN split_factor >= 1.2 + AND prev_close IS NOT NULL + AND prev_close > 0 + AND open > 0 AND ABS( - p.open / p.prev_close - 1.0 / sa.split_factor - ) / (1.0 / sa.split_factor) < 0.10 - THEN p.date -- shift to the day-before date - ELSE sa.date + open / prev_close - 1.0 / split_factor + ) / (1.0 / split_factor) < 0.10 + THEN prior_price_date -- shift to the day-before date + ELSE date END AS date, - sa.action_type, - sa.split_factor, - sa.dividend_amount, - sa.detection_method - FROM splits_api AS sa - LEFT JOIN ( + action_type, + split_factor, + dividend_amount, + detection_method + FROM ( SELECT - symbol, - CAST(date AS DATE) AS date, - open, - LAG(close) OVER (PARTITION BY symbol ORDER BY date) AS prev_close - FROM {{ ref('stg_sp500_companies_prices') }} - ) AS p - ON sa.symbol = p.symbol - AND p.date = ( - SELECT MAX(CAST(d.date AS DATE)) - FROM {{ ref('stg_sp500_companies_prices') }} AS d - WHERE d.symbol = sa.symbol - AND CAST(d.date AS DATE) < sa.date - ) + sa.*, + p.date AS prior_price_date, + p.open, + p.prev_close, + ROW_NUMBER() OVER ( + PARTITION BY sa.source_table, sa.symbol, sa.date, sa.action_type + ORDER BY p.date DESC + ) AS prior_price_rank + FROM splits_api AS sa + LEFT JOIN ( + SELECT + symbol, + CAST(date AS DATE) AS date, + open, + LAG(close) OVER (PARTITION BY symbol ORDER BY date) AS prev_close + FROM {{ ref('stg_sp500_companies_prices') }} + ) AS p + ON sa.symbol = p.symbol + AND p.date < sa.date + ) + WHERE prior_price_rank = 1 ), -- Collect all OHLC-based split and dividend detections @@ -170,25 +177,29 @@ UNION ALL -- the authoritative date by 1+ days when raw prices were already post-split. SELECT o.* FROM ohlc_based AS o -LEFT JOIN splits_api_adjusted AS s - ON o.source_table = s.source_table - AND o.symbol = s.symbol - AND o.action_type = 'split' - AND s.action_type = 'split' - AND ABS(DATEDIFF('day', o.date, s.date)) <= 5 WHERE o.detection_method != 'heuristic' - AND s.symbol IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM splits_api_adjusted AS s + WHERE o.source_table = s.source_table + AND o.symbol = s.symbol + AND o.action_type = 'split' + AND s.action_type = 'split' + AND ABS(DATE_DIFF(o.date, s.date, DAY)) <= 5 + ) UNION ALL -- heuristic rows: wider ±5 day window dedup against splits_api_adjusted SELECT o.* FROM ohlc_based AS o -LEFT JOIN splits_api_adjusted AS s - ON o.source_table = s.source_table - AND o.symbol = s.symbol - AND o.action_type = 'split' - AND s.action_type = 'split' - AND ABS(DATEDIFF('day', o.date, s.date)) <= 5 WHERE o.detection_method = 'heuristic' - AND s.symbol IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM splits_api_adjusted AS s + WHERE o.source_table = s.source_table + AND o.symbol = s.symbol + AND o.action_type = 'split' + AND s.action_type = 'split' + AND ABS(DATE_DIFF(o.date, s.date, DAY)) <= 5 + ) diff --git a/dbt_project/models/staging/schema.yml b/dbt_project/models/staging/schema.yml index c77174b..87a9b4c 100644 --- a/dbt_project/models/staging/schema.yml +++ b/dbt_project/models/staging/schema.yml @@ -12,8 +12,7 @@ models: tests: - not_null - name: value - tests: - - not_null + description: Numeric FRED observation value; null when FRED reports a missing-value sentinel. - name: stg_housing_inventory description: > @@ -417,6 +416,7 @@ models: - accepted_values: arguments: values: [1, 2, 3, 4] + quote: false - name: num_sections description: Number of sections in the meeting minutes - name: content_length diff --git a/dbt_project/models/staging/stg_currency.sql b/dbt_project/models/staging/stg_currency.sql index 6988e00..8959492 100644 --- a/dbt_project/models/staging/stg_currency.sql +++ b/dbt_project/models/staging/stg_currency.sql @@ -17,5 +17,5 @@ SELECT price_currency, symbol, exchange, - cast(date AS date) AS date + SAFE_CAST(SUBSTR(CAST(date AS STRING), 1, 10) AS DATE) AS date FROM {{ source('staging', 'currency_etfs_raw') }} diff --git a/dbt_project/models/staging/stg_earnings_calendar.sql b/dbt_project/models/staging/stg_earnings_calendar.sql index f039719..5a126d3 100644 --- a/dbt_project/models/staging/stg_earnings_calendar.sql +++ b/dbt_project/models/staging/stg_earnings_calendar.sql @@ -8,6 +8,18 @@ with source as ( select * from {{ source('staging', 'earnings_calendar') }} ), +typed as ( + select + *, + safe_cast(report_date as date) as report_dt, + safe_cast(eps_actual as float64) as eps_actual_float, + safe_cast(eps_estimated as float64) as eps_estimated_float, + safe_cast(revenue_actual as float64) as revenue_actual_float, + safe_cast(revenue_estimated as float64) as revenue_estimated_float + from source + where report_date is not null +), + cleaned as ( select event_id, @@ -23,40 +35,39 @@ cleaned as ( event_type, source, fetched_at, - try_cast(report_date as date) as report_date, + report_dt as report_date, -- Computed fields - extract(year from try_cast(report_date as date)) as year, - extract(month from try_cast(report_date as date)) as month, - extract(week from try_cast(report_date as date)) as week_of_year, - extract(dow from try_cast(report_date as date)) as day_of_week, + extract(year from report_dt) as year, + extract(month from report_dt) as month, + extract(week from report_dt) as week_of_year, + extract(dayofweek from report_dt) as day_of_week, -- EPS surprise calculation case - when try_cast(eps_actual as double) is not null and try_cast(eps_estimated as double) is not null and try_cast(eps_estimated as double) != 0 - then ((try_cast(eps_actual as double) - try_cast(eps_estimated as double)) / abs(try_cast(eps_estimated as double))) * 100 + when eps_actual_float is not null and eps_estimated_float is not null and eps_estimated_float != 0 + then ((eps_actual_float - eps_estimated_float) / abs(eps_estimated_float)) * 100 end as eps_surprise_pct, -- Beat/miss indicator case - when try_cast(eps_actual as double) is not null and try_cast(eps_estimated as double) is not null + when eps_actual_float is not null and eps_estimated_float is not null then case - when try_cast(eps_actual as double) > try_cast(eps_estimated as double) then 'beat' - when try_cast(eps_actual as double) < try_cast(eps_estimated as double) then 'miss' + when eps_actual_float > eps_estimated_float then 'beat' + when eps_actual_float < eps_estimated_float then 'miss' else 'met' end end as eps_result, -- Revenue surprise case - when try_cast(revenue_actual as double) is not null and try_cast(revenue_estimated as double) is not null and try_cast(revenue_estimated as double) != 0 - then ((try_cast(revenue_actual as double) - try_cast(revenue_estimated as double)) / abs(try_cast(revenue_estimated as double))) * 100 + when revenue_actual_float is not null and revenue_estimated_float is not null and revenue_estimated_float != 0 + then ((revenue_actual_float - revenue_estimated_float) / abs(revenue_estimated_float)) * 100 end as revenue_surprise_pct, -- Upcoming flag - coalesce(try_cast(report_date as date) > current_date, false) as is_upcoming, + coalesce(report_dt > current_date(), false) as is_upcoming, -- Days until report - date_diff('day', current_date, try_cast(report_date as date)) as days_until_report, + date_diff(report_dt, current_date(), day) as days_until_report, -- Has reported flag coalesce(eps_actual is not null, false) as has_reported - from source - where report_date is not null + from typed ) select * from cleaned diff --git a/dbt_project/models/staging/stg_economic_calendar.sql b/dbt_project/models/staging/stg_economic_calendar.sql index fcbb83e..203970d 100644 --- a/dbt_project/models/staging/stg_economic_calendar.sql +++ b/dbt_project/models/staging/stg_economic_calendar.sql @@ -8,29 +8,46 @@ with source as ( select * from {{ source('staging', 'economic_calendar') }} ), +typed as ( + select + *, + safe_cast(date as timestamp) as event_timestamp + from source + where date is not null +), + cleaned as ( select - event_id, + coalesce( + event_id, + farm_fingerprint(concat( + coalesce(cast(event_timestamp as string), ''), + '|', + coalesce(title, ''), + '|', + coalesce(country, '') + )) + ) as event_id, title, country, -- Parse event datetime - cast(try_cast(event_date as timestamp with time zone) as date) as event_date, + cast(event_timestamp as date) as event_date, impact, forecast, - forecast_numeric, + forecast as forecast_numeric, previous, - previous_numeric, + previous as previous_numeric, + actual, event_type, source, - feed, fetched_at, - try_cast(event_date as timestamp with time zone) as event_datetime, + event_timestamp as event_datetime, -- Extract time components - extract(year from try_cast(event_date as timestamp with time zone)) as year, - extract(month from try_cast(event_date as timestamp with time zone)) as month, - extract(week from try_cast(event_date as timestamp with time zone)) as week_of_year, - extract(dow from try_cast(event_date as timestamp with time zone)) as day_of_week, - extract(hour from try_cast(event_date as timestamp with time zone)) as hour, + extract(year from event_timestamp) as year, + extract(month from event_timestamp) as month, + extract(week from event_timestamp) as week_of_year, + extract(dayofweek from event_timestamp) as day_of_week, + extract(hour from event_timestamp) as hour, -- Impact level as numeric for sorting case impact when 'High' then 3 @@ -40,12 +57,15 @@ cleaned as ( else -1 end as impact_level, -- Flag for upcoming events - coalesce(try_cast(event_date as timestamp with time zone) > current_timestamp, false) as is_upcoming, + coalesce(event_timestamp > current_timestamp(), false) as is_upcoming, -- Days until event - date_diff('day', current_date, cast(try_cast(event_date as timestamp with time zone) as date)) + date_diff(cast(event_timestamp as date), current_date(), day) as days_until_event - from source - where event_date is not null + from typed ) select * from cleaned +qualify row_number() over ( + partition by event_id + order by fetched_at desc +) = 1 diff --git a/dbt_project/models/staging/stg_fixed_income.sql b/dbt_project/models/staging/stg_fixed_income.sql index 5390988..a792c06 100644 --- a/dbt_project/models/staging/stg_fixed_income.sql +++ b/dbt_project/models/staging/stg_fixed_income.sql @@ -17,5 +17,5 @@ SELECT price_currency, symbol, exchange, - cast(date AS date) AS date + SAFE_CAST(SUBSTR(CAST(date AS STRING), 1, 10) AS DATE) AS date FROM {{ source('staging', 'fixed_income_etfs_raw') }} diff --git a/dbt_project/models/staging/stg_fomc_meeting_summaries.sql b/dbt_project/models/staging/stg_fomc_meeting_summaries.sql index 604237e..7f2b311 100644 --- a/dbt_project/models/staging/stg_fomc_meeting_summaries.sql +++ b/dbt_project/models/staging/stg_fomc_meeting_summaries.sql @@ -1,5 +1,6 @@ {{ config( + enabled=false, tags=['staging', 'fomc', 'summaries'] ) }} diff --git a/dbt_project/models/staging/stg_fomc_meetings_enhanced.sql b/dbt_project/models/staging/stg_fomc_meetings_enhanced.sql index 2865a7f..c9d57f3 100644 --- a/dbt_project/models/staging/stg_fomc_meetings_enhanced.sql +++ b/dbt_project/models/staging/stg_fomc_meetings_enhanced.sql @@ -1,5 +1,6 @@ {{ config( + enabled=false, tags=['staging', 'fomc', 'calendar'] ) }} diff --git a/dbt_project/models/staging/stg_fomc_minutes.sql b/dbt_project/models/staging/stg_fomc_minutes.sql index 51354e8..9a7d494 100644 --- a/dbt_project/models/staging/stg_fomc_minutes.sql +++ b/dbt_project/models/staging/stg_fomc_minutes.sql @@ -8,9 +8,16 @@ with source as ( select * from {{ source('staging', 'fomc_minutes_metadata') }} ), +typed as ( + select + *, + safe_cast(meeting_date as date) as meeting_dt + from source +), + cleaned as ( select - meeting_date, + meeting_dt as meeting_date, year, title, gcs_path, @@ -20,10 +27,10 @@ cleaned as ( num_sections, content_length, -- Extract quarter from meeting date - extract(quarter from meeting_date) as quarter, + extract(quarter from meeting_dt) as quarter, -- Extract month - extract(month from meeting_date) as month - from source + extract(month from meeting_dt) as month + from typed where year >= 2010 ) diff --git a/dbt_project/models/staging/stg_fomc_transcripts.sql b/dbt_project/models/staging/stg_fomc_transcripts.sql index 3cbf07c..048f5dc 100644 --- a/dbt_project/models/staging/stg_fomc_transcripts.sql +++ b/dbt_project/models/staging/stg_fomc_transcripts.sql @@ -8,10 +8,18 @@ with source as ( select * from {{ source('staging', 'fomc_transcripts') }} ), +typed as ( + select + *, + safe_cast(meeting_date as date) as meeting_dt + from source + where meeting_date is not null +), + cleaned as ( select transcript_id, - meeting_date, + meeting_dt as meeting_date, full_text, word_count, page_count, @@ -20,13 +28,12 @@ cleaned as ( processed_date, created_at, -- Extract year and quarter for partitioning/filtering - extract(year from meeting_date) as year, - extract(quarter from meeting_date) as quarter, - extract(month from meeting_date) as month, + extract(year from meeting_dt) as year, + extract(quarter from meeting_dt) as quarter, + extract(month from meeting_dt) as month, -- Calculate transcript age (years since released) - date_diff('year', meeting_date, current_date) as years_since_meeting - from source - where meeting_date is not null + date_diff(current_date(), meeting_dt, year) as years_since_meeting + from typed ) select * from cleaned diff --git a/dbt_project/models/staging/stg_fred_series.sql b/dbt_project/models/staging/stg_fred_series.sql index e06e1cc..e3b8368 100644 --- a/dbt_project/models/staging/stg_fred_series.sql +++ b/dbt_project/models/staging/stg_fred_series.sql @@ -1,6 +1,6 @@ SELECT - fr.date, - fr.value, + CAST(fr.date AS DATE) AS date, + SAFE_CAST(NULLIF(fr.value, '.') AS FLOAT64) AS value, fr.series_code, fr.literal, map.series_name, diff --git a/dbt_project/models/staging/stg_global_markets.sql b/dbt_project/models/staging/stg_global_markets.sql index 1a20edb..60965e4 100644 --- a/dbt_project/models/staging/stg_global_markets.sql +++ b/dbt_project/models/staging/stg_global_markets.sql @@ -17,5 +17,5 @@ SELECT price_currency, symbol, exchange, - cast(date AS date) AS date + SAFE_CAST(SUBSTR(CAST(date AS STRING), 1, 10) AS DATE) AS date FROM {{ source('staging', 'global_markets_raw') }} diff --git a/dbt_project/models/staging/stg_major_indices.sql b/dbt_project/models/staging/stg_major_indices.sql index a262b04..c9c65b2 100644 --- a/dbt_project/models/staging/stg_major_indices.sql +++ b/dbt_project/models/staging/stg_major_indices.sql @@ -17,5 +17,5 @@ SELECT price_currency, symbol, exchange, - cast(date AS date) AS date + SAFE_CAST(SUBSTR(CAST(date AS STRING), 1, 10) AS DATE) AS date FROM {{ source('staging', 'major_indices_raw') }} diff --git a/dbt_project/models/staging/stg_nasdaq_companies_prices.sql b/dbt_project/models/staging/stg_nasdaq_companies_prices.sql index 3da63c5..96e2f66 100644 --- a/dbt_project/models/staging/stg_nasdaq_companies_prices.sql +++ b/dbt_project/models/staging/stg_nasdaq_companies_prices.sql @@ -1,3 +1,5 @@ +{{ config(enabled=false) }} + SELECT open, high, diff --git a/dbt_project/models/staging/stg_realtor_country_history.sql b/dbt_project/models/staging/stg_realtor_country_history.sql index 4d3b728..6b2de6a 100644 --- a/dbt_project/models/staging/stg_realtor_country_history.sql +++ b/dbt_project/models/staging/stg_realtor_country_history.sql @@ -1,2 +1,4 @@ +{{ config(enabled=false) }} + SELECT * FROM {{ source('staging', 'realtor_country_raw') }} diff --git a/dbt_project/models/staging/stg_realtor_county_history.sql b/dbt_project/models/staging/stg_realtor_county_history.sql index 30f17af..8330faa 100644 --- a/dbt_project/models/staging/stg_realtor_county_history.sql +++ b/dbt_project/models/staging/stg_realtor_county_history.sql @@ -1,2 +1,4 @@ +{{ config(enabled=false) }} + SELECT * FROM {{ source('staging', 'realtor_county_raw') }} diff --git a/dbt_project/models/staging/stg_realtor_metro_history.sql b/dbt_project/models/staging/stg_realtor_metro_history.sql index 65e7a9f..1cc8239 100644 --- a/dbt_project/models/staging/stg_realtor_metro_history.sql +++ b/dbt_project/models/staging/stg_realtor_metro_history.sql @@ -1,2 +1,4 @@ +{{ config(enabled=false) }} + SELECT * FROM {{ source('staging', 'realtor_metro_raw') }} diff --git a/dbt_project/models/staging/stg_realtor_state_history.sql b/dbt_project/models/staging/stg_realtor_state_history.sql index 24a2edd..e559ede 100644 --- a/dbt_project/models/staging/stg_realtor_state_history.sql +++ b/dbt_project/models/staging/stg_realtor_state_history.sql @@ -1,2 +1,4 @@ +{{ config(enabled=false) }} + SELECT * FROM {{ source('staging', 'realtor_state_raw') }} diff --git a/dbt_project/models/staging/stg_realtor_zip_history.sql b/dbt_project/models/staging/stg_realtor_zip_history.sql index 9bd12ec..744a18f 100644 --- a/dbt_project/models/staging/stg_realtor_zip_history.sql +++ b/dbt_project/models/staging/stg_realtor_zip_history.sql @@ -1,2 +1,4 @@ +{{ config(enabled=false) }} + SELECT * FROM {{ source('staging', 'realtor_zip_raw') }} diff --git a/dbt_project/models/staging/stg_reddit_comments.sql b/dbt_project/models/staging/stg_reddit_comments.sql index c0ce7b2..8ac07ed 100644 --- a/dbt_project/models/staging/stg_reddit_comments.sql +++ b/dbt_project/models/staging/stg_reddit_comments.sql @@ -27,7 +27,7 @@ cleaned as ( coalesce(links != '' and links is not null, false) as has_links, coalesce(author = '[deleted]', false) as is_deleted, -- Is this a top-level comment (reply to post, not another comment)? - coalesce(parent_id like 't3\_%' escape '\', false) as is_top_level, + coalesce(starts_with(parent_id, 't3_'), false) as is_top_level, -- Engagement bucket case when score >= 50 then 'high' diff --git a/dbt_project/models/staging/stg_reddit_post_content.sql b/dbt_project/models/staging/stg_reddit_post_content.sql index 4e9b90d..239331a 100644 --- a/dbt_project/models/staging/stg_reddit_post_content.sql +++ b/dbt_project/models/staging/stg_reddit_post_content.sql @@ -42,7 +42,7 @@ cleaned as ( left join posts p on s.post_id = p.post_id where -- Filter out ads that may have slipped through - lower(s.subreddit) not like 'u\_%' escape '\' + not starts_with(lower(s.subreddit), 'u_') ) select * from cleaned diff --git a/dbt_project/models/staging/stg_reddit_posts.sql b/dbt_project/models/staging/stg_reddit_posts.sql index 2d09644..cb5d616 100644 --- a/dbt_project/models/staging/stg_reddit_posts.sql +++ b/dbt_project/models/staging/stg_reddit_posts.sql @@ -20,7 +20,7 @@ cleaned as ( permalink, lower(subreddit) as subreddit, domain, - partition_date, + safe_cast(partition_date as date) as partition_date, fetched_at, -- Derive additional fields coalesce(domain like '%self.%', false) as is_self_post, @@ -28,8 +28,8 @@ cleaned as ( -- Engagement metrics score + num_comments as engagement_score, case - when num_comments > 0 then cast(score as double) / cast(num_comments as double) - else cast(score as double) + when num_comments > 0 then cast(score as float64) / cast(num_comments as float64) + else cast(score as float64) end as score_per_comment, -- Time features extract(dayofweek from created_utc) as day_of_week, @@ -38,12 +38,12 @@ cleaned as ( coalesce(author = '[deleted]', false) as is_deleted from source where - partition_date >= current_date - interval '90 days' + safe_cast(partition_date as date) >= date_sub(current_date(), interval 90 day) and score is not null and title is not null and length(title) > 0 -- Filter out Reddit promoted/ad posts (u_* subreddits) - and lower(subreddit) not like 'u\_%' escape '\' + and not starts_with(lower(subreddit), 'u_') ) select * from cleaned diff --git a/dbt_project/models/staging/stg_reddit_ticker_mentions.sql b/dbt_project/models/staging/stg_reddit_ticker_mentions.sql index 6aaa6c2..b2686ca 100644 --- a/dbt_project/models/staging/stg_reddit_ticker_mentions.sql +++ b/dbt_project/models/staging/stg_reddit_ticker_mentions.sql @@ -1,5 +1,6 @@ {{ config( + enabled=false, materialized='view', tags=['staging', 'reddit', 'tickers'] ) diff --git a/dbt_project/models/staging/stg_sp500_companies_prices.sql b/dbt_project/models/staging/stg_sp500_companies_prices.sql index b7b3ca9..17f0511 100644 --- a/dbt_project/models/staging/stg_sp500_companies_prices.sql +++ b/dbt_project/models/staging/stg_sp500_companies_prices.sql @@ -17,5 +17,5 @@ SELECT price_currency, symbol, exchange, - cast(date AS date) AS date + SAFE_CAST(SUBSTR(CAST(date AS STRING), 1, 10) AS DATE) AS date FROM {{ source('staging', 'sp500_companies_prices_raw') }} diff --git a/dbt_project/models/staging/stg_transcript_topics.sql b/dbt_project/models/staging/stg_transcript_topics.sql index 2c2023f..33d8bc3 100644 --- a/dbt_project/models/staging/stg_transcript_topics.sql +++ b/dbt_project/models/staging/stg_transcript_topics.sql @@ -1,5 +1,6 @@ {{ config( + enabled=false, tags=['staging', 'fomc', 'topics'] ) }} diff --git a/dbt_project/models/staging/stg_us_sectors.sql b/dbt_project/models/staging/stg_us_sectors.sql index 5e874fd..7d83b27 100644 --- a/dbt_project/models/staging/stg_us_sectors.sql +++ b/dbt_project/models/staging/stg_us_sectors.sql @@ -17,5 +17,5 @@ SELECT price_currency, symbol, exchange, - cast(date AS date) AS date + SAFE_CAST(SUBSTR(CAST(date AS STRING), 1, 10) AS DATE) AS date FROM {{ source('staging', 'us_sector_etfs_raw') }} diff --git a/dbt_project/models/staging/telemetry/schema.yml b/dbt_project/models/staging/telemetry/schema.yml deleted file mode 100644 index f957783..0000000 --- a/dbt_project/models/staging/telemetry/schema.yml +++ /dev/null @@ -1,164 +0,0 @@ -version: 2 - -models: - - name: stg_telemetry_events - description: Cleaned and parsed telemetry events with extracted properties - columns: - - name: id - description: Primary key from source SQLite table - tests: - - unique - - not_null - - - name: event_type - description: Type of event tracked - tests: - - not_null - - accepted_values: - arguments: - values: ['page_view', 'navigation_click', 'chat_query', 'example_question_click', - 'chart_export', 'theme_change', 'filter_applied', 'sign_in', 'sign_up', - 'sign_out', 'api_error', 'app_error', 'web_vitals'] - - - name: session_id - description: Session identifier for grouping events - tests: - - not_null - - - name: event_timestamp - description: Event timestamp converted from milliseconds - tests: - - not_null - - - name: event_date - description: Event date (truncated to day) - tests: - - not_null - - - name: is_error_event - description: Flag indicating if this is an error event - tests: - - not_null - - accepted_values: - arguments: - values: [true, false] - - - name: is_auth_event - description: Flag indicating if this is an authentication event - tests: - - not_null - - accepted_values: - arguments: - values: [true, false] - - - name: is_navigation_event - description: Flag indicating if this is a navigation event - tests: - - not_null - - accepted_values: - arguments: - values: [true, false] - - - name: is_feature_usage_event - description: Flag indicating if this is a feature usage event - tests: - - not_null - - accepted_values: - arguments: - values: [true, false] - - - name: is_performance_event - description: Flag indicating if this is a performance monitoring event - tests: - - not_null - - accepted_values: - arguments: - values: [true, false] - - - name: stg_sessions - description: Aggregated session-level metrics derived from telemetry events - columns: - - name: session_id - description: Unique session identifier - tests: - - unique - - not_null - - - name: session_start - description: Timestamp of first event in session - tests: - - not_null - - - name: session_end - description: Timestamp of last event in session - tests: - - not_null - - - name: session_date - description: Date of session start - tests: - - not_null - - - name: session_duration_seconds - description: Duration of session in seconds - tests: - - not_null - - dbt_utils.expression_is_true: - arguments: - expression: ">= 0" - - - name: session_duration_minutes - description: Duration of session in minutes - tests: - - not_null - - dbt_utils.expression_is_true: - arguments: - expression: ">= 0" - - - name: total_events - description: Total number of events in session - tests: - - not_null - - dbt_utils.expression_is_true: - arguments: - expression: "> 0" - - - name: page_views - description: Number of page views in session - tests: - - not_null - - dbt_utils.expression_is_true: - arguments: - expression: ">= 0" - - - name: is_authenticated - description: Whether the session is authenticated - tests: - - not_null - - accepted_values: - arguments: - values: [true, false] - - - name: is_bounce - description: Whether the session was a bounce (single page view) - tests: - - not_null - - accepted_values: - arguments: - values: [true, false] - - - name: has_errors - description: Whether the session encountered any errors - tests: - - not_null - - accepted_values: - arguments: - values: [true, false] - - - name: engagement_score - description: Calculated engagement score (0-7 scale) - tests: - - not_null - - dbt_utils.expression_is_true: - arguments: - expression: ">= 0 and <= 7" diff --git a/dbt_project/models/staging/telemetry/sources.yml b/dbt_project/models/staging/telemetry/sources.yml deleted file mode 100644 index a9072e1..0000000 --- a/dbt_project/models/staging/telemetry/sources.yml +++ /dev/null @@ -1,54 +0,0 @@ -version: 2 - -sources: - - name: raw_telemetry - description: Raw telemetry data - database: "{{ env_var('BIGQUERY_PROJECT', env_var('BIGQUERY_PROJECT_ID', 'econ-data-project-478800')) }}" - schema: economics_raw - tables: - - name: telemetry_events_raw - description: User interaction events from frontend telemetry system - columns: - - name: id - description: Primary key from SQLite - tests: - - not_null - - name: event_type - description: Type of event tracked (page_view, chat_query, sign_in, etc.) - - name: session_id - description: Session identifier for grouping events - tests: - - not_null - - name: user_id - description: User ID (nullable for anonymous sessions) - - name: timestamp_ms - description: Event timestamp in milliseconds since epoch - tests: - - not_null - - name: properties - description: JSON string of event-specific properties - - name: created_at - description: Database insertion timestamp - - name: replicated_at - description: Timestamp when replicated to BigQuery - - - name: users_raw - description: User account information - columns: - - name: id - description: Primary key - tests: - - unique - - not_null - - name: email - description: User email address - tests: - - not_null - - name: password_hash - description: Hashed password (never in plaintext) - - name: created_at - description: Account creation timestamp - - name: updated_at - description: Last update timestamp - - name: replicated_at - description: Timestamp when replicated to BigQuery diff --git a/dbt_project/models/staging/telemetry/stg_sessions.sql b/dbt_project/models/staging/telemetry/stg_sessions.sql deleted file mode 100644 index 000c939..0000000 --- a/dbt_project/models/staging/telemetry/stg_sessions.sql +++ /dev/null @@ -1,123 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - -with session_events as ( - select - session_id, - user_id, - event_type, - event_timestamp, - event_date, - is_error_event, - is_auth_event, - is_navigation_event, - is_feature_usage_event, - page_url, - page_path - from {{ ref('stg_telemetry_events') }} -), - -session_aggregates as ( - select - session_id, - - max(user_id) as user_id, - count(distinct if(user_id is not null, user_id, null)) - as unique_users_in_session, - - min(event_timestamp) as session_start, - max(event_timestamp) as session_end, - date_trunc(min(event_timestamp), day) as session_date, - date_trunc(min(event_timestamp), hour) as session_hour, - - count(*) as total_events, - countif(is_navigation_event) as page_views, - countif(is_feature_usage_event) as feature_interactions, - countif(is_auth_event) as auth_events, - countif(is_error_event) as error_count, - - count(distinct if(page_path is not null, page_path, null)) - as unique_pages_visited, - - -- SAFE_OFFSET(0) = first element; array ordered asc gives landing page - (array_agg(if(page_path is not null, page_path, null) ignore nulls - order by event_timestamp asc))[safe_offset(0)] as landing_page, - (array_agg(if(page_path is not null, page_path, null) ignore nulls - order by event_timestamp desc))[safe_offset(0)] as exit_page, - - countif(event_type = 'page_view') as page_view_count, - countif(event_type = 'chat_query') as chat_query_count, - countif(event_type = 'chart_export') as chart_export_count, - countif(event_type = 'filter_applied') as filter_applied_count, - countif(event_type = 'sign_in') as sign_in_count, - countif(event_type = 'sign_up') as sign_up_count, - countif(event_type = 'sign_out') as sign_out_count, - countif(event_type = 'api_error') as api_error_count, - countif(event_type = 'app_error') as app_error_count, - countif(event_type = 'web_vitals') as web_vitals_count - - from session_events - group by session_id -), - -final as ( - select - session_id, - user_id, - unique_users_in_session, - - session_start, - session_end, - session_date, - session_hour, - - total_events, - page_views, - feature_interactions, - auth_events, - error_count, - unique_pages_visited, - landing_page, - exit_page, - - page_view_count, - chat_query_count, - chart_export_count, - filter_applied_count, - sign_in_count, - sign_up_count, - sign_out_count, - api_error_count, - app_error_count, - web_vitals_count, - timestamp_diff(session_end, session_start, second) - as session_duration_seconds, - round( - cast(timestamp_diff(session_end, session_start, second) as float64) - / 60.0, - 2 - ) as session_duration_minutes, - - coalesce(user_id is not null, false) as is_authenticated, - coalesce(page_views = 1, false) as is_bounce, - coalesce(error_count > 0, false) as has_errors, - coalesce(chat_query_count > 0, false) as used_chat, - coalesce(chart_export_count > 0, false) as exported_chart, - coalesce(sign_up_count > 0, false) as is_signup_session, - coalesce(sign_in_count > 0, false) as is_signin_session, - - ( - (case when page_views > 0 then 1 else 0 end) - + (case when chat_query_count > 0 then 2 else 0 end) - + (case when chart_export_count > 0 then 2 else 0 end) - + (case when filter_applied_count > 0 then 1 else 0 end) - + (case when unique_pages_visited > 3 then 1 else 0 end) - ) as engagement_score - - from session_aggregates -) - -select * from final diff --git a/dbt_project/models/staging/telemetry/stg_telemetry_events.sql b/dbt_project/models/staging/telemetry/stg_telemetry_events.sql deleted file mode 100644 index 595a4aa..0000000 --- a/dbt_project/models/staging/telemetry/stg_telemetry_events.sql +++ /dev/null @@ -1,104 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='append_new_columns' - ) -}} - -with source as ( - select * from {{ source('raw_telemetry', 'telemetry_events_raw') }} - {% if is_incremental() %} - where replicated_at > (select max(replicated_at) from {{ this }}) - {% endif %} -), - -parsed as ( - select - -- Primary identifiers - id, - event_type, - session_id, - user_id, - - -- Timestamps - properties as raw_properties, - created_at, - replicated_at, - - -- Parse JSON properties (using DuckDB's JSON functions) - -- Navigation properties - to_timestamp(timestamp_ms / 1000.0) as event_timestamp, - date_trunc('day', to_timestamp(timestamp_ms / 1000.0)) as event_date, - date_trunc('hour', to_timestamp(timestamp_ms / 1000.0)) as event_hour, - - -- Feature usage properties - try_cast(json_extract_string(properties, '$.page_url') as varchar) as page_url, - try_cast(json_extract_string(properties, '$.page_title') as varchar) as page_title, - try_cast(json_extract_string(properties, '$.referrer') as varchar) as referrer, - try_cast(json_extract_string(properties, '$.feature_name') as varchar) as feature_name, - - -- Error properties - try_cast(json_extract_string(properties, '$.query_text') as varchar) as query_text, - try_cast(json_extract_string(properties, '$.chart_type') as varchar) as chart_type, - try_cast(json_extract_string(properties, '$.export_format') as varchar) as export_format, - - -- Performance properties (Web Vitals) - try_cast(json_extract_string(properties, '$.error_message') as varchar) as error_message, - try_cast(json_extract_string(properties, '$.error_stack') as varchar) as error_stack, - try_cast(json_extract_string(properties, '$.error_code') as varchar) as error_code, - - -- Auth properties - try_cast(json_extract_string(properties, '$.metric_name') as varchar) as metric_name, - try_cast(json_extract_string(properties, '$.metric_value') as double) as metric_value, - - -- Device/Browser context - try_cast(json_extract_string(properties, '$.metric_rating') as varchar) as metric_rating, - try_cast(json_extract_string(properties, '$.auth_method') as varchar) as auth_method, - try_cast(json_extract_string(properties, '$.auth_provider') as varchar) as auth_provider, - try_cast(json_extract_string(properties, '$.user_agent') as varchar) as user_agent, - try_cast(json_extract_string(properties, '$.screen_width') as integer) as screen_width, - - -- Keep full properties JSON for ad-hoc analysis - try_cast(json_extract_string(properties, '$.screen_height') as integer) as screen_height, - - -- Metadata - try_cast(json_extract_string(properties, '$.viewport_width') as integer) as viewport_width, - try_cast(json_extract_string(properties, '$.viewport_height') as integer) as viewport_height - - from source -), - -final as ( - select - *, - - -- Derived flags - coalesce(event_type in ('api_error', 'app_error'), false) as is_error_event, - - coalesce(event_type in ('sign_in', 'sign_up', 'sign_out'), false) as is_auth_event, - - coalesce(event_type = 'page_view', false) as is_navigation_event, - - coalesce(event_type in ('chat_query', 'chart_export', 'filter_applied', 'example_question_click'), false) as is_feature_usage_event, - - coalesce(event_type = 'web_vitals', false) as is_performance_event, - - -- Extract page path from full URL - case - when page_url is not null - then - regexp_extract(page_url, '^https?://[^/]+(/.*)$', 1) - end as page_path, - - -- Extract domain from URL - case - when page_url is not null - then - regexp_extract(page_url, '^https?://([^/]+)', 1) - end as page_domain - - from parsed -) - -select * from final diff --git a/dbt_project/profiles.yml b/dbt_project/profiles.yml index 1df59cc..6a947a9 100644 --- a/dbt_project/profiles.yml +++ b/dbt_project/profiles.yml @@ -3,6 +3,8 @@ econ_database: outputs: dev: type: bigquery + # ADC-backed auth. Locally run `gcloud auth application-default login`; + # deployed environments should use an attached or federated service account. method: oauth project: "{{ env_var('BIGQUERY_PROJECT', env_var('BIGQUERY_PROJECT_ID', 'econ-data-project-478800')) }}" # Models land in *_dev datasets: economics_staging_dev, economics_marts_dev, etc. @@ -14,6 +16,8 @@ econ_database: staging: type: bigquery + # ADC-backed auth. Locally run `gcloud auth application-default login`; + # deployed environments should use an attached or federated service account. method: oauth project: "{{ env_var('BIGQUERY_PROJECT', env_var('BIGQUERY_PROJECT_ID', 'econ-data-project-478800')) }}" # Models land in *_staging datasets for pre-prod validation. @@ -23,10 +27,10 @@ econ_database: timeout_seconds: 300 prod: - # On GCE, ADC automatically uses the attached VM service account — - # no keyfile needed. Ensure the service account has bigquery.dataEditor - # and bigquery.jobUser on the project (provisioned in Terraform Phase 0). type: bigquery + # ADC-backed auth. On GCE, ADC automatically uses the attached VM service + # account — no keyfile needed. Ensure the service account has + # bigquery.dataEditor and bigquery.jobUser on the project. method: oauth project: "{{ env_var('BIGQUERY_PROJECT', env_var('BIGQUERY_PROJECT_ID', 'econ-data-project-478800')) }}" # Models land in canonical datasets: economics_staging, economics_marts, etc. diff --git a/dbt_project/tests/diagnostic_zero_forward_returns.sql b/dbt_project/tests/diagnostic_zero_forward_returns.sql deleted file mode 100644 index efd0f3a..0000000 --- a/dbt_project/tests/diagnostic_zero_forward_returns.sql +++ /dev/null @@ -1,36 +0,0 @@ --- Diagnostic query to investigate zero forward returns --- Run this to see the actual rows that are failing the test --- This will help determine if they're legitimate 0% returns or data quality issues - -{% set models_to_test = [ - 'currency_analysis_return', - 'global_markets_analysis_return', - 'major_indicies_analysis_return', - 'us_sector_analysis_return', - 'fixed_income_analysis_return' -] %} - -{% for model in models_to_test %} - SELECT - '{{ model }}' as model_name, - symbol, - exchange, - month_date, - year_val, - quarter_num, - quarterly_avg_close, - pct_change_q1_forward, - pct_change_q2_forward, - pct_change_q3_forward, - pct_change_q4_forward, - quarterly_avg_close as expected_forward_price_for_zero_return - FROM {{ ref(model) }} - WHERE pct_change_q1_forward = 0 - AND quarterly_avg_close > 0 - {% if not loop.last %} - UNION ALL - {% endif %} -{% endfor %} - -ORDER BY model_name, symbol, exchange, month_date - diff --git a/dbt_project/tests/test_forward_returns_all_quarters.sql b/dbt_project/tests/test_forward_returns_all_quarters.sql deleted file mode 100644 index 96b89ff..0000000 --- a/dbt_project/tests/test_forward_returns_all_quarters.sql +++ /dev/null @@ -1,39 +0,0 @@ --- Test: Check for unexpected inconsistencies in forward returns --- This test flags cases where Q1 forward exists but Q2 is missing (unexpected) --- Note: It's normal for Q2/Q3/Q4 to be missing at the end of the dataset, so we exclude recent quarters - -{{ config(severity='warn') }} - -{% set models_to_test = [ - 'currency_analysis_return', - 'global_markets_analysis_return', - 'major_indicies_analysis_return', - 'us_sector_analysis_return', - 'fixed_income_analysis_return' -] %} - -{% for model in models_to_test %} - SELECT - t.symbol, - t.exchange, - t.year_val, - t.quarter_num, - t.month_date, - t.pct_change_q1_forward, - t.pct_change_q2_forward, - 'q2_missing_but_q1_exists' as inconsistency_type - FROM {{ ref(model) }} t - CROSS JOIN ( - SELECT MAX(month_date) as latest_date - FROM {{ ref(model) }} - ) m - WHERE t.pct_change_q1_forward IS NOT NULL - AND t.pct_change_q2_forward IS NULL - -- Only flag if this isn't within the last year (where missing Q2 is expected) - AND t.month_date < DATE_TRUNC('year', m.latest_date) - - {% if not loop.last %} - UNION ALL - {% endif %} -{% endfor %} - diff --git a/dbt_project/tests/test_forward_returns_not_zero.sql b/dbt_project/tests/test_forward_returns_not_zero.sql deleted file mode 100644 index bb0c939..0000000 --- a/dbt_project/tests/test_forward_returns_not_zero.sql +++ /dev/null @@ -1,36 +0,0 @@ --- Test: Forward returns should not be exactly zero (rounded) when prices differ --- A 0% return is valid if prices are identical, but we flag cases where --- the calculation might have rounded to 0 incorrectly or there's a data issue --- Note: This test allows for legitimate 0% returns (no price change) - -{{ config(severity='warn') }} - -{% set models_to_test = [ - 'currency_analysis_return', - 'global_markets_analysis_return', - 'major_indicies_analysis_return', - 'us_sector_analysis_return', - 'fixed_income_analysis_return' -] %} - -{% for model in models_to_test %} - SELECT - symbol, - exchange, - month_date, - pct_change_q1_forward, - quarterly_avg_close, - 'zero_forward_return' as issue_type - FROM {{ ref(model) }} - WHERE ABS(pct_change_q1_forward) < 0.01 - AND pct_change_q1_forward IS NOT NULL - AND quarterly_avg_close > 0 - -- Only flag if this seems suspicious (you can adjust this condition) - -- For now, we'll flag all 0 values for investigation - -- In production, you might want to allow legitimate 0% returns - - {% if not loop.last %} - UNION ALL - {% endif %} -{% endfor %} - diff --git a/dbt_project/tests/test_forward_returns_same_quarter.sql b/dbt_project/tests/test_forward_returns_same_quarter.sql deleted file mode 100644 index b88d100..0000000 --- a/dbt_project/tests/test_forward_returns_same_quarter.sql +++ /dev/null @@ -1,29 +0,0 @@ --- Test: Months within the same quarter should have the same forward return values --- This test catches the LEAD() window function bug where forward returns were calculated incorrectly - -{% set models_to_test = [ - 'currency_analysis_return', - 'global_markets_analysis_return', - 'major_indicies_analysis_return', - 'us_sector_analysis_return', - 'fixed_income_analysis_return' -] %} - -{% for model in models_to_test %} - SELECT - symbol, - exchange, - year_val, - quarter_num, - COUNT(DISTINCT pct_change_q1_forward) as distinct_q1_values, - COUNT(*) as total_rows - FROM {{ ref(model) }} - WHERE pct_change_q1_forward IS NOT NULL - GROUP BY symbol, exchange, year_val, quarter_num - HAVING COUNT(DISTINCT pct_change_q1_forward) > 1 - - {% if not loop.last %} - UNION ALL - {% endif %} -{% endfor %} - diff --git a/dbt_project/tests/test_weekly_data_completeness.sql b/dbt_project/tests/test_weekly_data_completeness.sql index b8bc6b9..36ce347 100644 --- a/dbt_project/tests/test_weekly_data_completeness.sql +++ b/dbt_project/tests/test_weekly_data_completeness.sql @@ -5,89 +5,89 @@ WITH all_model_results AS ( SELECT 'stg_us_sectors' AS model_name, - DATE_TRUNC('week', date) AS week_start, + DATE_TRUNC(date, WEEK) AS week_start, COUNT(*) AS record_count FROM {{ ref('stg_us_sectors') }} - WHERE date >= CURRENT_DATE - INTERVAL '12 weeks' - AND date < CURRENT_DATE - INTERVAL '1 week' - GROUP BY DATE_TRUNC('week', date) + WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 12 WEEK) + AND date < DATE_SUB(CURRENT_DATE(), INTERVAL 1 WEEK) + GROUP BY DATE_TRUNC(date, WEEK) UNION ALL SELECT 'stg_currency' AS model_name, - DATE_TRUNC('week', date) AS week_start, + DATE_TRUNC(date, WEEK) AS week_start, COUNT(*) AS record_count FROM {{ ref('stg_currency') }} - WHERE date >= CURRENT_DATE - INTERVAL '12 weeks' - AND date < CURRENT_DATE - INTERVAL '1 week' - GROUP BY DATE_TRUNC('week', date) + WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 12 WEEK) + AND date < DATE_SUB(CURRENT_DATE(), INTERVAL 1 WEEK) + GROUP BY DATE_TRUNC(date, WEEK) UNION ALL SELECT 'stg_major_indices' AS model_name, - DATE_TRUNC('week', date) AS week_start, + DATE_TRUNC(date, WEEK) AS week_start, COUNT(*) AS record_count FROM {{ ref('stg_major_indices') }} - WHERE date >= CURRENT_DATE - INTERVAL '12 weeks' - AND date < CURRENT_DATE - INTERVAL '1 week' - GROUP BY DATE_TRUNC('week', date) + WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 12 WEEK) + AND date < DATE_SUB(CURRENT_DATE(), INTERVAL 1 WEEK) + GROUP BY DATE_TRUNC(date, WEEK) UNION ALL SELECT 'stg_fixed_income' AS model_name, - DATE_TRUNC('week', date) AS week_start, + DATE_TRUNC(date, WEEK) AS week_start, COUNT(*) AS record_count FROM {{ ref('stg_fixed_income') }} - WHERE date >= CURRENT_DATE - INTERVAL '12 weeks' - AND date < CURRENT_DATE - INTERVAL '1 week' - GROUP BY DATE_TRUNC('week', date) + WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 12 WEEK) + AND date < DATE_SUB(CURRENT_DATE(), INTERVAL 1 WEEK) + GROUP BY DATE_TRUNC(date, WEEK) UNION ALL SELECT 'stg_global_markets' AS model_name, - DATE_TRUNC('week', date) AS week_start, + DATE_TRUNC(date, WEEK) AS week_start, COUNT(*) AS record_count FROM {{ ref('stg_global_markets') }} - WHERE date >= CURRENT_DATE - INTERVAL '12 weeks' - AND date < CURRENT_DATE - INTERVAL '1 week' - GROUP BY DATE_TRUNC('week', date) + WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 12 WEEK) + AND date < DATE_SUB(CURRENT_DATE(), INTERVAL 1 WEEK) + GROUP BY DATE_TRUNC(date, WEEK) UNION ALL SELECT 'stg_agriculture_commodities' AS model_name, - DATE_TRUNC('week', date) AS week_start, + DATE_TRUNC(date, WEEK) AS week_start, COUNT(*) AS record_count FROM {{ ref('stg_agriculture_commodities') }} - WHERE date >= CURRENT_DATE - INTERVAL '12 weeks' - AND date < CURRENT_DATE - INTERVAL '1 week' - GROUP BY DATE_TRUNC('week', date) + WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 12 WEEK) + AND date < DATE_SUB(CURRENT_DATE(), INTERVAL 1 WEEK) + GROUP BY DATE_TRUNC(date, WEEK) UNION ALL SELECT 'stg_energy_commodities' AS model_name, - DATE_TRUNC('week', date) AS week_start, + DATE_TRUNC(date, WEEK) AS week_start, COUNT(*) AS record_count FROM {{ ref('stg_energy_commodities') }} - WHERE date >= CURRENT_DATE - INTERVAL '12 weeks' - AND date < CURRENT_DATE - INTERVAL '1 week' - GROUP BY DATE_TRUNC('week', date) + WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 12 WEEK) + AND date < DATE_SUB(CURRENT_DATE(), INTERVAL 1 WEEK) + GROUP BY DATE_TRUNC(date, WEEK) UNION ALL SELECT 'stg_input_commodities' AS model_name, - DATE_TRUNC('week', date) AS week_start, + DATE_TRUNC(date, WEEK) AS week_start, COUNT(*) AS record_count FROM {{ ref('stg_input_commodities') }} - WHERE date >= CURRENT_DATE - INTERVAL '12 weeks' - AND date < CURRENT_DATE - INTERVAL '1 week' - GROUP BY DATE_TRUNC('week', date) + WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 12 WEEK) + AND date < DATE_SUB(CURRENT_DATE(), INTERVAL 1 WEEK) + GROUP BY DATE_TRUNC(date, WEEK) ), expected_weeks AS ( SELECT DISTINCT week_start diff --git a/dbt_project/tests/test_yearly_data_completeness.sql b/dbt_project/tests/test_yearly_data_completeness.sql index 7aabd9a..29f1edb 100644 --- a/dbt_project/tests/test_yearly_data_completeness.sql +++ b/dbt_project/tests/test_yearly_data_completeness.sql @@ -6,9 +6,9 @@ WITH all_model_results AS ( SELECT 'stg_fred_series' AS model_name, series_code AS identifier, - EXTRACT(YEAR FROM date) AS year_val + EXTRACT(YEAR FROM SAFE_CAST(date AS DATE)) AS year_val FROM {{ ref('stg_fred_series') }} - WHERE date >= CURRENT_DATE - INTERVAL '5 years' + WHERE SAFE_CAST(date AS DATE) >= DATE_SUB(CURRENT_DATE(), INTERVAL 5 YEAR) UNION ALL @@ -17,7 +17,7 @@ WITH all_model_results AS ( 'housing' AS identifier, CAST(LEFT(time, 4) AS INTEGER) AS year_val FROM {{ ref('stg_housing_inventory') }} - WHERE CAST(LEFT(time, 4) AS INTEGER) >= EXTRACT(YEAR FROM CURRENT_DATE) - 5 + WHERE CAST(LEFT(time, 4) AS INTEGER) >= EXTRACT(YEAR FROM CURRENT_DATE()) - 5 UNION ALL @@ -26,16 +26,16 @@ WITH all_model_results AS ( 'housing_pulse' AS identifier, CAST(LEFT(TIME, 4) AS INTEGER) AS year_val FROM {{ ref('stg_housing_pulse') }} - WHERE CAST(LEFT(TIME, 4) AS INTEGER) >= EXTRACT(YEAR FROM CURRENT_DATE) - 5 + WHERE CAST(LEFT(TIME, 4) AS INTEGER) >= EXTRACT(YEAR FROM CURRENT_DATE()) - 5 UNION ALL SELECT 'stg_treasury_yields' AS model_name, 'treasury' AS identifier, - EXTRACT(YEAR FROM date) AS year_val + EXTRACT(YEAR FROM SAFE_CAST(date AS DATE)) AS year_val FROM {{ ref('stg_treasury_yields') }} - WHERE date >= CURRENT_DATE - INTERVAL '5 years' + WHERE SAFE_CAST(date AS DATE) >= DATE_SUB(CURRENT_DATE(), INTERVAL 5 YEAR) ), expected_years AS ( SELECT DISTINCT diff --git a/docs/dbt_project/dbt-platform-fusion-onboarding.md b/docs/dbt_project/dbt-platform-fusion-onboarding.md index 7de9bbd..c7990ac 100644 --- a/docs/dbt_project/dbt-platform-fusion-onboarding.md +++ b/docs/dbt_project/dbt-platform-fusion-onboarding.md @@ -46,6 +46,31 @@ Required environment variables: | `ICEBERG_BUCKET_NAME` | GCS bucket backing BigLake Iceberg tables. | | `DBT_TARGET` | `dev`, `staging`, or `prod`. | +## Local ADC Setup + +Local dbt runs use Application Default Credentials through `method: oauth` in +`dbt_project/profiles.yml`. No service-account keyfile is required for the +standard path. + +For user ADC: + +```bash +gcloud auth application-default login +gcloud auth application-default set-quota-project "$BIGQUERY_PROJECT" +``` + +For service-account ADC via impersonation: + +```bash +gcloud auth application-default login \ + --impersonate-service-account service-account@project.iam.gserviceaccount.com +gcloud auth application-default set-quota-project "$BIGQUERY_PROJECT" +``` + +The impersonating user needs `roles/iam.serviceAccountTokenCreator` on the +service account. The service account still needs the BigQuery permissions listed +above. + ## Environments Configure three environments: @@ -113,6 +138,8 @@ Local Wizard validation after issue setup: - moving source freshness settings under `config:`; - moving generic test parameters under `arguments:`; - updating `catalogs.yml` to the Fusion BigLake catalog schema. +- BigQuery auth remains ADC-based through `method: oauth`; local compile requires + `gcloud auth application-default login` or service-account ADC impersonation. ## Dagster Orchestration Decision From 164258bc88afac93f443fc8296777e21905c3eef Mon Sep 17 00:00:00 2001 From: Alex Noonan <48368867+C00ldudeNoonan@users.noreply.github.com> Date: Thu, 11 Jun 2026 16:58:08 -0400 Subject: [PATCH 4/4] docs: clarify codex PR review request --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 30262df..83069e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,7 +66,7 @@ - Prefer small, focused commits with clear intent. - PRs should include a concise summary, linked issues (if applicable), and screenshots for UI changes. - Never merge PRs. Always leave merging to a human reviewer. -- Add a PR comment tagging `@codex` to request review. +- Add a PR comment tagging `@codex` to request review immediately after opening every PR. Use `gh pr comment --body "@codex please review this PR."` and mention the review request in the handoff. ## Configuration & Secrets - Create `macro_agents/.env` with required API keys (FRED, MarketStack, OpenAI, MotherDuck).