From b13fe1e8d1db7f852d60c3a182a6302497d80b8f Mon Sep 17 00:00:00 2001 From: Alex Noonan <48368867+C00ldudeNoonan@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:59:46 -0400 Subject: [PATCH] fix: replace DuckDB-style quoted INTERVAL literals with BigQuery syntax (#139) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BigQuery rejects `INTERVAL '3 years'` (quoted string, DuckDB/Postgres style) with a confusing downstream parse error reported at the next clause rather than at the INTERVAL itself. The fix is BigQuery's native unquoted form: `INTERVAL 3 YEAR`. Fixed 4 confirmed-broken jobs plus 3 more instances found by inspection: - signals/fear_greed_composite.py (6 occurrences, f-string lookback var) - signals/entropy_complexity.py - signals/absorption_ratio.py - signals/network_correlation.py - domains/sec/cik_history.py - analysis/economy_state/domain_data_fetchers.py (4 occurrences) - analysis/economy_state/data_access.py (3 occurrences) Verified by running each of the 4 job-level fixes directly against a live BigQuery target (dagster job execute) before and after: all four previously failed with "Syntax error: Unexpected keyword ORDER"; after the fix, none show that error. fear_greed_job and entropy_complexity_job now fail on an unrelated, separate bug (unqualified stg_* table references resolving to the wrong dataset — filed separately); absorption_ratio_job and network_correlation_job now fail only because this local dev environment's BigQuery dataset has never been backfilled with sp500_companies_prices_raw (data gap, not a code defect). Co-Authored-By: Claude Fable 5 --- .../defs/analysis/economy_state/data_access.py | 8 ++++---- .../economy_state/domain_data_fetchers.py | 8 ++++---- .../macro_agents/defs/domains/sec/cik_history.py | 2 +- .../defs/signals/absorption_ratio.py | 2 +- .../defs/signals/entropy_complexity.py | 2 +- .../defs/signals/fear_greed_composite.py | 16 +++++++++------- .../defs/signals/network_correlation.py | 2 +- 7 files changed, 21 insertions(+), 19 deletions(-) diff --git a/macro_agents/src/macro_agents/defs/analysis/economy_state/data_access.py b/macro_agents/src/macro_agents/defs/analysis/economy_state/data_access.py index 1e97cac..55b835e 100644 --- a/macro_agents/src/macro_agents/defs/analysis/economy_state/data_access.py +++ b/macro_agents/src/macro_agents/defs/analysis/economy_state/data_access.py @@ -17,7 +17,7 @@ def get_economic_data( month_filter = f"AND month = (SELECT MAX(month) FROM agent_fred_series_latest_aggregates_snapshot WHERE snapshot_date = '{cutoff_date}')" limit_clause = f"LIMIT {max_series}" if max_series else "" else: - month_filter = f"AND month >= (SELECT MAX(month) - INTERVAL '{months_per_series - 1} months' FROM agent_fred_series_latest_aggregates_snapshot WHERE snapshot_date = '{cutoff_date}')" + month_filter = f"AND month >= (SELECT MAX(month) - INTERVAL {months_per_series - 1} MONTH FROM agent_fred_series_latest_aggregates_snapshot WHERE snapshot_date = '{cutoff_date}')" limit_clause = ( f"LIMIT {max_series * months_per_series}" if max_series else "" ) @@ -43,7 +43,7 @@ def get_economic_data( month_filter = "AND month = (SELECT MAX(month) FROM agent_fred_series_latest_aggregates)" limit_clause = f"LIMIT {max_series}" if max_series else "" else: - month_filter = f"AND month >= (SELECT MAX(month) - INTERVAL '{months_per_series - 1} months' FROM agent_fred_series_latest_aggregates)" + month_filter = f"AND month >= (SELECT MAX(month) - INTERVAL {months_per_series - 1} MONTH FROM agent_fred_series_latest_aggregates)" limit_clause = ( f"LIMIT {max_series * months_per_series}" if max_series else "" ) @@ -370,12 +370,12 @@ def get_housing_data( if latest_month_only: month_filter = f"AND month = (SELECT MAX(month) FROM agent_housing_inventory_latest_aggregates WHERE month <= '{cutoff_date}')" else: - month_filter = f"AND month >= (SELECT MAX(month) - INTERVAL '{months_limit - 1} months' FROM agent_housing_inventory_latest_aggregates WHERE month <= '{cutoff_date}')" + month_filter = f"AND month >= (SELECT MAX(month) - INTERVAL {months_limit - 1} MONTH FROM agent_housing_inventory_latest_aggregates WHERE month <= '{cutoff_date}')" else: if latest_month_only: month_filter = "AND month = (SELECT MAX(month) FROM agent_housing_inventory_latest_aggregates)" else: - month_filter = f"AND month >= (SELECT MAX(month) - INTERVAL '{months_limit - 1} months' FROM agent_housing_inventory_latest_aggregates)" + month_filter = f"AND month >= (SELECT MAX(month) - INTERVAL {months_limit - 1} MONTH FROM agent_housing_inventory_latest_aggregates)" inventory_query = f""" SELECT diff --git a/macro_agents/src/macro_agents/defs/analysis/economy_state/domain_data_fetchers.py b/macro_agents/src/macro_agents/defs/analysis/economy_state/domain_data_fetchers.py index f9ad029..945c234 100644 --- a/macro_agents/src/macro_agents/defs/analysis/economy_state/domain_data_fetchers.py +++ b/macro_agents/src/macro_agents/defs/analysis/economy_state/domain_data_fetchers.py @@ -73,7 +73,7 @@ def get_labor_market_data( AND series_code IN ('{series_list}') AND current_value IS NOT NULL AND month >= ( - SELECT MAX(month) - INTERVAL '{max_months - 1} months' + SELECT MAX(month) - INTERVAL {max_months - 1} MONTH FROM agent_fred_series_latest_aggregates_snapshot WHERE snapshot_date = '{cutoff_date}' ) @@ -94,7 +94,7 @@ def get_labor_market_data( WHERE series_code IN ('{series_list}') AND current_value IS NOT NULL AND month >= ( - SELECT MAX(month) - INTERVAL '{max_months - 1} months' + SELECT MAX(month) - INTERVAL {max_months - 1} MONTH FROM agent_fred_series_latest_aggregates ) ORDER BY series_name, month DESC @@ -271,7 +271,7 @@ def get_credit_data( AND series_code IN ('{series_list}') AND current_value IS NOT NULL AND month >= ( - SELECT MAX(month) - INTERVAL '{max_months - 1} months' + SELECT MAX(month) - INTERVAL {max_months - 1} MONTH FROM agent_fred_series_latest_aggregates_snapshot WHERE snapshot_date = '{cutoff_date}' ) @@ -291,7 +291,7 @@ def get_credit_data( WHERE series_code IN ('{series_list}') AND current_value IS NOT NULL AND month >= ( - SELECT MAX(month) - INTERVAL '{max_months - 1} months' + SELECT MAX(month) - INTERVAL {max_months - 1} MONTH FROM agent_fred_series_latest_aggregates ) ORDER BY series_name, month DESC diff --git a/macro_agents/src/macro_agents/defs/domains/sec/cik_history.py b/macro_agents/src/macro_agents/defs/domains/sec/cik_history.py index dc1ba4a..6b3a875 100644 --- a/macro_agents/src/macro_agents/defs/domains/sec/cik_history.py +++ b/macro_agents/src/macro_agents/defs/domains/sec/cik_history.py @@ -160,7 +160,7 @@ def sec_company_cik_history( SELECT current_symbol, MAX(created_at) as last_updated FROM sec_company_cik_history GROUP BY current_symbol - HAVING MAX(created_at) > CURRENT_TIMESTAMP - INTERVAL '90 days' + HAVING MAX(created_at) > CURRENT_TIMESTAMP - INTERVAL 90 DAY """) recent_symbols = set(existing_df["current_symbol"].to_list()) except Exception: diff --git a/macro_agents/src/macro_agents/defs/signals/absorption_ratio.py b/macro_agents/src/macro_agents/defs/signals/absorption_ratio.py index 23ae84f..420847f 100644 --- a/macro_agents/src/macro_agents/defs/signals/absorption_ratio.py +++ b/macro_agents/src/macro_agents/defs/signals/absorption_ratio.py @@ -36,7 +36,7 @@ def absorption_ratio_signals( SELECT date, symbol, adj_close FROM sp500_companies_prices_raw WHERE adj_close IS NOT NULL - AND CAST(date AS DATE) >= CURRENT_DATE - INTERVAL '3 years' + AND CAST(date AS DATE) >= CURRENT_DATE - INTERVAL 3 YEAR ORDER BY date, symbol """, read_only=True, diff --git a/macro_agents/src/macro_agents/defs/signals/entropy_complexity.py b/macro_agents/src/macro_agents/defs/signals/entropy_complexity.py index eb4caf6..24f10b4 100644 --- a/macro_agents/src/macro_agents/defs/signals/entropy_complexity.py +++ b/macro_agents/src/macro_agents/defs/signals/entropy_complexity.py @@ -72,7 +72,7 @@ def entropy_complexity_signals( FROM stg_major_indices WHERE symbol IN ('SPY', 'QQQ') AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL '3 years' + AND date >= CURRENT_DATE - INTERVAL 3 YEAR ORDER BY date """, read_only=True, diff --git a/macro_agents/src/macro_agents/defs/signals/fear_greed_composite.py b/macro_agents/src/macro_agents/defs/signals/fear_greed_composite.py index 6342add..99389f4 100644 --- a/macro_agents/src/macro_agents/defs/signals/fear_greed_composite.py +++ b/macro_agents/src/macro_agents/defs/signals/fear_greed_composite.py @@ -52,7 +52,9 @@ def fear_greed_signals( context: dg.AssetExecutionContext, bq: BigQueryWarehouseResource, ) -> dg.MaterializeResult: - lookback = "3 years" + # BigQuery INTERVAL literals must be unquoted (`INTERVAL 3 YEAR`), unlike + # the DuckDB/Postgres quoted-string form (`INTERVAL '3 years'`). + lookback_years = 3 # 1. Market Momentum: SPY vs 125-day MA context.log.info("Computing market momentum component...") @@ -62,7 +64,7 @@ def fear_greed_signals( AVG(adj_close) OVER (ORDER BY date ROWS BETWEEN 124 PRECEDING AND CURRENT ROW) AS sma_125 FROM stg_major_indices WHERE symbol = 'SPY' AND adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL '{lookback}' + AND date >= CURRENT_DATE - INTERVAL {lookback_years} YEAR ORDER BY date """, read_only=True, @@ -81,7 +83,7 @@ def fear_greed_signals( MIN(adj_close) OVER (PARTITION BY symbol ORDER BY date ROWS BETWEEN 251 PRECEDING AND CURRENT ROW) AS low_52w FROM stg_sp500_companies_prices WHERE adj_close IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL '{lookback}' - INTERVAL '1 year' + AND date >= CURRENT_DATE - INTERVAL {lookback_years} YEAR - INTERVAL 1 YEAR ), daily_stats AS ( SELECT @@ -93,7 +95,7 @@ def fear_greed_signals( ) SELECT date, new_highs, new_lows, new_highs - new_lows AS net_new_highs FROM daily_stats - WHERE date >= CURRENT_DATE - INTERVAL '{lookback}' + WHERE date >= CURRENT_DATE - INTERVAL {lookback_years} YEAR ORDER BY date """, read_only=True, @@ -107,7 +109,7 @@ def fear_greed_signals( AVG(literal) OVER (ORDER BY date ROWS BETWEEN 49 PRECEDING AND CURRENT ROW) AS vix_sma_50 FROM stg_fred_series WHERE series_code = 'VIXCLS' AND literal IS NOT NULL - AND date >= CURRENT_DATE - INTERVAL '{lookback}' + AND date >= CURRENT_DATE - INTERVAL {lookback_years} YEAR ORDER BY date """, read_only=True, @@ -129,7 +131,7 @@ def fear_greed_signals( s.spy_20d_ret - g.govt_20d_ret AS stock_bond_diff FROM spy_ret s INNER JOIN govt_ret g ON s.date = g.date - WHERE s.date >= CURRENT_DATE - INTERVAL '{lookback}' + WHERE s.date >= CURRENT_DATE - INTERVAL {lookback_years} YEAR AND s.spy_20d_ret IS NOT NULL AND g.govt_20d_ret IS NOT NULL ORDER BY s.date """, @@ -150,7 +152,7 @@ def fear_greed_signals( ) SELECT h.date, h.hy_spread, i.ig_spread, h.hy_spread - i.ig_spread AS hy_ig_diff FROM hy h INNER JOIN ig i ON h.date = i.date - WHERE h.date >= CURRENT_DATE - INTERVAL '{lookback}' + WHERE h.date >= CURRENT_DATE - INTERVAL {lookback_years} YEAR ORDER BY h.date """, read_only=True, diff --git a/macro_agents/src/macro_agents/defs/signals/network_correlation.py b/macro_agents/src/macro_agents/defs/signals/network_correlation.py index 7235b5d..62c5dcf 100644 --- a/macro_agents/src/macro_agents/defs/signals/network_correlation.py +++ b/macro_agents/src/macro_agents/defs/signals/network_correlation.py @@ -38,7 +38,7 @@ def network_correlation_signals( SELECT date, symbol, adj_close FROM sp500_companies_prices_raw WHERE adj_close IS NOT NULL - AND CAST(date AS DATE) >= CURRENT_DATE - INTERVAL '2 years' + AND CAST(date AS DATE) >= CURRENT_DATE - INTERVAL 2 YEAR ORDER BY date, symbol """, read_only=True,