Merge pull request #36 from C00ldudeNoonan/backtesting

analysis updates

Author: C00ldudeNoonan

.github/workflows/branch_deployments.yml

@@ -65,7 +65,7 @@ jobs:
           pip install dbt-core dbt-duckdb
           cd project-repo/macro_agents/dbt_project
           dbt deps || true
-          dbt parse --target prod || dbt parse
+          dbt parse
 
       - name: Python Executable Deploy
         if: steps.prerun.outputs.result == 'pex-deploy'

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
     - name: Install dependencies with uv
       run: |
         cd macro_agents
-        uv sync --dev
+        uv sync --extra dev
     
     - name: Lint with ruff
       continue-on-error: true

.github/workflows/deploy.yml

@@ -68,7 +68,7 @@ jobs:
           pip install dbt-core dbt-duckdb
           cd project-repo/macro_agents/dbt_project
           dbt deps || true
-          dbt parse --target prod || dbt parse
+          dbt parse
 
       - name: Python Executable Deploy
         if: steps.prerun.outputs.result == 'pex-deploy'

macro_agents/pyproject.toml

@@ -3,34 +3,32 @@ name = "macro_agents"
 requires-python = ">=3.10.19,<3.14"
 version = "0.1.0"
 dependencies = [
-    "dagster>=1.11.4",
+    "dagster>=1.11.4,<2.0.0",  # Add upper bound
     "dagster-cloud",
     "dagster-duckdb",
-    "polars",
+    "polars>=0.20.0,<1.0.0",  # Pin polars
     "dagster-dbt",
-    "requests",
-    "dbt-core",
-    "google-api-core",
-    "google-auth",
-    "google-cloud-bigquery",
-    "dbt-duckdb",
-    "dagster-shared>=1.11.4",
-    "dagster-pipes>=1.11.4",
-    "dagster-dg-cli>=1.11.4",
-    "dagster-webserver>=1.11.4",
-    "pyarrow>=21.0.0",
-    "beautifulsoup4",
-    "lxml",
-    "ruff>=0.12.7",
-    "tenacity>=9.1.2",
-    "dspy>=2.6.27",
-    "sqlfluff>=3.4.2",
-    "sqlfluff-templater-dbt>=3.4.2",
+    "requests>=2.31.0,<3.0.0",
+    "dbt-core>=1.10.0,<2.0.0",  # Pin dbt-core
+    "google-api-core>=2.0.0,<3.0.0",
+    "google-auth>=2.0.0,<3.0.0",
+    "google-cloud-bigquery>=3.0.0,<4.0.0",
+    "dbt-duckdb>=1.10.0,<2.0.0",
+    "dagster-shared>=1.11.4,<2.0.0",
+    "dagster-pipes>=1.11.4,<2.0.0",
+    "dagster-dg-cli>=1.11.4,<2.0.0",
+    "dagster-webserver>=1.11.4,<2.0.0",
+    "pyarrow>=21.0.0,<22.0.0",  # Pin pyarrow
+    "beautifulsoup4>=4.12.0,<5.0.0",
+    "lxml>=5.0.0,<6.0.0",
+    # Move dev tools to optional-dependencies
+    "tenacity>=9.1.2,<10.0.0",
+    "dspy>=2.6.27,<3.0.0",
     "dagster-sling",
-    "google-cloud-storage>=2.10.0",
-    "openai>=1.0.0",
-    "anthropic>=0.18.0",
-    "google-generativeai>=0.3.0",
+    "google-cloud-storage>=2.10.0,<3.0.0",
+    "openai>=1.0.0,<2.0.0",
+    "anthropic>=0.18.0,<1.0.0",
+    "google-generativeai>=0.3.0,<1.0.0",
 ]
 
 [project.optional-dependencies]
@@ -40,7 +38,9 @@ dev = [
     "pytest-cov",
     "pytest-xdist",
     "mypy",
-    "ruff",
+    "ruff>=0.12.7,<1.0.0",  # Move ruff here
+    "sqlfluff>=3.4.2,<4.0.0",  # Move sqlfluff here
+    "sqlfluff-templater-dbt>=3.4.2,<4.0.0",  # Move here
 ]
 
 [build-system]

macro_agents/src/macro_agents/defs/agents/backtest_economy_state_analyzer.py

@@ -40,12 +40,17 @@ class BacktestConfig(dg.Config):
     description="Backtest economy state analysis with historical data cutoff",
     deps=[
         dg.AssetKey(["fred_series_latest_aggregates_snapshot"]),
+        dg.AssetKey(["fred_monthly_diff"]),
         dg.AssetKey(["leading_econ_return_indicator_snapshot"]),
         dg.AssetKey(["us_sector_summary_snapshot"]),
+        dg.AssetKey(["major_indicies_summary"]),
         dg.AssetKey(["energy_commodities_summary_snapshot"]),
         dg.AssetKey(["input_commodities_summary_snapshot"]),
         dg.AssetKey(["agriculture_commodities_summary_snapshot"]),
         dg.AssetKey(["financial_conditions_index"]),
+        dg.AssetKey(["housing_inventory_latest_aggregates"]),
+        dg.AssetKey(["housing_mortgage_rates"]),
+        dg.AssetKey(["stg_treasury_yields"]),
     ],
 )
 def backtest_analyze_economy_state(
@@ -97,6 +102,21 @@ def backtest_analyze_economy_state(
         md, cutoff_date=config.backtest_date
     )
 
+    context.log.info("Gathering housing market data with cutoff date...")
+    housing_data = economic_analysis.get_housing_data(
+        md, cutoff_date=config.backtest_date
+    )
+
+    context.log.info("Gathering yield curve data with cutoff date...")
+    yield_curve_data = economic_analysis.get_yield_curve_data(
+        md, cutoff_date=config.backtest_date
+    )
+
+    context.log.info("Gathering economic trends data with cutoff date...")
+    economic_trends = economic_analysis.get_economic_trends(
+        md, cutoff_date=config.backtest_date
+    )
+
     optimized_analyzer = None
     if config.use_optimized_models and economic_analysis.use_optimized_models:
         optimized_analyzer = economic_analysis.load_optimized_module(
@@ -126,12 +146,15 @@ def backtest_analyze_economy_state(
     )
 
     context.log.info(
-        f"Running economy state analysis with historical data including FCI (personality: {config.personality})..."
+        f"Running economy state analysis with historical data including FCI, housing, yield curve, and trends (personality: {config.personality})..."
     )
     analysis_result = analyzer_to_use(
         economic_data=economic_data,
         commodity_data=commodity_data,
         financial_conditions_index=fci_data,
+        housing_data=housing_data,
+        yield_curve_data=yield_curve_data,
+        economic_trends=economic_trends,
         personality=config.personality,
     )
 
@@ -155,6 +178,16 @@ def backtest_analyze_economy_state(
                 "agriculture_commodities_summary_snapshot",
             ],
             "financial_conditions_index_table": "financial_conditions_index",
+            "housing_data_tables": [
+                "housing_inventory_latest_aggregates",
+                "housing_mortgage_rates",
+            ],
+            "yield_curve_table": "stg_treasury_yields",
+            "economic_trends_table": "fred_monthly_diff",
+            "market_data_tables": [
+                "us_sector_summary_snapshot",
+                "major_indicies_summary",
+            ],
         },
     }