Skip to content

Commit 9911773

Browse files
C00ldudeNoonanC00ldudeNoonan
authored
Merge pull request #26 from C00ldudeNoonan/sync-to-bq
Sync to bq
2 parents 59cb6e2 + ae75c8f commit 9911773

15 files changed

Lines changed: 768 additions & 155 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ tmpd_*
1313
.tmp_dagster_*
1414
*.csv
1515
*.log
16+
*.json
1617

1718
# Python cache files
1819
__pycache__/

‎macro_agents/.env.example‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,7 @@ PYTHONLEGACYWINDOWSSTDIO=
1111
MARKETSTACK_API_KEY=
1212
OPENAI_API_KEY=
1313
MODEL_NAME=gpt-4-turbo-preview
14+
GOOGLE_APPLICATION_CREDENTIALS=/path/to/your/service-account-key.json
15+
BIGQUERY_PROJECT_ID=
16+
BIGQUERY_LOCATION=
17+
BIGQUERY_DATASET=

‎macro_agents/pyproject.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ dependencies = [
1212
"dbt-core",
1313
"google-api-core",
1414
"google-auth",
15+
"google-cloud-bigquery",
1516
"dbt-duckdb",
1617
"dagster-shared>=1.11.4",
1718
"dagster-pipes>=1.11.4",
@@ -25,6 +26,7 @@ dependencies = [
2526
"dspy>=2.6.27",
2627
"sqlfluff>=3.4.2",
2728
"sqlfluff-templater-dbt>=3.4.2",
29+
"dagster-sling",
2830
]
2931

3032
[project.optional-dependencies]

‎macro_agents/src/macro_agents/definitions.py‎

Lines changed: 102 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -5,83 +5,130 @@
55
from macro_agents.defs.resources.fred import fred_resource
66
from macro_agents.defs.resources.market_stack import marketstack_resource
77
from macro_agents.defs.transformation.dbt import dbt_cli_resource
8-
from macro_agents.defs.agents.analysis_agent import EconomicAnalyzer
9-
from macro_agents.defs.agents.economic_cycle_analyzer import EconomicCycleAnalyzer
8+
from macro_agents.defs.agents.analysis_agent import (
9+
EconomicAnalyzer,
10+
sector_inflation_analysis,
11+
sector_inflation_specific_analysis,
12+
)
13+
from macro_agents.defs.agents.economic_cycle_analyzer import (
14+
EconomicCycleAnalyzer,
15+
economic_cycle_analysis,
16+
)
1017
from macro_agents.defs.agents.enhanced_economic_cycle_analyzer import (
1118
EnhancedEconomicCycleAnalyzer,
1219
)
13-
from macro_agents.defs.agents.asset_allocation_analyzer import AssetAllocationAnalyzer
20+
from macro_agents.defs.agents.asset_allocation_analyzer import (
21+
AssetAllocationAnalyzer,
22+
asset_allocation_recommendations,
23+
custom_asset_allocation,
24+
)
25+
from macro_agents.defs.agents.economic_dashboard import economic_monitoring_alerts
1426
from macro_agents.defs.agents.dspy_evaluation import FinancialEvaluator, PromptOptimizer
1527
from macro_agents.defs.agents.backtesting import BacktestingEngine
1628
from macro_agents.defs.agents.model_improvement_pipeline import ModelImprovementPipeline
1729
from macro_agents.defs.agents.backtesting_visualization import BacktestingVisualizer
1830
from macro_agents.defs.schedules import schedules, sensors, jobs
31+
from macro_agents.defs.replication.sling import replication_assets, sling_resource
32+
from macro_agents.defs.transformation.dbt import full_dbt_assets
33+
from macro_agents.defs.transformation.financial_condition_index import (
34+
financial_conditions_index,
35+
fci_weights_config,
36+
)
37+
from macro_agents.defs.ingestion.market_stack import (
38+
us_sector_etfs_raw,
39+
currency_etfs_raw,
40+
major_indices_raw,
41+
fixed_income_etfs_raw,
42+
global_markets_raw,
43+
energy_commodities_raw,
44+
input_commodities_raw,
45+
agriculture_commodities_raw,
46+
)
47+
from macro_agents.defs.ingestion.fred import fred_raw
48+
from macro_agents.defs.ingestion.treasury_yields import treasury_yields_raw
49+
from macro_agents.defs.ingestion.bls import housing_inventory_raw, housing_pulse_raw
1950

2051

21-
# Find the project root by looking for pyproject.toml
22-
# In local dev: use __file__ relative path
23-
# In Dagster Cloud: use working directory (where pyproject.toml is bundled)
2452
def find_project_root():
25-
# Try working directory first (for Dagster Cloud deployments)
2653
cwd = Path.cwd()
2754
if (cwd / "pyproject.toml").exists():
2855
return cwd
2956

30-
# Try __file__ relative path (for local dev)
3157
file_root = Path(__file__).parent.parent
3258
if (file_root / "pyproject.toml").exists():
3359
return file_root
3460

35-
# Fallback: use __file__ relative path anyway
3661
return file_root
3762

3863

3964
project_root = find_project_root()
4065

41-
defs = dg.Definitions.merge(
42-
dg.load_from_defs_folder(project_root=project_root),
43-
dg.Definitions(
44-
resources={
45-
"md": motherduck_resource,
46-
"fred": fred_resource,
47-
"marketstack": marketstack_resource,
48-
"dbt": dbt_cli_resource,
49-
"analyzer": EconomicAnalyzer(
50-
model_name=dg.EnvVar("MODEL_NAME"),
51-
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
52-
),
53-
"cycle_analyzer": EconomicCycleAnalyzer(
54-
model_name=dg.EnvVar("MODEL_NAME"),
55-
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
56-
),
57-
"enhanced_cycle_analyzer": EnhancedEconomicCycleAnalyzer(
58-
model_name=dg.EnvVar("MODEL_NAME"),
59-
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
60-
),
61-
"allocation_analyzer": AssetAllocationAnalyzer(
62-
model_name=dg.EnvVar("MODEL_NAME"),
63-
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
64-
),
65-
"evaluator": FinancialEvaluator(
66-
model_name=dg.EnvVar("MODEL_NAME"),
67-
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
68-
),
69-
"optimizer": PromptOptimizer(
70-
model_name=dg.EnvVar("MODEL_NAME"),
71-
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
72-
),
73-
"backtesting_engine": BacktestingEngine(
74-
model_name=dg.EnvVar("MODEL_NAME"),
75-
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
76-
),
77-
"model_pipeline": ModelImprovementPipeline(
78-
model_name=dg.EnvVar("MODEL_NAME"),
79-
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
80-
),
81-
"visualizer": BacktestingVisualizer(),
82-
},
83-
schedules=schedules,
84-
sensors=sensors,
85-
jobs=list(jobs.values()),
86-
),
66+
defs = dg.Definitions(
67+
assets=[
68+
replication_assets,
69+
full_dbt_assets,
70+
us_sector_etfs_raw,
71+
currency_etfs_raw,
72+
major_indices_raw,
73+
fixed_income_etfs_raw,
74+
global_markets_raw,
75+
energy_commodities_raw,
76+
input_commodities_raw,
77+
agriculture_commodities_raw,
78+
fred_raw,
79+
treasury_yields_raw,
80+
housing_inventory_raw,
81+
housing_pulse_raw,
82+
financial_conditions_index,
83+
fci_weights_config,
84+
sector_inflation_analysis,
85+
sector_inflation_specific_analysis,
86+
economic_cycle_analysis,
87+
asset_allocation_recommendations,
88+
custom_asset_allocation,
89+
economic_monitoring_alerts,
90+
],
91+
resources={
92+
"md": motherduck_resource,
93+
"fred": fred_resource,
94+
"marketstack": marketstack_resource,
95+
"dbt": dbt_cli_resource,
96+
"sling": sling_resource,
97+
"analyzer": EconomicAnalyzer(
98+
model_name=dg.EnvVar("MODEL_NAME"),
99+
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
100+
),
101+
"cycle_analyzer": EconomicCycleAnalyzer(
102+
model_name=dg.EnvVar("MODEL_NAME"),
103+
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
104+
),
105+
"enhanced_cycle_analyzer": EnhancedEconomicCycleAnalyzer(
106+
model_name=dg.EnvVar("MODEL_NAME"),
107+
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
108+
),
109+
"allocation_analyzer": AssetAllocationAnalyzer(
110+
model_name=dg.EnvVar("MODEL_NAME"),
111+
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
112+
),
113+
"evaluator": FinancialEvaluator(
114+
model_name=dg.EnvVar("MODEL_NAME"),
115+
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
116+
),
117+
"optimizer": PromptOptimizer(
118+
model_name=dg.EnvVar("MODEL_NAME"),
119+
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
120+
),
121+
"backtesting_engine": BacktestingEngine(
122+
model_name=dg.EnvVar("MODEL_NAME"),
123+
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
124+
),
125+
"model_pipeline": ModelImprovementPipeline(
126+
model_name=dg.EnvVar("MODEL_NAME"),
127+
openai_api_key=dg.EnvVar("OPENAI_API_KEY"),
128+
),
129+
"visualizer": BacktestingVisualizer(),
130+
},
131+
schedules=schedules,
132+
sensors=sensors,
133+
jobs=list(jobs.values()),
87134
)

‎macro_agents/src/macro_agents/defs/ingestion/bls.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
kinds={"polars", "duckdb"},
1717
partitions_def=year_partition,
1818
description="Raw data from BLS API for housing inventory",
19-
automation_condition=dg.AutomationCondition.on_cron("0 0 * * 1"),
2019
)
2120
def housing_inventory_raw(
2221
context: dg.AssetExecutionContext, md: MotherDuckResource
@@ -41,6 +40,7 @@ def housing_inventory_raw(
4140
"year": year,
4241
"num_records": len(df),
4342
"columns": df.columns,
43+
"first_10_rows": df.head(10).to_dicts(),
4444
}
4545
)
4646

@@ -49,7 +49,6 @@ def housing_inventory_raw(
4949
group_name="ingestion",
5050
kinds={"polars", "duckdb"},
5151
description="Raw data from BLS API for housing pulse",
52-
automation_condition=dg.AutomationCondition.on_cron("0 0 * * 1"),
5352
)
5453
def housing_pulse_raw(
5554
context: dg.AssetExecutionContext, md: MotherDuckResource
@@ -78,5 +77,6 @@ def housing_pulse_raw(
7877
return dg.MaterializeResult(
7978
metadata={
8079
"num_records": len(main_df),
80+
"first_10_rows": main_df.head(10).to_dicts(),
8181
}
8282
)

‎macro_agents/src/macro_agents/defs/ingestion/fred.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@
4949
group_name="ingestion",
5050
kinds={"polars", "duckdb"},
5151
partitions_def=fred_series_partition,
52-
automation_condition=dg.AutomationCondition.on_cron("0 0 * * 1"),
5352
description="Raw data from FRED API",
5453
)
5554
def fred_raw(
@@ -66,5 +65,6 @@ def fred_raw(
6665
"num_records": len(data),
6766
"max_date": str(data["date"].max()),
6867
"min_date": str(data["date"].min()),
68+
"first_10_rows": data.head(10).to_dicts(),
6969
}
7070
)

0 commit comments

Comments
 (0)