fix: patch vulnerable python dependencies (#104)

* fix: patch vulnerable python dependencies

* chore: refresh python dependency lockfile

* fix: satisfy updated ty checks

* style: apply ruff formatting

Author: C00ldudeNoonan

macro_agents/pyproject.toml

@@ -47,20 +47,26 @@ dependencies = [
     "vadersentiment>=3.3.2",
     "sqlglot>=28.1.0,<29.0.0",
     "metaxy[dagster]>=0.1.10,<0.2.0",
+    "aiohttp>=3.14.1",
+    "cryptography>=48.0.1",
+    "msgpack>=1.2.1",
+    "pydantic-settings>=2.14.2",
+    "pyjwt>=2.13.0",
+    "starlette>=1.3.1",
 ]
 
 [project.optional-dependencies]
 dev = [
     "dagster-webserver",
-    "pytest",
+    "pytest>=9.0.3,<9.1.0",
     "pytest-cov",
     "pytest-xdist",
     "mypy",
     "ty",
     "duckdb>=1.0.0,<2.0.0",
     "ruff>=0.12.7,<1.0.0",  # Move ruff here
-    "sqlfluff>=3.4.2,<4.0.0",  # Move sqlfluff here
-    "sqlfluff-templater-dbt>=3.4.2,<4.0.0",  # Move here
+    "sqlfluff>=4.2.0,<5.0.0",  # Move sqlfluff here
+    "sqlfluff-templater-dbt>=4.2.0,<5.0.0",  # Move here
 ]
 
 [build-system]

macro_agents/src/macro_agents/defs/analysis/data_points/interesting_data_points_assets.py

@@ -327,9 +327,10 @@ def detect_interesting_data_points_weekly(
 
     # 10. Return metadata
     finding_types = findings_df["finding_type"].unique().to_list()
-    top_finding = (
+    top_finding_value = (
         analyzed_findings[0]["significance_narrative"] if analyzed_findings else None
     )
+    top_finding = top_finding_value if isinstance(top_finding_value, str) else None
 
     context.log.info(
         f"Successfully completed weekly analysis with {len(analyzed_findings)} findings"

macro_agents/src/macro_agents/defs/backtesting/backtest_asset_class_relationship_analyzer.py

@@ -1,3 +1,4 @@
+import json
 from datetime import datetime
 
 import dagster as dg
@@ -22,6 +23,14 @@
 from macro_agents.defs.resources.bigquery_warehouse import BigQueryWarehouseResource
 
 
+def _content_as_text(value: object) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, default=str)
+
+
 def get_latest_backtest_economy_state_analysis(
     md_resource: BigQueryWarehouseResource,
     backtest_date: str,
@@ -454,7 +463,7 @@ def backtest_analyze_asset_class_relationships(
     )
 
     first_result = all_results[0]
-    analysis_content = first_result.get("analysis_content") or ""
+    analysis_content = _content_as_text(first_result.get("analysis_content"))
     analysis_summary = (
         extract_relationship_summary(analysis_content) if analysis_content else {}
     )

macro_agents/src/macro_agents/defs/backtesting/backtest_economy_state_analyzer.py

@@ -1,3 +1,4 @@
+import json
 from datetime import datetime
 
 import dagster as dg
@@ -15,6 +16,14 @@
 from macro_agents.defs.resources.bigquery_warehouse import BigQueryWarehouseResource
 
 
+def _content_as_text(value: object) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, default=str)
+
+
 class BacktestConfig(dg.Config):
     """Configuration for backtesting - can be specified at runtime."""
 
@@ -522,7 +531,7 @@ def backtest_analyze_economy_state(
     )
 
     first_result = all_results[0]
-    analysis_content = first_result.get("analysis_content") or ""
+    analysis_content = _content_as_text(first_result.get("analysis_content"))
     analysis_summary = (
         extract_economy_state_summary(analysis_content) if analysis_content else {}
     )

macro_agents/src/macro_agents/defs/backtesting/backtest_evaluator.py

@@ -102,6 +102,32 @@ def recommendation_accuracy_metric(example, prediction, trace=None):
     return score
 
 
+def _period_result(evaluation_result: dict[str, object], period: str) -> dict | None:
+    period_results = evaluation_result.get("period_results", {})
+    if not isinstance(period_results, dict):
+        return None
+
+    period_result = period_results.get(period)
+    if not isinstance(period_result, dict):
+        return None
+
+    return period_result
+
+
+def _period_outperformance(
+    evaluation_result: dict[str, object], period: str
+) -> float | None:
+    period_result = _period_result(evaluation_result, period)
+    if period_result is None:
+        return None
+
+    value = period_result.get("outperformance")
+    if not isinstance(value, (int, float)):
+        return None
+
+    return float(value)
+
+
 @dg.asset(
     kinds={"dspy", "duckdb"},
     group_name="backtesting",
@@ -228,7 +254,7 @@ def evaluate_backtest_recommendations(
             f"Running evaluation on {len(examples)} examples for {backtest_date}..."
         )
 
-        evaluation_results = []
+        evaluation_results: list[dict[str, object]] = []
         for example in examples:
             prediction = module(
                 recommendation=example.recommendation, outcomes=example.outcomes
@@ -238,7 +264,8 @@ def evaluate_backtest_recommendations(
 
             if hasattr(example, "_metric_details") and example._metric_details:
                 detail = list(example._metric_details.values())[0]
-                evaluation_results.append(detail)
+                if isinstance(detail, dict):
+                    evaluation_results.append(detail)
             else:
                 recommendation = example.recommendation
                 evaluation_results.append(
@@ -260,22 +287,23 @@ def evaluate_backtest_recommendations(
         misses_6m = 0
 
         for r in evaluation_results:
-            period_results = r.get("period_results", {})
-
-            if "1m" in period_results:
-                if period_results["1m"].get("is_hit", False):
+            period_result_1m = _period_result(r, "1m")
+            if period_result_1m is not None:
+                if period_result_1m.get("is_hit", False):
                     hits_1m += 1
                 else:
                     misses_1m += 1
 
-            if "3m" in period_results:
-                if period_results["3m"].get("is_hit", False):
+            period_result_3m = _period_result(r, "3m")
+            if period_result_3m is not None:
+                if period_result_3m.get("is_hit", False):
                     hits_3m += 1
                 else:
                     misses_3m += 1
 
-            if "6m" in period_results:
-                if period_results["6m"].get("is_hit", False):
+            period_result_6m = _period_result(r, "6m")
+            if period_result_6m is not None:
+                if period_result_6m.get("is_hit", False):
                     hits_6m += 1
                 else:
                     misses_6m += 1
@@ -289,10 +317,9 @@ def evaluate_backtest_recommendations(
         accuracy_6m = hits_6m / total_6m if total_6m > 0 else 0.0
 
         outperformance_1m = [
-            r.get("period_results", {}).get("1m", {}).get("outperformance", 0.0)
+            value
             for r in evaluation_results
-            if r.get("period_results", {}).get("1m", {}).get("outperformance")
-            is not None
+            if (value := _period_outperformance(r, "1m")) is not None
         ]
         avg_outperformance_1m = (
             sum(outperformance_1m) / len(outperformance_1m)
@@ -301,10 +328,9 @@ def evaluate_backtest_recommendations(
         )
 
         outperformance_3m = [
-            r.get("period_results", {}).get("3m", {}).get("outperformance", 0.0)
+            value
             for r in evaluation_results
-            if r.get("period_results", {}).get("3m", {}).get("outperformance")
-            is not None
+            if (value := _period_outperformance(r, "3m")) is not None
         ]
         avg_outperformance_3m = (
             sum(outperformance_3m) / len(outperformance_3m)
@@ -313,10 +339,9 @@ def evaluate_backtest_recommendations(
         )
 
         outperformance_6m = [
-            r.get("period_results", {}).get("6m", {}).get("outperformance", 0.0)
+            value
             for r in evaluation_results
-            if r.get("period_results", {}).get("6m", {}).get("outperformance")
-            is not None
+            if (value := _period_outperformance(r, "6m")) is not None
         ]
         avg_outperformance_6m = (
             sum(outperformance_6m) / len(outperformance_6m)

macro_agents/src/macro_agents/defs/backtesting/backtest_investment_recommendations.py

@@ -1,3 +1,4 @@
+import json
 from datetime import datetime
 
 import dagster as dg
@@ -23,6 +24,14 @@
 from macro_agents.defs.resources.bigquery_warehouse import BigQueryWarehouseResource
 
 
+def _content_as_text(value: object) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, default=str)
+
+
 def get_latest_backtest_economy_state_analysis(
     md_resource: BigQueryWarehouseResource,
     backtest_date: str,
@@ -414,7 +423,9 @@ def backtest_generate_investment_recommendations(
     )
 
     first_result = all_results[0]
-    recommendations_content = first_result.get("recommendations_content") or ""
+    recommendations_content = _content_as_text(
+        first_result.get("recommendations_content")
+    )
     recommendations_summary = (
         extract_recommendations_summary(recommendations_content)
         if recommendations_content

macro_agents/src/macro_agents/defs/utils/sec_text_extractor.py

@@ -265,7 +265,9 @@ def extract_sections(
         # TOC entries appear first in the document, actual section content appears later
         last_occurrence: dict[str, dict] = {}
         for pos in section_positions:
-            last_occurrence[pos["key"]] = pos
+            section_key = pos["key"]
+            if isinstance(section_key, str):
+                last_occurrence[section_key] = pos
 
         # Convert back to list and sort by position
         unique_positions = sorted(last_occurrence.values(), key=lambda x: x["start"])