Skip to content

Commit a4a679a

Browse files
backtest metadata fixes
1 parent 056825e commit a4a679a

9 files changed

Lines changed: 917 additions & 339 deletions

macro_agents/src/macro_agents/defs/agents/asset_class_relationship_analyzer.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,9 @@ def extract_relationship_summary(analysis_content: str) -> Dict[str, Any]:
127127
"""Extract key insights from asset class relationship analysis for metadata."""
128128
summary = {}
129129

130+
if not analysis_content:
131+
return summary
132+
130133
correlation_matches = re.findall(
131134
r"(?:strongest|highest|strong).*?correlation[:\s]+([^.\n]+)",
132135
analysis_content,

macro_agents/src/macro_agents/defs/agents/backtest_asset_class_relationship_analyzer.py

Lines changed: 225 additions & 111 deletions
Large diffs are not rendered by default.

macro_agents/src/macro_agents/defs/agents/backtest_economy_state_analyzer.py

Lines changed: 237 additions & 123 deletions
Large diffs are not rendered by default.

macro_agents/src/macro_agents/defs/agents/backtest_investment_recommendations.py

Lines changed: 210 additions & 94 deletions
Large diffs are not rendered by default.

macro_agents/src/macro_agents/defs/agents/backtest_optimizer.py

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,15 @@ def optimize_dspy_modules(
340340
model_name_override=config.model_name,
341341
)
342342

343+
from macro_agents.defs.agents.economy_state_analyzer import (
344+
_get_token_usage,
345+
_calculate_cost,
346+
)
347+
348+
initial_history_length = (
349+
len(economic_analysis._lm.history) if hasattr(economic_analysis, "_lm") else 0
350+
)
351+
343352
results = {}
344353

345354
personalities_to_test = []
@@ -704,7 +713,26 @@ def optimize_dspy_modules(
704713
"error": str(e),
705714
}
706715

707-
return dg.MaterializeResult(metadata={"optimization_results": results})
716+
token_usage = _get_token_usage(economic_analysis, initial_history_length, context)
717+
718+
if "total_cost_usd" not in token_usage or token_usage.get("total_cost_usd", 0) == 0:
719+
provider = economic_analysis._get_provider()
720+
model_name = economic_analysis._get_model_name()
721+
cost_data = _calculate_cost(
722+
provider=provider,
723+
model_name=model_name,
724+
prompt_tokens=token_usage.get("prompt_tokens", 0),
725+
completion_tokens=token_usage.get("completion_tokens", 0),
726+
)
727+
token_usage.update(cost_data)
728+
729+
return dg.MaterializeResult(
730+
metadata={
731+
"optimization_results": results,
732+
"token_usage": token_usage,
733+
"provider": economic_analysis._get_provider(),
734+
}
735+
)
708736

709737

710738
@dg.asset(

macro_agents/src/macro_agents/defs/agents/backtest_utils.py

Lines changed: 34 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,26 @@ def extract_recommendations(recommendations_content: str) -> List[Dict[str, Any]
3333
confidence_match = re.search(
3434
r"confidence[:\s]+([0-9.]+)", context, re.IGNORECASE
3535
)
36-
confidence = float(confidence_match.group(1)) if confidence_match else None
37-
if confidence and confidence > 1:
38-
confidence = confidence / 100
36+
confidence = None
37+
if confidence_match:
38+
confidence_str = confidence_match.group(1).rstrip(".,;")
39+
try:
40+
confidence = float(confidence_str)
41+
if confidence > 1:
42+
confidence = confidence / 100
43+
except (ValueError, AttributeError):
44+
confidence = None
3945

4046
return_match = re.search(
41-
r"(?:expected|return)[:\s]+([0-9.]+)%?", context, re.IGNORECASE
47+
r"(?:expected|return)[:\s]+([-0-9.]+)%?", context, re.IGNORECASE
4248
)
43-
expected_return = float(return_match.group(1)) if return_match else None
49+
expected_return = None
50+
if return_match:
51+
return_str = return_match.group(1).rstrip(".,;%")
52+
try:
53+
expected_return = float(return_str)
54+
except (ValueError, AttributeError):
55+
expected_return = None
4456

4557
recommendations.append(
4658
{
@@ -66,14 +78,26 @@ def extract_recommendations(recommendations_content: str) -> List[Dict[str, Any]
6678
confidence_match = re.search(
6779
r"confidence[:\s]+([0-9.]+)", context, re.IGNORECASE
6880
)
69-
confidence = float(confidence_match.group(1)) if confidence_match else None
70-
if confidence and confidence > 1:
71-
confidence = confidence / 100
81+
confidence = None
82+
if confidence_match:
83+
confidence_str = confidence_match.group(1).rstrip(".,;")
84+
try:
85+
confidence = float(confidence_str)
86+
if confidence > 1:
87+
confidence = confidence / 100
88+
except (ValueError, AttributeError):
89+
confidence = None
7290

7391
return_match = re.search(
74-
r"(?:expected|return)[:\s]+([0-9.]+)%?", context, re.IGNORECASE
92+
r"(?:expected|return)[:\s]+([-0-9.]+)%?", context, re.IGNORECASE
7593
)
76-
expected_return = float(return_match.group(1)) if return_match else None
94+
expected_return = None
95+
if return_match:
96+
return_str = return_match.group(1).rstrip(".,;%")
97+
try:
98+
expected_return = float(return_str)
99+
except (ValueError, AttributeError):
100+
expected_return = None
77101

78102
recommendations.append(
79103
{

macro_agents/src/macro_agents/defs/agents/economy_state_analyzer.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1434,6 +1434,9 @@ def extract_economy_state_summary(analysis_content: str) -> Dict[str, Any]:
14341434
"""Extract key insights from economy state analysis for metadata."""
14351435
summary = {}
14361436

1437+
if not analysis_content:
1438+
return summary
1439+
14371440
cycle_match = re.search(
14381441
r"(?:Current Economic Cycle Position|Cycle Position|Economic Cycle):\s*([^.\n]+)",
14391442
analysis_content,

macro_agents/src/macro_agents/defs/agents/investment_recommendations.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,11 @@ def extract_recommendations_summary(recommendations_content: str) -> Dict[str, A
146146
"""Extract key insights from investment recommendations for metadata."""
147147
summary = {}
148148

149+
if not recommendations_content:
150+
summary["total_overweight_count"] = 0
151+
summary["total_underweight_count"] = 0
152+
return summary
153+
149154
outlook_match = re.search(
150155
r"(?:market outlook|outlook)[:\s]+(bullish|bearish|neutral|positive|negative)",
151156
recommendations_content,

macro_agents/tests/test_dspy_modules.py

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -473,3 +473,174 @@ def test_asset_class_relationship_module_with_mock_lm(self):
473473

474474
assert result.relationship_analysis == "Strong correlation between tech and GDP"
475475
assert module.analyze_relationships.called
476+
477+
478+
class TestExtractFunctionsWithEmptyContent:
479+
"""Test cases for extract functions handling None/empty content."""
480+
481+
def test_extract_economy_state_summary_with_none(self):
482+
"""Test extract_economy_state_summary handles None content."""
483+
from macro_agents.defs.agents.economy_state_analyzer import (
484+
extract_economy_state_summary,
485+
)
486+
487+
result = extract_economy_state_summary(None)
488+
assert isinstance(result, dict)
489+
assert len(result) == 0
490+
491+
def test_extract_economy_state_summary_with_empty_string(self):
492+
"""Test extract_economy_state_summary handles empty string."""
493+
from macro_agents.defs.agents.economy_state_analyzer import (
494+
extract_economy_state_summary,
495+
)
496+
497+
result = extract_economy_state_summary("")
498+
assert isinstance(result, dict)
499+
assert len(result) == 0
500+
501+
def test_extract_economy_state_summary_with_valid_content(self):
502+
"""Test extract_economy_state_summary with valid content."""
503+
from macro_agents.defs.agents.economy_state_analyzer import (
504+
extract_economy_state_summary,
505+
)
506+
507+
content = """
508+
Current Economic Cycle Position: Expansion
509+
Confidence: 0.75
510+
Risk Factors: Inflation concerns, geopolitical tensions
511+
"""
512+
result = extract_economy_state_summary(content)
513+
assert isinstance(result, dict)
514+
assert "economic_cycle_position" in result
515+
assert result["economic_cycle_position"] == "Expansion"
516+
assert "confidence_level" in result
517+
assert result["confidence_level"] == 0.75
518+
519+
def test_extract_relationship_summary_with_none(self):
520+
"""Test extract_relationship_summary handles None content."""
521+
from macro_agents.defs.agents.asset_class_relationship_analyzer import (
522+
extract_relationship_summary,
523+
)
524+
525+
result = extract_relationship_summary(None)
526+
assert isinstance(result, dict)
527+
assert len(result) == 0
528+
529+
def test_extract_relationship_summary_with_empty_string(self):
530+
"""Test extract_relationship_summary handles empty string."""
531+
from macro_agents.defs.agents.asset_class_relationship_analyzer import (
532+
extract_relationship_summary,
533+
)
534+
535+
result = extract_relationship_summary("")
536+
assert isinstance(result, dict)
537+
assert len(result) == 0
538+
539+
def test_extract_recommendations_summary_with_none(self):
540+
"""Test extract_recommendations_summary handles None content."""
541+
from macro_agents.defs.agents.investment_recommendations import (
542+
extract_recommendations_summary,
543+
)
544+
545+
result = extract_recommendations_summary(None)
546+
assert isinstance(result, dict)
547+
assert "total_overweight_count" in result
548+
assert "total_underweight_count" in result
549+
assert result["total_overweight_count"] == 0
550+
assert result["total_underweight_count"] == 0
551+
552+
def test_extract_recommendations_summary_with_empty_string(self):
553+
"""Test extract_recommendations_summary handles empty string."""
554+
from macro_agents.defs.agents.investment_recommendations import (
555+
extract_recommendations_summary,
556+
)
557+
558+
result = extract_recommendations_summary("")
559+
assert isinstance(result, dict)
560+
assert "total_overweight_count" in result
561+
assert "total_underweight_count" in result
562+
assert result["total_overweight_count"] == 0
563+
assert result["total_underweight_count"] == 0
564+
565+
566+
class TestExtractRecommendationsFloatParsing:
567+
"""Test cases for extract_recommendations handling trailing punctuation in floats."""
568+
569+
def test_extract_recommendations_with_trailing_period(self):
570+
"""Test extract_recommendations handles confidence with trailing period."""
571+
from macro_agents.defs.agents.backtest_utils import extract_recommendations
572+
573+
content = "OVERWEIGHT XLK with confidence 0.6. Expected return 5.2%."
574+
recommendations = extract_recommendations(content)
575+
576+
assert len(recommendations) > 0
577+
xlk_rec = next((r for r in recommendations if r["symbol"] == "XLK"), None)
578+
assert xlk_rec is not None
579+
assert xlk_rec["direction"] == "OVERWEIGHT"
580+
assert xlk_rec["confidence"] == 0.6
581+
assert xlk_rec["expected_return"] == 5.2
582+
583+
def test_extract_recommendations_with_trailing_comma(self):
584+
"""Test extract_recommendations handles confidence with trailing comma."""
585+
from macro_agents.defs.agents.backtest_utils import extract_recommendations
586+
587+
content = "OVERWEIGHT SPY with confidence 0.75, expected return 8.3%"
588+
recommendations = extract_recommendations(content)
589+
590+
assert len(recommendations) > 0
591+
spy_rec = next((r for r in recommendations if r["symbol"] == "SPY"), None)
592+
assert spy_rec is not None
593+
assert spy_rec["confidence"] == 0.75
594+
assert spy_rec["expected_return"] == 8.3
595+
596+
def test_extract_recommendations_with_trailing_semicolon(self):
597+
"""Test extract_recommendations handles confidence with trailing semicolon."""
598+
from macro_agents.defs.agents.backtest_utils import extract_recommendations
599+
600+
content = "UNDERWEIGHT XLE with confidence 0.4; expected return -2.1%"
601+
recommendations = extract_recommendations(content)
602+
603+
assert len(recommendations) > 0
604+
xle_rec = next((r for r in recommendations if r["symbol"] == "XLE"), None)
605+
assert xle_rec is not None
606+
assert xle_rec["direction"] == "UNDERWEIGHT"
607+
assert xle_rec["confidence"] == 0.4
608+
assert xle_rec["expected_return"] == -2.1
609+
610+
def test_extract_recommendations_with_percent_sign(self):
611+
"""Test extract_recommendations handles return values with percent sign."""
612+
from macro_agents.defs.agents.backtest_utils import extract_recommendations
613+
614+
content = "OVERWEIGHT QQQ confidence 0.8 expected return 12.5%"
615+
recommendations = extract_recommendations(content)
616+
617+
assert len(recommendations) > 0
618+
qqq_rec = next((r for r in recommendations if r["symbol"] == "QQQ"), None)
619+
assert qqq_rec is not None
620+
assert qqq_rec["expected_return"] == 12.5
621+
622+
def test_extract_recommendations_with_invalid_float_gracefully(self):
623+
"""Test extract_recommendations handles invalid float strings gracefully."""
624+
from macro_agents.defs.agents.backtest_utils import extract_recommendations
625+
626+
content = "OVERWEIGHT XLK with confidence invalid. Expected return also.invalid"
627+
recommendations = extract_recommendations(content)
628+
629+
assert len(recommendations) > 0
630+
xlk_rec = next((r for r in recommendations if r["symbol"] == "XLK"), None)
631+
assert xlk_rec is not None
632+
assert xlk_rec["confidence"] is None
633+
assert xlk_rec["expected_return"] is None
634+
635+
def test_extract_recommendations_with_multiple_trailing_punctuation(self):
636+
"""Test extract_recommendations handles multiple trailing punctuation."""
637+
from macro_agents.defs.agents.backtest_utils import extract_recommendations
638+
639+
content = "OVERWEIGHT DIA confidence 0.9., expected return 6.7%."
640+
recommendations = extract_recommendations(content)
641+
642+
assert len(recommendations) > 0
643+
dia_rec = next((r for r in recommendations if r["symbol"] == "DIA"), None)
644+
assert dia_rec is not None
645+
assert dia_rec["confidence"] == 0.9
646+
assert dia_rec["expected_return"] == 6.7

0 commit comments

Comments
 (0)