fix(regime): harden history fetching (#679)

Retry empty regime history responses, require fresh completed-session data before using API or cached bars, and fail closed when the cache cannot validate the lookback window.

Add regression coverage for empty/stale API history, cache recovery, unreadable cache failures, and ignoring extra partial-session bars.

Author: brndnmtthws

tests/test_db.py

@@ -157,6 +157,46 @@ def test_record_historical_bars_upserts_and_parses_dates(tmp_path) -> None:
     assert volume == 20
 
 
+def test_get_historical_bars_filters_by_symbol_timeframe_and_time(tmp_path) -> None:
+    db_path = tmp_path / "state.db"
+    data_store = DataStore(
+        f"sqlite:///{db_path}",
+        str(tmp_path / "thetagang.toml"),
+        dry_run=False,
+        config_text="test",
+    )
+
+    data_store.record_historical_bars(
+        "AAA",
+        "1 day",
+        [
+            SimpleNamespace(date="20240104", close=1.0),
+            SimpleNamespace(date="20240105", close=2.0),
+        ],
+    )
+    data_store.record_historical_bars(
+        "AAA",
+        "1 hour",
+        [SimpleNamespace(date="20240105 12:00:00", close=99.0)],
+    )
+    data_store.record_historical_bars(
+        "BBB",
+        "1 day",
+        [SimpleNamespace(date="20240105", close=100.0)],
+    )
+
+    bars = data_store.get_historical_bars(
+        "AAA",
+        "1 day",
+        datetime(2024, 1, 5, 0, 0, 0),
+        datetime(2024, 1, 5, 23, 59, 59),
+    )
+
+    assert len(bars) == 1
+    assert bars[0].date == datetime(2024, 1, 5)
+    assert bars[0].close == 2.0
+
+
 def test_record_executions_parses_string_times(tmp_path) -> None:
     db_path = tmp_path / "state.db"
     data_store = DataStore(

tests/test_regime_rebalance.py

@@ -1,5 +1,5 @@
 import math
-from datetime import datetime, timedelta
+from datetime import date, datetime, timedelta
 from types import SimpleNamespace
 from typing import cast
 
@@ -16,6 +16,13 @@
     normalize_config,
 )
 from thetagang.portfolio_manager import PortfolioManager
+from thetagang.strategies.regime_engine import (
+    REGIME_HISTORY_MAX_ATTEMPTS,
+    REGIME_HISTORY_TIMEFRAME,
+)
+
+REGIME_HISTORY_START = datetime(2024, 1, 2)
+REGIME_SYMBOLS = ("AAA", "BBB")
 
 
 @pytest.fixture
@@ -157,11 +164,8 @@ def now(cls, tz=None):
 
 
 def _mock_regime_history(portfolio_manager, mocker, closes):
-    start_date = datetime(2024, 1, 2)
-    bars = [
-        SimpleNamespace(date=start_date + timedelta(days=offset), close=close)
-        for offset, close in enumerate(closes)
-    ]
+    bars = _regime_bars(closes)
+    _mock_required_regime_history_dates(portfolio_manager, mocker)
 
     async def _get_history(*_args, **_kwargs):
         return bars
@@ -172,15 +176,18 @@ async def _get_history(*_args, **_kwargs):
     return bars
 
 
+def _regime_bars(closes, start_date: datetime = REGIME_HISTORY_START):
+    return [
+        SimpleNamespace(date=start_date + timedelta(days=offset), close=close)
+        for offset, close in enumerate(closes)
+    ]
+
+
 def _mock_regime_histories(portfolio_manager, mocker, closes_by_symbol):
-    start_date = datetime(2024, 1, 2)
     bars_by_symbol = {
-        symbol: [
-            SimpleNamespace(date=start_date + timedelta(days=offset), close=close)
-            for offset, close in enumerate(closes)
-        ]
-        for symbol, closes in closes_by_symbol.items()
+        symbol: _regime_bars(closes) for symbol, closes in closes_by_symbol.items()
     }
+    _mock_required_regime_history_dates(portfolio_manager, mocker)
 
     async def _get_history(contract, *_args, **_kwargs):
         return bars_by_symbol[contract.symbol]
@@ -216,6 +223,14 @@ async def _get_ticker(symbol, _primary_exchange):
     )
 
 
+def _mock_regime_broker(portfolio_manager, mocker, **history_mock_kwargs) -> None:
+    _mock_regime_tickers(portfolio_manager, mocker)
+    portfolio_manager.ibkr.request_historical_data = mocker.AsyncMock(
+        **history_mock_kwargs
+    )
+    portfolio_manager.ibkr.request_executions = mocker.AsyncMock(return_value=[])
+
+
 def _stock_position(symbol: str, position: int, market_value: float | None = None):
     return SimpleNamespace(
         contract=Stock(symbol, "SMART", "USD"),
@@ -224,6 +239,63 @@ def _stock_position(symbol: str, position: int, market_value: float | None = Non
     )
 
 
+def _regime_account_summary(value: str = "400"):
+    return {"NetLiquidation": SimpleNamespace(value=value)}
+
+
+def _regime_stock_positions(aaa: int = 3, bbb: int = 1):
+    return {
+        "AAA": [_stock_position("AAA", aaa)],
+        "BBB": [_stock_position("BBB", bbb)],
+    }
+
+
+def _expected_regime_history_fetches(symbols=REGIME_SYMBOLS) -> int:
+    return len(symbols) * REGIME_HISTORY_MAX_ATTEMPTS
+
+
+def _disable_regime_history_retry_delay(monkeypatch) -> None:
+    monkeypatch.setattr(
+        "thetagang.strategies.regime_engine.REGIME_HISTORY_RETRY_DELAY_SECONDS",
+        0.0,
+    )
+
+
+def _required_regime_history_dates(required_points: int) -> list[date]:
+    return [
+        (REGIME_HISTORY_START + timedelta(days=offset)).date()
+        for offset in range(required_points)
+    ]
+
+
+def _set_required_regime_history_dates(
+    portfolio_manager, monkeypatch, required_points: int
+) -> list[date]:
+    required_dates = _required_regime_history_dates(required_points)
+    monkeypatch.setattr(
+        portfolio_manager.regime_engine,
+        "_get_required_history_dates",
+        lambda _required_points: required_dates,
+    )
+    return required_dates
+
+
+def _mock_required_regime_history_dates(portfolio_manager, mocker) -> None:
+    mocker.patch.object(
+        portfolio_manager.regime_engine,
+        "_get_required_history_dates",
+        side_effect=_required_regime_history_dates,
+    )
+
+
+def _seed_regime_history_cache(portfolio_manager, closes, symbols=REGIME_SYMBOLS):
+    cache_bars = _regime_bars(closes)
+    for symbol in symbols:
+        portfolio_manager.data_store.record_historical_bars(
+            symbol, REGIME_HISTORY_TIMEFRAME, cache_bars
+        )
+
+
 def _option_position(
     symbol: str,
     position: int,
@@ -347,6 +419,172 @@ async def test_regime_rebalance_generates_orders(portfolio_manager, mocker):
     assert orders == [("AAA", "NYSE", -1), ("BBB", "NYSE", 1)]
 
 
+@pytest.mark.asyncio
+async def test_regime_rebalance_retries_empty_history(
+    portfolio_manager, mocker, monkeypatch
+):
+    _disable_regime_history_retry_delay(monkeypatch)
+    _mock_required_regime_history_dates(portfolio_manager, mocker)
+    account_summary = _regime_account_summary()
+    portfolio_positions = _regime_stock_positions()
+    bars = _regime_bars([100.0, 110.0, 100.0, 110.0])
+    attempts_by_symbol = {"AAA": 0, "BBB": 0}
+
+    async def _get_history(contract, *_args, **_kwargs):
+        attempts_by_symbol[contract.symbol] += 1
+        if attempts_by_symbol[contract.symbol] == 1:
+            return []
+        return bars
+
+    _mock_regime_broker(portfolio_manager, mocker, side_effect=_get_history)
+
+    _, orders = await portfolio_manager.check_regime_rebalance_positions(
+        account_summary, portfolio_positions
+    )
+
+    assert attempts_by_symbol == {"AAA": 2, "BBB": 2}
+    assert orders == [("AAA", "NYSE", -1), ("BBB", "NYSE", 1)]
+
+
+@pytest.mark.asyncio
+async def test_regime_rebalance_uses_fresh_cached_history_when_api_empty(
+    portfolio_manager_with_db, mocker, monkeypatch
+):
+    _disable_regime_history_retry_delay(monkeypatch)
+    _set_required_regime_history_dates(
+        portfolio_manager_with_db, monkeypatch, required_points=4
+    )
+    _seed_regime_history_cache(portfolio_manager_with_db, [100.0, 110.0, 100.0, 110.0])
+    account_summary = _regime_account_summary()
+    portfolio_positions = _regime_stock_positions()
+
+    _mock_regime_broker(portfolio_manager_with_db, mocker, return_value=[])
+
+    _, orders = await portfolio_manager_with_db.check_regime_rebalance_positions(
+        account_summary, portfolio_positions
+    )
+
+    assert (
+        portfolio_manager_with_db.ibkr.request_historical_data.call_count
+        == _expected_regime_history_fetches()
+    )
+    assert orders == [("AAA", "NYSE", -1), ("BBB", "NYSE", 1)]
+
+
+@pytest.mark.asyncio
+async def test_regime_rebalance_uses_fresh_cache_when_api_history_is_stale(
+    portfolio_manager_with_db, mocker, monkeypatch
+):
+    _disable_regime_history_retry_delay(monkeypatch)
+    _set_required_regime_history_dates(
+        portfolio_manager_with_db, monkeypatch, required_points=4
+    )
+    _seed_regime_history_cache(portfolio_manager_with_db, [100.0, 110.0, 100.0, 110.0])
+    stale_bars = _regime_bars(
+        [100.0, 99.0, 98.0, 97.0],
+        start_date=datetime(2023, 12, 20),
+    )
+    account_summary = _regime_account_summary()
+    portfolio_positions = _regime_stock_positions()
+
+    _mock_regime_broker(portfolio_manager_with_db, mocker, return_value=stale_bars)
+
+    _, orders = await portfolio_manager_with_db.check_regime_rebalance_positions(
+        account_summary, portfolio_positions
+    )
+
+    assert portfolio_manager_with_db.ibkr.request_historical_data.call_count == len(
+        REGIME_SYMBOLS
+    )
+    assert orders == [("AAA", "NYSE", -1), ("BBB", "NYSE", 1)]
+
+
+@pytest.mark.asyncio
+async def test_regime_rebalance_ignores_history_after_required_sessions(
+    portfolio_manager, mocker, monkeypatch
+):
+    _disable_regime_history_retry_delay(monkeypatch)
+    required_dates = _set_required_regime_history_dates(
+        portfolio_manager, monkeypatch, required_points=4
+    )
+    bars_with_extra_partial_session = _regime_bars([100.0, 110.0, 100.0, 110.0, 1.0])
+
+    _mock_regime_broker(
+        portfolio_manager, mocker, return_value=bars_with_extra_partial_session
+    )
+
+    (
+        dates,
+        aligned_closes,
+    ) = await portfolio_manager.regime_engine._get_regime_aligned_closes(
+        list(REGIME_SYMBOLS),
+        lookback_days=3,
+        cooldown_days=0,
+    )
+
+    assert dates == required_dates
+    assert aligned_closes == {
+        "AAA": [100.0, 110.0, 100.0, 110.0],
+        "BBB": [100.0, 110.0, 100.0, 110.0],
+    }
+    assert portfolio_manager.ibkr.request_historical_data.call_count == len(
+        REGIME_SYMBOLS
+    )
+
+
+@pytest.mark.asyncio
+async def test_regime_rebalance_rejects_incomplete_cached_history(
+    portfolio_manager_with_db, mocker, monkeypatch
+):
+    _disable_regime_history_retry_delay(monkeypatch)
+    _set_required_regime_history_dates(
+        portfolio_manager_with_db, monkeypatch, required_points=4
+    )
+    _seed_regime_history_cache(portfolio_manager_with_db, [100.0, 110.0, 100.0])
+    account_summary = _regime_account_summary()
+    portfolio_positions = _regime_stock_positions()
+
+    _mock_regime_broker(portfolio_manager_with_db, mocker, return_value=[])
+
+    with pytest.raises(ValueError, match="fresh historical data"):
+        await portfolio_manager_with_db.check_regime_rebalance_positions(
+            account_summary, portfolio_positions
+        )
+
+    assert (
+        portfolio_manager_with_db.ibkr.request_historical_data.call_count
+        == _expected_regime_history_fetches()
+    )
+
+
+@pytest.mark.asyncio
+async def test_regime_rebalance_rejects_unreadable_cached_history(
+    portfolio_manager_with_db, mocker, monkeypatch
+):
+    _disable_regime_history_retry_delay(monkeypatch)
+    _set_required_regime_history_dates(
+        portfolio_manager_with_db, monkeypatch, required_points=4
+    )
+    account_summary = _regime_account_summary()
+    portfolio_positions = _regime_stock_positions()
+
+    _mock_regime_broker(portfolio_manager_with_db, mocker, return_value=[])
+    portfolio_manager_with_db.data_store.get_historical_bars = mocker.Mock(
+        side_effect=RuntimeError("database unavailable")
+    )
+
+    with pytest.raises(ValueError, match="readable history cache"):
+        await portfolio_manager_with_db.check_regime_rebalance_positions(
+            account_summary, portfolio_positions
+        )
+
+    assert (
+        portfolio_manager_with_db.ibkr.request_historical_data.call_count
+        == _expected_regime_history_fetches()
+    )
+    assert portfolio_manager_with_db.data_store.get_historical_bars.call_count == 1
+
+
 @pytest.mark.asyncio
 async def test_regime_rebalance_volatility_weight_scales_down_without_renormalizing(
     portfolio_manager, mocker
@@ -1392,7 +1630,7 @@ async def test_regime_rebalance_insufficient_history(portfolio_manager, mocker):
     _mock_regime_history(portfolio_manager, mocker, [100.0, 110.0, 100.0])
     portfolio_manager.ibkr.request_executions = mocker.AsyncMock(return_value=[])
 
-    with pytest.raises(ValueError, match="full lookback history"):
+    with pytest.raises(ValueError, match="fresh historical data"):
         await portfolio_manager.check_regime_rebalance_positions(
             account_summary, portfolio_positions
         )
@@ -2123,6 +2361,27 @@ async def _get_history(contract, *_args, **_kwargs):
         )
 
 
+@pytest.mark.asyncio
+async def test_regime_rebalance_empty_history_stays_hard_failure(
+    portfolio_manager, mocker, monkeypatch
+):
+    _disable_regime_history_retry_delay(monkeypatch)
+    account_summary = _regime_account_summary()
+    portfolio_positions = _regime_stock_positions()
+
+    _mock_regime_broker(portfolio_manager, mocker, return_value=[])
+
+    with pytest.raises(ValueError, match="aligned history"):
+        await portfolio_manager.check_regime_rebalance_positions(
+            account_summary, portfolio_positions
+        )
+
+    assert (
+        portfolio_manager.ibkr.request_historical_data.call_count
+        == _expected_regime_history_fetches()
+    )
+
+
 @pytest.mark.asyncio
 async def test_regime_rebalance_zero_weights_raises(portfolio_manager, mocker):
     account_summary = {"NetLiquidation": SimpleNamespace(value="400")}