Fix QuantBook Option/Future history api date range (#8671)

* Fix QuantBook Option/Future history api date range

Properly calculate date range when passed range is not a whole date but a time of day

* Extended unit tests

Author: jhonabreul

Research/QuantBook.cs

@@ -420,7 +420,7 @@ public OptionHistory OptionHistory(Symbol symbol, DateTime start, DateTime? end
                         // only add underlying if not present
                         AddIndex(symbol.Underlying.Value, resolutionToUseForUnderlying, fillForward: fillForward);
                     }
-                    else if(symbol.Underlying.SecurityType == SecurityType.Future && symbol.Underlying.IsCanonical())
+                    else if (symbol.Underlying.SecurityType == SecurityType.Future && symbol.Underlying.IsCanonical())
                     {
                         AddFuture(symbol.Underlying.ID.Symbol, resolutionToUseForUnderlying, fillForward: fillForward,
                             extendedMarketHours: extendedMarketHours);
@@ -435,13 +435,11 @@ public OptionHistory OptionHistory(Symbol symbol, DateTime start, DateTime? end
                 var allSymbols = new HashSet<Symbol>();
                 var optionFilterUniverse = new OptionFilterUniverse(option);
 
-                foreach (var date in QuantConnect.Time.EachTradeableDay(option, start, end.Value.AddDays(-1), extendedMarketHours))
+                foreach (var (date, chainData, underlyingData) in GetChainHistory<OptionUniverse>(option, start, end.Value, extendedMarketHours))
                 {
-                    var universeData = GetChainHistory<OptionUniverse>(symbol, date, out var underlyingData);
-
                     if (underlyingData is not null)
                     {
-                        optionFilterUniverse.Refresh(universeData, underlyingData, underlyingData.EndTime);
+                        optionFilterUniverse.Refresh(chainData, underlyingData, underlyingData.EndTime);
                         allSymbols.UnionWith(option.ContractFilter.Filter(optionFilterUniverse).Select(x => x.Symbol));
                     }
                 }
@@ -499,13 +497,9 @@ public FutureHistory FutureHistory(Symbol symbol, DateTime start, DateTime? end
                 // canonical symbol, lets find the contracts
                 var future = Securities[symbol] as Future;
 
-                for (var date = start; date < end; date = date.AddDays(1))
+                foreach (var (date, chainData, underlyingData) in GetChainHistory<FutureUniverse>(future, start, end.Value, extendedMarketHours))
                 {
-                    if (future.Exchange.DateIsOpen(date, extendedMarketHours))
-                    {
-                        var universeData = GetChainHistory<FutureUniverse>(future.Symbol, date, out _);
-                        allSymbols.UnionWith(future.ContractFilter.Filter(new FutureFilterUniverse(universeData, date)).Select(x => x.Symbol));
-                    }
+                    allSymbols.UnionWith(future.ContractFilter.Filter(new FutureFilterUniverse(chainData, date)).Select(x => x.Symbol));
                 }
             }
             else
@@ -882,6 +876,20 @@ private IEnumerable<T> GetChainHistory<T>(Symbol canonicalSymbol, DateTime date,
             return Enumerable.Empty<T>();
         }
 
+        /// <summary>
+        /// Helper method to get option/future chain historical data for a given date range
+        /// </summary>
+        private IEnumerable<(DateTime Date, IEnumerable<T> ChainData, BaseData UnderlyingData)> GetChainHistory<T>(
+            Security security, DateTime start, DateTime end, bool extendedMarketHours)
+            where T : BaseChainUniverseData
+        {
+            foreach (var date in QuantConnect.Time.EachTradeableDay(security, start.Date, end.Date, extendedMarketHours))
+            {
+                var universeData = GetChainHistory<T>(security.Symbol, date, out var underlyingData);
+                yield return (date, universeData, underlyingData);
+            }
+        }
+
         /// <summary>
         /// Helper method to perform selection on the given data and filter it
         /// </summary>

Tests/Research/QuantBookHistoryTests.cs

@@ -22,6 +22,12 @@
 using QuantConnect.Research;
 using QuantConnect.Logging;
 using QuantConnect.Data.Fundamental;
+using System.Data;
+using QuantConnect.Securities.Future;
+using QuantConnect.Data;
+using NodaTime;
+using QuantConnect.Interfaces;
+using QuantConnect.Data.UniverseSelection;
 
 namespace QuantConnect.Tests.Research
 {
@@ -179,6 +185,65 @@ public void CanonicalOptionIntradayQuantBookHistory()
             }
         }
 
+        private static TestCaseData[] CanonicalOptionIntradayHistoryTestCases
+        {
+            get
+            {
+                var twx = Symbol.Create("TWX", SecurityType.Equity, Market.USA);
+                var twxOption = Symbol.CreateCanonicalOption(twx);
+
+                return
+                [
+                    new TestCaseData(twxOption, new DateTime(2014, 06, 05), (DateTime?)null),
+                    new TestCaseData(twxOption, new DateTime(2014, 06, 05), new DateTime(2014, 06, 05)),
+                    new TestCaseData(twxOption, new DateTime(2014, 06, 05), new DateTime(2014, 06, 06)),
+                    new TestCaseData(twxOption, new DateTime(2014, 06, 05, 0, 0, 0), new DateTime(2014, 06, 05, 15, 0, 0)),
+                    new TestCaseData(twxOption, new DateTime(2014, 06, 05, 10, 0, 0), new DateTime(2014, 06, 05, 15, 0, 0)),
+                    new TestCaseData(twxOption, new DateTime(2014, 06, 05, 10, 0, 0), new DateTime(2014, 06, 06)),
+                    new TestCaseData(twxOption, new DateTime(2014, 06, 05, 10, 0, 0), new DateTime(2014, 06, 06, 10, 0, 0)),
+                    new TestCaseData(twxOption, new DateTime(2014, 06, 05, 10, 0, 0), new DateTime(2014, 06, 06, 15, 0, 0))
+                ];
+            }
+        }
+
+        [TestCaseSource(nameof(CanonicalOptionIntradayHistoryTestCases))]
+        public void CanonicalOptionIntradayQuantBookHistoryWithIntradayRange(Symbol canonicalOption, DateTime start, DateTime? end)
+        {
+            var quantBook = new QuantBook();
+            var historyProvider = new TestHistoryProvider(quantBook.HistoryProvider);
+            quantBook.SetHistoryProvider(historyProvider);
+            quantBook.SetStartDate((end ?? start).Date.AddDays(1));
+
+            var option = quantBook.AddSecurity(canonicalOption);
+            var history = quantBook.OptionHistory(canonicalOption, start, end, Resolution.Minute);
+
+            Assert.Greater(history.Count, 0);
+
+            var symbolsInHistory = history.SelectMany(slice => slice.AllData.Select(x => x.Symbol)).Distinct().ToList();
+            Assert.Greater(symbolsInHistory.Count, 1);
+
+            var underlying = symbolsInHistory.Where(x => x == canonicalOption.Underlying).ToList();
+            Assert.AreEqual(1, underlying.Count);
+
+            var contractsSymbols = symbolsInHistory.Where(x => x.SecurityType == canonicalOption.SecurityType).ToList();
+            Assert.Greater(contractsSymbols.Count, 1);
+
+            var expectedDates = new HashSet<DateTime> { start.Date };
+            if (end.HasValue && end.Value > end.Value.Date)
+            {
+                expectedDates.Add(end.Value.Date);
+            }
+
+            var dataDates = history.SelectMany(slice => slice.AllData.Where(x => contractsSymbols.Contains(x.Symbol)).Select(x => x.EndTime.Date)).ToHashSet();
+            CollectionAssert.AreEqual(expectedDates, dataDates);
+
+            // OptionUniverse must have been requested for all dates in the range
+            foreach (var date in Time.EachTradeableDay(option, start.Date, (end ?? start).Date))
+            {
+                Assert.AreEqual(1, historyProvider.HistoryRequests.Count(request => request.DataType == typeof(OptionUniverse) && request.EndTimeLocal == date));
+            }
+        }
+
         [Test]
         public void OptionContractQuantBookHistory()
         {
@@ -301,6 +366,57 @@ public void CanonicalFutureIntradayQuantBookHistory(int maxFilter, int numberOfF
             }
         }
 
+        private static TestCaseData[] CanonicalFutureIntradayHistoryTestCases
+        {
+            get
+            {
+                var es = Symbol.Create(Futures.Indices.SP500EMini, SecurityType.Future, Market.CME);
+                return
+                [
+                    new TestCaseData(es, new DateTime(2013, 10, 10), (DateTime?)null),
+                    new TestCaseData(es, new DateTime(2013, 10, 10), new DateTime(2013, 10, 10)),
+                    new TestCaseData(es, new DateTime(2013, 10, 10), new DateTime(2013, 10, 11)),
+                    new TestCaseData(es, new DateTime(2013, 10, 10, 0, 0, 0), new DateTime(2013, 10, 10, 15, 0, 0)),
+                    new TestCaseData(es, new DateTime(2013, 10, 10, 10, 0, 0), new DateTime(2013, 10, 10, 15, 0, 0)),
+                    new TestCaseData(es, new DateTime(2013, 10, 10, 10, 0, 0), new DateTime(2013, 10, 11)),
+                    new TestCaseData(es, new DateTime(2013, 10, 10, 10, 0, 0), new DateTime(2013, 10, 11, 10, 0, 0)),
+                    new TestCaseData(es, new DateTime(2013, 10, 10, 10, 0, 0), new DateTime(2013, 10, 11, 15, 0, 0))
+                ];
+            }
+        }
+
+        [TestCaseSource(nameof(CanonicalFutureIntradayHistoryTestCases))]
+        public void CanonicalFutureIntradayQuantBookHistoryWithIntradayRange(Symbol canonicalFuture, DateTime start, DateTime? end)
+        {
+            var quantBook = new QuantBook();
+            var historyProvider = new TestHistoryProvider(quantBook.HistoryProvider);
+            quantBook.SetHistoryProvider(historyProvider);
+            quantBook.SetStartDate((end ?? start).Date.AddDays(1));
+            var future = quantBook.AddSecurity(canonicalFuture) as Future;
+            future.SetFilter(universe => universe);
+
+            var history = quantBook.FutureHistory(canonicalFuture, start, end, Resolution.Minute);
+            Assert.Greater(history.Count, 0);
+
+            var symbolsInHistory = history.SelectMany(slice => slice.AllData.Select(x => x.Symbol)).Distinct().ToList();
+            Assert.Greater(symbolsInHistory.Count, 1);
+
+            var expectedDates = new HashSet<DateTime> { start.Date };
+            if (end.HasValue && end.Value > end.Value.Date)
+            {
+                expectedDates.Add(end.Value.Date);
+            }
+
+            var dataDates = history.SelectMany(slice => slice.AllData.Select(x => x.EndTime.Date)).ToHashSet();
+            CollectionAssert.AreEqual(expectedDates, dataDates);
+
+            // FutureUniverse must have been requested for all dates in the range
+            foreach (var date in Time.EachTradeableDay(future, start.Date, (end ?? start).Date))
+            {
+                Assert.AreEqual(1, historyProvider.HistoryRequests.Count(request => request.DataType == typeof(FutureUniverse) && request.EndTimeLocal == date));
+            }
+        }
+
         [Test]
         public void FutureContractQuantBookHistory()
         {
@@ -685,5 +801,30 @@ def getHistory():
                 Assert.IsFalse(pyHistory.HasAttr("data"));
             }
         }
+
+        private class TestHistoryProvider : HistoryProviderBase
+        {
+            private IHistoryProvider _provider;
+
+            public List<HistoryRequest> HistoryRequests { get; } = new();
+
+            public override int DataPointCount => _provider.DataPointCount;
+
+            public TestHistoryProvider(IHistoryProvider provider)
+            {
+                _provider = provider;
+            }
+
+            public override void Initialize(HistoryProviderInitializeParameters parameters)
+            {
+            }
+
+            public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> requests, DateTimeZone sliceTimeZone)
+            {
+                requests = requests.ToList();
+                HistoryRequests.AddRange(requests);
+                return _provider.GetHistory(requests, sliceTimeZone);
+            }
+        }
     }
 }