Generate daily option universe files in RandomDataGenerator (#9740)

* Generate daily option universe files in RandomDataGenerator (#8854)

- Automatically generate daily option universe CSV files when generating synthetic options data

- Add standalone data provider fallback to InterestRateProvider

- Add OptionUniverseWriter unit test suite

* Filter option contracts in OptionUniverseWriter to handle mixed security dictionaries

* Refactor OptionUniverseWriter to accept filtered option tick histories adhering to SRP

* Generate derivative universe files reusing the universe downloader

Feed the generator's existing daily aggregators, adding open interest for
derivatives, into an in memory IDataDownloader and write the option and
future universe files through UniverseExtensions.RunUniverseDownloader,
replacing the bespoke option writer. Universe generation is isolated so a
failure there logs and does not abort the data generation.

Fix the universe file header, which interpolated the CsvHeader method
group, and route future rows through FutureUniverse.ToCsv.

Revert the unrelated InterestRateProvider change, the data provider is
already registered by the ToolBox entry point.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013Zic3mz9ZogfCvhzoTXvDG

---------

Co-authored-by: Martin Molinero <martin.molinero1@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

Author: 3 people

Common/Data/UniverseSelection/DerivativeUniverseData.cs

@@ -117,6 +117,10 @@ public void UpdateByOpenInterest(OpenInterest openInterest)
     /// <returns>A CSV formatted string representing the data.</returns>
     public string ToCsv()
     {
+        if (_symbol.SecurityType == SecurityType.Future)
+        {
+            return FutureUniverse.ToCsv(_symbol, _open, _high, _low, _close, _volume, _openInterest);
+        }
         return OptionUniverse.ToCsv(_symbol, _open, _high, _low, _close, _volume, _openInterest, null, NullGreeks.Instance);
     }
 }

Common/Data/UniverseSelection/UniverseExtensions.cs

@@ -206,7 +206,8 @@ public static void RunUniverseDownloader(IDataDownloader dataDownloader, DataUni
 
                 using var writer = new StreamWriter(universeDownloadParameters.GetUniverseFileName(processingDate));
 
-                writer.WriteLine($"#{OptionUniverse.CsvHeader}");
+                var securityType = universeDownloadParameters.Symbol.SecurityType;
+                writer.WriteLine($"#{(securityType == SecurityType.Future ? FutureUniverse.CsvHeader : OptionUniverse.CsvHeader(securityType))}");
 
                 // Write option data, sorted by contract type (Call/Put), strike price, expiration date, and then by full ID
                 foreach (var universeData in universeDataBySymbol

Tests/Common/Data/UniverseSelection/DerivativeUniverseDataTests.cs

@@ -0,0 +1,54 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using NUnit.Framework;
+using QuantConnect.Data.Market;
+using QuantConnect.Data.UniverseSelection;
+
+namespace QuantConnect.Tests.Common.Data.UniverseSelection
+{
+    [TestFixture]
+    public class DerivativeUniverseDataTests
+    {
+        [Test]
+        public void FutureRowUsesTheContractMonth()
+        {
+            var symbol = Symbol.CreateFuture("ES", QuantConnect.Market.CME, new DateTime(2020, 3, 20));
+            var data = new DerivativeUniverseData(new TradeBar(new DateTime(2020, 1, 6), symbol, 10, 12, 9, 11, 100));
+            data.UpdateByOpenInterest(new OpenInterest(new DateTime(2020, 1, 6), symbol, 5000));
+
+            Assert.AreEqual("202003,10,12,9,11,100,5000", data.ToCsv());
+        }
+
+        [Test]
+        public void OptionRowUsesTheContractDetails()
+        {
+            var symbol = Symbol.CreateOption(Symbols.SPY, QuantConnect.Market.USA, OptionStyle.American, OptionRight.Put, 300, new DateTime(2020, 3, 20));
+            var data = new DerivativeUniverseData(new TradeBar(new DateTime(2020, 1, 6), symbol, 10, 12, 9, 11, 100));
+            data.UpdateByOpenInterest(new OpenInterest(new DateTime(2020, 1, 6), symbol, 5000));
+
+            Assert.AreEqual("20200320,300,P,10,12,9,11,100,5000,,0,0,0,0,0", data.ToCsv());
+        }
+
+        [Test]
+        public void UnderlyingRowHasNoContractDetails()
+        {
+            var data = new DerivativeUniverseData(new TradeBar(new DateTime(2020, 1, 6), Symbols.SPY, 10, 12, 9, 11, 100));
+
+            Assert.AreEqual(",,,10,12,9,11,100,,,0,0,0,0,0", data.ToCsv());
+        }
+    }
+}

Tests/ToolBox/RandomDataGenerator/RandomDataGeneratorTests.cs

@@ -20,6 +20,7 @@
 using QuantConnect.Data;
 using QuantConnect.ToolBox.RandomDataGenerator;
 using QuantConnect.Data.Market;
+using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
 using QuantConnect.Configuration;
 using QuantConnect.Data.Auxiliary;
@@ -222,6 +223,116 @@ public void RandomDataGeneratorCompletesSuccessfully()
             }
         }
 
+        [TestCase(SecurityType.Option, Market.USA, "AAPL", Resolution.Minute)]
+        [TestCase(SecurityType.Future, Market.CME, "ES", Resolution.Minute)]
+        [TestCase(SecurityType.Future, Market.CME, "ES", Resolution.Hour)]
+        [TestCase(SecurityType.Future, Market.CME, "ES", Resolution.Daily)]
+        [TestCase(SecurityType.Future, Market.CME, "ES", Resolution.Tick)]
+        public void RandomDataGeneratorWritesDerivativeUniverseFiles(SecurityType securityType, string market, string ticker, Resolution resolution)
+        {
+            var tempFolder = Path.Combine(Path.GetTempPath(), $"LeanTest_{Guid.NewGuid()}");
+            var originalDataFolder = Config.Get("data-folder");
+            try
+            {
+                Directory.CreateDirectory(tempFolder);
+                Config.Set("data-folder", tempFolder);
+                Globals.Reset();
+
+                var settings = new RandomDataGeneratorSettings
+                {
+                    Start = new DateTime(2020, 1, 6),
+                    End = new DateTime(2020, 1, 10),
+                    SymbolCount = 1,
+                    Market = market,
+                    SecurityType = securityType,
+                    Resolution = resolution,
+                    // keep the minute option case fast, the option price model is expensive
+                    DataDensity = resolution == Resolution.Minute ? DataDensity.Sparse : DataDensity.Dense,
+                    IncludeCoarse = false,
+                    QuoteTradeRatio = 1.0,
+                    RandomSeed = 123456,
+                    RandomSeedSet = true,
+                    ChainSymbolCount = 2,
+                    OptionPriceEngineName = "BaroneAdesiWhaleyApproximationEngine",
+                    Tickers = new List<string>() { ticker }
+                };
+
+                var generator = GetGenerator(settings);
+                Assert.DoesNotThrow(() => generator.Run());
+
+                var canonical = securityType == SecurityType.Future
+                    ? Symbol.Create(ticker, SecurityType.Future, market)
+                    : Symbol.CreateCanonicalOption(Symbol.Create(ticker, SecurityType.Equity, market));
+                var universeFiles = Directory.GetFiles(LeanData.GenerateUniversesDirectory(tempFolder, canonical), "*.csv");
+                Assert.IsNotEmpty(universeFiles);
+
+                var config = new SubscriptionDataConfig(securityType == SecurityType.Future ? typeof(FutureUniverse) : typeof(OptionUniverse),
+                    canonical, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, true, false);
+                BaseChainUniverseData factory = securityType == SecurityType.Future ? new FutureUniverse() : new OptionUniverse();
+                var expectedHeader = securityType == SecurityType.Future ? FutureUniverse.CsvHeader : OptionUniverse.CsvHeader(securityType);
+                var expectedContractsCount = securityType == SecurityType.Future ? 1 : settings.ChainSymbolCount * 2;
+                var maxContractsCount = 0;
+                var anyOpenInterest = false;
+                var anyContractPrice = false;
+                foreach (var universeFile in universeFiles)
+                {
+                    var date = DateTime.ParseExact(Path.GetFileNameWithoutExtension(universeFile), DateFormat.EightCharacter, null);
+                    Assert.IsTrue(date >= settings.Start && date <= settings.End, universeFile);
+
+                    var lines = File.ReadAllLines(universeFile);
+                    Assert.AreEqual($"#{expectedHeader}", lines[0], universeFile);
+
+                    // make sure Lean can read them back
+                    var rows = new List<BaseChainUniverseData>();
+                    using var reader = new StreamReader(universeFile);
+                    while (!reader.EndOfStream)
+                    {
+                        var data = (BaseChainUniverseData)factory.Reader(config, reader, date, false);
+                        if (data != null)
+                        {
+                            rows.Add(data);
+                        }
+                    }
+                    Assert.AreEqual(lines.Length - 1, rows.Count, universeFile);
+
+                    // options have an underlying data row first, then the contracts
+                    var contracts = rows.Where(x => x.Symbol.SecurityType == securityType).ToList();
+                    Assert.AreEqual(securityType == SecurityType.Future ? 0 : 1, rows.Count - contracts.Count, universeFile);
+                    Assert.IsTrue(securityType == SecurityType.Future || rows[0].Symbol == canonical.Underlying, universeFile);
+                    // options warm up on the first underlying data points, so the first days might have no contracts
+                    Assert.LessOrEqual(contracts.Count, expectedContractsCount, universeFile);
+                    maxContractsCount = Math.Max(maxContractsCount, contracts.Count);
+
+                    foreach (var row in rows)
+                    {
+                        Assert.IsFalse(row.Symbol.IsCanonical(), universeFile);
+                        Assert.AreEqual(canonical.ID.Symbol, row.Symbol.ID.Symbol, universeFile);
+                        if (row.Symbol.SecurityType == securityType)
+                        {
+                            Assert.GreaterOrEqual(row.Symbol.ID.Date, date, universeFile);
+                            // the price model can price a contract at zero and open interest is generated once a day, starting the second day
+                            anyContractPrice |= row.Close > 0 && row.Volume > 0;
+                            anyOpenInterest |= row.OpenInterest > 0;
+                        }
+                        else
+                        {
+                            Assert.Greater(row.Close, 0, universeFile);
+                            Assert.Greater(row.Volume, 0, universeFile);
+                        }
+                    }
+                }
+                Assert.AreEqual(expectedContractsCount, maxContractsCount);
+                Assert.IsTrue(anyContractPrice);
+                Assert.IsTrue(anyOpenInterest);
+            }
+            finally
+            {
+                Config.Set("data-folder", originalDataFolder);
+                Globals.Reset();
+                Directory.Delete(tempFolder, true);
+            }
+        }
+
         private static QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator GetGenerator(RandomDataGeneratorSettings settings)
         {
             var securityManager = new SecurityManager(new TimeKeeper(settings.Start, new[] { TimeZones.Utc }));

ToolBox/RandomDataGenerator/InMemoryDataDownloader.cs

@@ -0,0 +1,74 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Securities;
+
+namespace QuantConnect.ToolBox.RandomDataGenerator
+{
+    /// <summary>
+    /// <see cref="IDataDownloader"/> implementation serving the generated data it was fed,
+    /// used to create the derivative universe files with the same code path as the data downloaders
+    /// </summary>
+    public class InMemoryDataDownloader : IDataDownloader
+    {
+        private readonly List<(TickType TickType, BaseData Data)> _data = new();
+
+        /// <summary>
+        /// The exchange hours of the data this instance holds
+        /// </summary>
+        public SecurityExchangeHours ExchangeHours { get; }
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="exchangeHours">The exchange hours of the data this instance will hold, which is in the exchange time zone</param>
+        public InMemoryDataDownloader(SecurityExchangeHours exchangeHours)
+        {
+            ExchangeHours = exchangeHours;
+        }
+
+        /// <summary>
+        /// Adds data to serve
+        /// </summary>
+        /// <param name="tickType">The tick type of the data</param>
+        /// <param name="data">The data to add, in the exchange time zone</param>
+        public void Add(TickType tickType, IEnumerable<BaseData> data)
+        {
+            _data.AddRange(data.Select(x => (tickType, x)));
+        }
+
+        /// <summary>
+        /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
+        /// Requests for a canonical symbol will return the data of all its contracts
+        /// </summary>
+        /// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
+        /// <returns>Enumerable of base data for this symbol</returns>
+        public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
+        {
+            var symbol = dataDownloaderGetParameters.Symbol;
+            var start = dataDownloaderGetParameters.StartUtc.ConvertFromUtc(ExchangeHours.TimeZone);
+            var end = dataDownloaderGetParameters.EndUtc.ConvertFromUtc(ExchangeHours.TimeZone);
+
+            return _data
+                .Where(x => x.TickType == dataDownloaderGetParameters.TickType
+                    && x.Data.Time >= start && x.Data.Time < end
+                    && (symbol.IsCanonical() ? x.Data.Symbol.HasCanonical() && x.Data.Symbol.Canonical == symbol : x.Data.Symbol == symbol))
+                .Select(x => x.Data);
+        }
+    }
+}

ToolBox/RandomDataGenerator/RandomDataGenerator.cs

@@ -16,12 +16,15 @@
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
+using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Securities;
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
 using QuantConnect.Logging;
+using QuantConnect.Util;
 
 namespace QuantConnect.ToolBox.RandomDataGenerator
 {
@@ -74,6 +77,8 @@ public void Run()
             var count = 0;
             var progress = 0d;
             var previousMonth = -1;
+            // daily data of the generated derivative contracts and their underlying, by canonical symbol, used to generate the universe files
+            var universeDownloaders = new Dictionary<Symbol, InMemoryDataDownloader>();
 
             foreach (var (symbolRef, currentSymbolGroup) in symbolGenerator.GenerateRandomSymbols()
                 .GroupBy(s => s.HasUnderlying ? s.Underlying : s)
@@ -84,6 +89,7 @@ public void Run()
                 var tickGenerators = new List<IEnumerator<Tick>>();
                 var tickHistories = new Dictionary<Symbol, List<Tick>>();
                 Security underlyingSecurity = null;
+                InMemoryDataDownloader universeDownloader = null;
                 foreach (var currentSymbol in currentSymbolGroup)
                 {
                     if (!_securityManager.TryGetValue(currentSymbol, out var security))
@@ -97,6 +103,11 @@ public void Run()
 
                     underlyingSecurity ??= security;
 
+                    if (currentSymbol.HasCanonical() && !universeDownloaders.TryGetValue(currentSymbol.Canonical, out universeDownloader))
+                    {
+                        universeDownloader = universeDownloaders[currentSymbol.Canonical] = new InMemoryDataDownloader(security.Exchange.Hours);
+                    }
+
                     tickGenerators.Add(
                         new TickGenerator(_settings, tickTypesPerSecurityType[currentSymbol.SecurityType].ToArray(), security, randomValueGenerator)
                             .GenerateTicks()
@@ -268,7 +279,14 @@ public void Run()
                             // lest we likely wouldn't get the last piece of data stuck in the consolidator
                             // Filter out the data we're going to write here because filtering them in the consolidator update phase
                             // makes it write all dates for some unknown reason
-                            writer.Write(item.Flush().Where(data => data.Time > previousRenameDate && previousRenameDateDay != DataDay(data)));
+                            var consolidated = item.Flush().Where(data => data.Time > previousRenameDate && previousRenameDateDay != DataDay(data)).ToList();
+                            writer.Write(consolidated);
+
+                            if (item.Resolution == Resolution.Daily)
+                            {
+                                // the daily data is what we use to generate the derivative universe files
+                                universeDownloader?.Add(item.TickType, consolidated);
+                            }
                         }
 
                         // update progress
@@ -281,6 +299,22 @@ public void Run()
                 }
             }
 
+            foreach (var (canonicalSymbol, universeDownloader) in universeDownloaders)
+            {
+                Log.Trace($"RandomDataGenerator.Run(): {canonicalSymbol} - Generating universe files...");
+                try
+                {
+                    Directory.CreateDirectory(LeanData.GenerateUniversesDirectory(Globals.DataFolder, canonicalSymbol));
+                    UniverseExtensions.RunUniverseDownloader(universeDownloader,
+                        new DataUniverseDownloaderGetParameters(canonicalSymbol, _settings.Start, _settings.End, universeDownloader.ExchangeHours));
+                }
+                catch (Exception exception)
+                {
+                    // the universe files are a nice to have, let's not fail the whole generation
+                    Log.Error(exception, $"RandomDataGenerator.Run(): {canonicalSymbol} - Failed to generate universe files");
+                }
+            }
+
             Log.Trace("RandomDataGenerator.Run(): Random data generation has completed.");
 
             DateTime TickDay(Tick tick) => new(tick.Time.Year, tick.Time.Month, tick.Time.Day);
@@ -316,8 +350,9 @@ public static IEnumerable<TickAggregator> CreateAggregators(RandomDataGeneratorS
             }
 
 
-            // ensure we have a daily consolidator when coarse is enabled
-            if (settings.IncludeCoarse && settings.Resolution != Resolution.Daily)
+            // ensure we have a daily consolidator when coarse is enabled or for derivatives, whose universe files are generated from daily data
+            var isDerivative = settings.SecurityType.IsOption() || settings.SecurityType == SecurityType.Future;
+            if ((settings.IncludeCoarse || isDerivative) && settings.Resolution != Resolution.Daily)
             {
                 // prefer trades for coarse - in practice equity only does trades, but leaving this as configurable
                 if (tickTypes.Contains(TickType.Trade))
@@ -328,6 +363,11 @@ public static IEnumerable<TickAggregator> CreateAggregators(RandomDataGeneratorS
                 {
                     yield return TickAggregator.ForTickTypes(settings.SecurityType, Resolution.Daily, TickType.Quote).Single();
                 }
+
+                if (isDerivative && tickTypes.Contains(TickType.OpenInterest))
+                {
+                    yield return TickAggregator.ForTickTypes(settings.SecurityType, Resolution.Daily, TickType.OpenInterest).Single();
+                }
             }
         }
     }