From 242256d64733ba3b0d059516f9b2643f1f577131 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 28 Feb 2025 17:18:05 -0400 Subject: [PATCH 1/9] Fix security re-addition behavior SecurityService to re-use existing securities as a cache. Make sure securities are seeded on re-addition. --- ...tyReAdditionBehaviorRegressionAlgorithm.cs | 177 ++++++++++++++++++ Common/Securities/Security.cs | 20 +- Common/Securities/SecurityService.cs | 15 +- Engine/DataFeeds/UniverseSelection.cs | 11 +- Tests/Algorithm/AlgorithmAddDataTests.cs | 34 +++- Tests/Algorithm/AlgorithmAddSecurityTests.cs | 3 +- .../AlgorithmUniverseSettingsTests.cs | 6 +- ...eInsightPortfolioConstructionModelTests.cs | 10 +- Tests/Brokerages/Paper/PaperBrokerageTests.cs | 2 +- .../SecurityPortfolioManagerTests.cs | 4 +- .../Common/Securities/SecurityServiceTests.cs | 3 +- Tests/Common/Util/ExtensionsTests.cs | 3 +- Tests/Engine/AlgorithmManagerTests.cs | 3 +- .../BrokerageTransactionHandlerTests.cs | 4 +- Tests/Engine/DataFeeds/DataManagerTests.cs | 4 +- .../DataFeeds/FileSystemDataFeedTests.cs | 2 +- .../InternalSubscriptionManagerTests.cs | 3 +- .../DataFeeds/LiveTradingDataFeedTests.cs | 11 +- .../SubscriptionSynchronizerTests.cs | 3 +- .../Setup/BrokerageSetupHandlerTests.cs | 28 ++- Tests/Report/PortfolioLooperAlgorithmTests.cs | 6 +- .../RandomDataGeneratorTests.cs | 6 +- 22 files changed, 318 insertions(+), 40 deletions(-) create mode 100644 Algorithm.CSharp/SecurityEquityReAdditionBehaviorRegressionAlgorithm.cs diff --git a/Algorithm.CSharp/SecurityEquityReAdditionBehaviorRegressionAlgorithm.cs b/Algorithm.CSharp/SecurityEquityReAdditionBehaviorRegressionAlgorithm.cs new file mode 100644 index 000000000000..0f26e3ca65af --- /dev/null +++ b/Algorithm.CSharp/SecurityEquityReAdditionBehaviorRegressionAlgorithm.cs @@ -0,0 +1,177 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Interfaces; +using QuantConnect.Securities; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// + public class SecurityEquityReAdditionBehaviorRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + private Security _equity; + private Queue _tradableDates; + private bool _securityWasRemoved; + + public override void Initialize() + { + SetStartDate(2013, 10, 04); + SetEndDate(2013, 10, 30); + + var seeder = new FuncSecuritySeeder((security) => + { + Debug($"[{Time}] Seeding {security.Symbol}"); + + return GetLastKnownPrices(security); + }); + SetSecurityInitializer(security => + { + seeder.SeedSecurity(security); + }); + + _equity = AddEquity("SPY"); + + _tradableDates = new(QuantConnect.Time.EachTradeableDay(_equity.Exchange.Hours, StartDate, EndDate)); + } + + public override void OnEndOfDay(Symbol symbol) + { + if (symbol != _equity.Symbol) + { + throw new RegressionTestException($"Expected the symbol to be the equity symbol. Got {symbol}"); + } + + var currentTradableDate = _tradableDates.Dequeue(); + if (currentTradableDate != Time.Date) + { + throw new RegressionTestException($"Expected the current tradable date to be {Time.Date}. Got {currentTradableDate}"); + } + + // Remove the security every day + Debug($"[{Time}] Removing the equity"); + _securityWasRemoved = RemoveSecurity(symbol); + + if (!_securityWasRemoved) + { + throw new RegressionTestException($"Expected the equity to be removed"); + } + } + + public override void OnSecuritiesChanged(SecurityChanges changes) + { + if (_securityWasRemoved) + { + if (changes.AddedSecurities.Count > 0) + { + throw new RegressionTestException($"Expected no securities to be added. Got {changes.AddedSecurities.Count}"); + } + + if (!changes.RemovedSecurities.Contains(_equity)) + { + throw new RegressionTestException($"Expected the equity to be removed. Got {changes.RemovedSecurities.Count}"); + } + + _securityWasRemoved = false; + + // Add the security back + Debug($"[{Time}] Re-adding the equity"); + var reAddedEquity = AddEquity("SPY"); + + if (reAddedEquity != _equity) + { + throw new RegressionTestException($"Expected the re-added equity to be the same as the original equity"); + } + + if (!reAddedEquity.IsTradable) + { + throw new RegressionTestException($"Expected the re-added equity to be tradable"); + } + } + else if (!changes.AddedSecurities.Contains(_equity)) + { + throw new RegressionTestException($"Expected the equity to be added back"); + } + } + + public override void OnEndOfAlgorithm() + { + + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 774; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 10; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.RuntimeError; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "0"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100000"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "0"}, + {"Tracking Error", "0"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "$0.00"}, + {"Estimated Strategy Capacity", "$0"}, + {"Lowest Capacity Asset", ""}, + {"Portfolio Turnover", "0%"}, + {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"} + }; + } +} diff --git a/Common/Securities/Security.cs b/Common/Securities/Security.cs index c06f10cc6040..939dae743306 100644 --- a/Common/Securities/Security.cs +++ b/Common/Securities/Security.cs @@ -1069,7 +1069,7 @@ internal void AddData(SubscriptionDataConfig subscription) { throw new ArgumentException(Messages.Security.UnmatchingExchangeTimeZones, $"{nameof(subscription)}.{nameof(subscription.ExchangeTimeZone)}"); } - _subscriptionsBag.Add(subscription); + AddSubscription(subscription); UpdateSubscriptionProperties(); } } @@ -1092,7 +1092,7 @@ internal void AddData(SubscriptionDataConfigList subscriptions) { throw new ArgumentException(Messages.Security.UnmatchingExchangeTimeZones, $"{nameof(subscription)}.{nameof(subscription.ExchangeTimeZone)}"); } - _subscriptionsBag.Add(subscription); + AddSubscription(subscription); } UpdateSubscriptionProperties(); } @@ -1173,5 +1173,21 @@ internal virtual void UpdateSymbolProperties(SymbolProperties symbolProperties) SymbolProperties = symbolProperties; } } + + /// + /// Add a new subscription to this security if not already present + /// + /// + /// The caller must acquire the lock. + /// Using this instead of changing from a list to a hash set to keep + /// the same behavior. Don't want to change the ordering. + /// + private void AddSubscription(SubscriptionDataConfig subscription) + { + if (!_subscriptionsBag.Contains(subscription)) + { + _subscriptionsBag.Add(subscription); + } + } } } diff --git a/Common/Securities/SecurityService.cs b/Common/Securities/SecurityService.cs index 4fb2f4a85f65..4f8d81cf2fdb 100644 --- a/Common/Securities/SecurityService.cs +++ b/Common/Securities/SecurityService.cs @@ -76,8 +76,21 @@ private Security CreateSecurity(Symbol symbol, var configList = new SubscriptionDataConfigList(symbol); configList.AddRange(subscriptionDataConfigList); + if (_algorithm != null && _algorithm.Securities.TryGetValue(symbol, out var existingSecurity)) + { + existingSecurity.AddData(configList); + + // invoke the security initializer + if (initializeSecurity && !_algorithm.UniverseManager.Values.Any(u => u.Selected != null && u.Selected.Contains(existingSecurity.Symbol))) + { + _securityInitializerProvider.SecurityInitializer.Initialize(existingSecurity); + } + + return existingSecurity; + } + var dataTypes = Enumerable.Empty(); - if(symbol.SecurityType == SecurityType.Base && SecurityIdentifier.TryGetCustomDataTypeInstance(symbol.ID.Symbol, out var type)) + if (symbol.SecurityType == SecurityType.Base && SecurityIdentifier.TryGetCustomDataTypeInstance(symbol.ID.Symbol, out var type)) { dataTypes = new[] { type }; } diff --git a/Engine/DataFeeds/UniverseSelection.cs b/Engine/DataFeeds/UniverseSelection.cs index dcc68a081e2d..da407e5cc785 100644 --- a/Engine/DataFeeds/UniverseSelection.cs +++ b/Engine/DataFeeds/UniverseSelection.cs @@ -209,8 +209,12 @@ public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTi if (!member.Security.IsDelisted) { - // TODO: here we are not checking if other universes have this security still selected - _securityChangesConstructor.Remove(member.Security, member.IsInternal); + // let's not emit remove event for securities that are active in other universes + var universes = _algorithm.UniverseManager.Values.Where(u => !ReferenceEquals(u, universe)); + if (universes.All(u => (u.Selected == null || !u.Selected.Contains(security.Symbol)) && (!u.Members.ContainsKey(security.Symbol) || u.CanRemoveMember(dateTimeUtc, security)))) + { + _securityChangesConstructor.Remove(member.Security, member.IsInternal); + } } RemoveSecurityFromUniverse(_pendingRemovalsManager.TryRemoveMember(security, universe), @@ -490,8 +494,7 @@ private void RemoveSecurityFromUniverse( private Security GetOrCreateSecurity(Dictionary pendingAdditions, Symbol symbol, UniverseSettings universeSettings, Security underlying = null) { // create the new security, the algorithm thread will add this at the appropriate time - Security security; - if (!pendingAdditions.TryGetValue(symbol, out security) && !_algorithm.Securities.TryGetValue(symbol, out security)) + if (!pendingAdditions.TryGetValue(symbol, out var security)) { security = _securityService.CreateSecurity(symbol, new List(), universeSettings.Leverage, symbol.ID.SecurityType.IsOption(), underlying); diff --git a/Tests/Algorithm/AlgorithmAddDataTests.cs b/Tests/Algorithm/AlgorithmAddDataTests.cs index e39d0adce033..0721cb492ddf 100644 --- a/Tests/Algorithm/AlgorithmAddDataTests.cs +++ b/Tests/Algorithm/AlgorithmAddDataTests.cs @@ -65,8 +65,10 @@ public void DefaultDataFeeds_CanBeOverwritten_Successfully() // new forex - should be tradebar var forexQuote = algo.AddForex("EURUSD"); - Assert.IsTrue(forexQuote.Subscriptions.Count() == 1); + // The quote bar subscription is kept + Assert.IsTrue(forexQuote.Subscriptions.Count() == 2); Assert.IsTrue(GetMatchingSubscription(algo, forexQuote.Symbol, typeof(TradeBar)) != null); + Assert.IsTrue(GetMatchingSubscription(algo, forexQuote.Symbol, typeof(QuoteBar)) != null); // reset to empty string, affects other tests because config is static Config.Set("security-data-feeds", ""); @@ -91,9 +93,20 @@ public void DefaultDataFeeds_AreAdded_Successfully() // equity low resolution var equityDaily = algo.AddSecurity(SecurityType.Equity, "goog", Resolution.Daily); - Assert.IsTrue(equityDaily.Subscriptions.Count() == 2); + Assert.IsTrue(equityDaily.Subscriptions.Count() == 3); Assert.IsTrue(GetMatchingSubscription(algo, equityDaily.Symbol, typeof(TradeBar)) != null); + Assert.IsTrue(GetMatchingSubscription(algo, equityMinute.Symbol, typeof(QuoteBar)) != null); + + Assert.IsTrue(ReferenceEquals(equityMinute, equityDaily)); + var equitySubscriptions = algo.SubscriptionManager.SubscriptionDataConfigService + .GetSubscriptionDataConfigs(equityMinute.Symbol); + Assert.IsTrue(equitySubscriptions.SingleOrDefault( + s => s.TickType == TickType.Trade && s.Type == typeof(TradeBar) && s.Resolution == Resolution.Minute) != null); + Assert.IsTrue(equitySubscriptions.SingleOrDefault( + s => s.TickType == TickType.Quote && s.Type == typeof(QuoteBar) && s.Resolution == Resolution.Minute) != null); + Assert.IsTrue(equitySubscriptions.SingleOrDefault( + s => s.TickType == TickType.Trade && s.Type == typeof(TradeBar) && s.Resolution == Resolution.Daily) != null); // option var option = algo.AddSecurity(SecurityType.Option, "goog"); @@ -116,16 +129,29 @@ public void DefaultDataFeeds_AreAdded_Successfully() Assert.IsTrue(future.Subscriptions.FirstOrDefault(x => typeof(FutureUniverse) == x.Type) != null); // Crypto high resolution - var cryptoMinute = algo.AddSecurity(SecurityType.Equity, "goog"); + var cryptoMinute = algo.AddSecurity(SecurityType.Crypto, "btcusd"); Assert.IsTrue(cryptoMinute.Subscriptions.Count() == 2); Assert.IsTrue(GetMatchingSubscription(algo, cryptoMinute.Symbol, typeof(TradeBar)) != null); Assert.IsTrue(GetMatchingSubscription(algo, cryptoMinute.Symbol, typeof(QuoteBar)) != null); // Crypto low resolution var cryptoHourly = algo.AddSecurity(SecurityType.Crypto, "btcusd", Resolution.Hour); - Assert.IsTrue(cryptoHourly.Subscriptions.Count() == 2); + Assert.IsTrue(cryptoHourly.Subscriptions.Count() == 4); Assert.IsTrue(GetMatchingSubscription(algo, cryptoHourly.Symbol, typeof(TradeBar)) != null); Assert.IsTrue(GetMatchingSubscription(algo, cryptoHourly.Symbol, typeof(QuoteBar)) != null); + + Assert.IsTrue(ReferenceEquals(cryptoMinute, cryptoHourly)); + + var cryptoSubscriptions = algo.SubscriptionManager.SubscriptionDataConfigService + .GetSubscriptionDataConfigs(cryptoMinute.Symbol); + Assert.IsTrue(cryptoSubscriptions.SingleOrDefault( + s => s.TickType == TickType.Trade && s.Type == typeof(TradeBar) && s.Resolution == Resolution.Minute) != null); + Assert.IsTrue(cryptoSubscriptions.SingleOrDefault( + s => s.TickType == TickType.Quote && s.Type == typeof(QuoteBar) && s.Resolution == Resolution.Minute) != null); + Assert.IsTrue(cryptoSubscriptions.SingleOrDefault( + s => s.TickType == TickType.Trade && s.Type == typeof(TradeBar) && s.Resolution == Resolution.Hour) != null); + Assert.IsTrue(cryptoSubscriptions.SingleOrDefault( + s => s.TickType == TickType.Quote && s.Type == typeof(QuoteBar) && s.Resolution == Resolution.Hour) != null); } diff --git a/Tests/Algorithm/AlgorithmAddSecurityTests.cs b/Tests/Algorithm/AlgorithmAddSecurityTests.cs index 873e25a9d1c7..d2b701ee7bc9 100644 --- a/Tests/Algorithm/AlgorithmAddSecurityTests.cs +++ b/Tests/Algorithm/AlgorithmAddSecurityTests.cs @@ -139,7 +139,8 @@ public void AddFutureWithExtendedMarketHours(Func getFutu SymbolPropertiesDatabase.FromDataFolder(), _algo, RegisteredSecurityDataTypesProvider.Null, - new SecurityCacheProvider(_algo.Portfolio)); + new SecurityCacheProvider(_algo.Portfolio), + algorithm: _algo); _algo.Securities.SetSecurityService(securityService); var future = getFuture(_algo); diff --git a/Tests/Algorithm/AlgorithmUniverseSettingsTests.cs b/Tests/Algorithm/AlgorithmUniverseSettingsTests.cs index d3d9f0c4a255..c01923ee5689 100644 --- a/Tests/Algorithm/AlgorithmUniverseSettingsTests.cs +++ b/Tests/Algorithm/AlgorithmUniverseSettingsTests.cs @@ -144,7 +144,8 @@ private Tuple GetAlgorithmAndDataManager() symbolPropertiesDatabase, algorithm, RegisteredSecurityDataTypesProvider.Null, - new SecurityCacheProvider(algorithm.Portfolio)), + new SecurityCacheProvider(algorithm.Portfolio), + algorithm: algorithm), dataPermissionManager, TestGlobals.DataProvider), algorithm, @@ -160,7 +161,8 @@ private Tuple GetAlgorithmAndDataManager() symbolPropertiesDatabase, algorithm, RegisteredSecurityDataTypesProvider.Null, - new SecurityCacheProvider(algorithm.Portfolio)); + new SecurityCacheProvider(algorithm.Portfolio), + algorithm: algorithm); algorithm.SubscriptionManager.SetDataManager(dataManager); algorithm.Securities.SetSecurityService(securityService); diff --git a/Tests/Algorithm/Framework/Portfolio/AccumulativeInsightPortfolioConstructionModelTests.cs b/Tests/Algorithm/Framework/Portfolio/AccumulativeInsightPortfolioConstructionModelTests.cs index 18c9b91278be..a92062d3e21b 100644 --- a/Tests/Algorithm/Framework/Portfolio/AccumulativeInsightPortfolioConstructionModelTests.cs +++ b/Tests/Algorithm/Framework/Portfolio/AccumulativeInsightPortfolioConstructionModelTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -91,7 +91,7 @@ public void InsightsReturnsTargetsConsistentWithDirection(Language language, Ins var insights = _algorithm.Securities.Keys.Select(x => GetInsight(x, direction, _algorithm.UtcTime)); var actualTargets = _algorithm.PortfolioConstruction.CreateTargets(_algorithm, insights.ToArray()); - + AssertTargets( expectedTargets, actualTargets); } @@ -391,7 +391,7 @@ public void GeneratesNormalTargetForZeroInsightConfidence(Language language) .Select(x => new PortfolioTarget(x.Key, (int)InsightDirection.Down * Math.Floor(amount / x.Value.Price))); var actualTargets = _algorithm.PortfolioConstruction.CreateTargets(_algorithm, insights).ToList(); - + AssertTargets(expectedTargets, actualTargets); } @@ -448,10 +448,10 @@ public void PortfolioBiasIsRespected(Language language, PortfolioBias bias) private Security GetSecurity(Symbol symbol) { - var config = SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc); + var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); return new Equity( symbol, - config, + exchangeHours, new Cash(Currencies.USD, 0, 1), SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, diff --git a/Tests/Brokerages/Paper/PaperBrokerageTests.cs b/Tests/Brokerages/Paper/PaperBrokerageTests.cs index 688fa15ef0ec..cff2c8626c31 100644 --- a/Tests/Brokerages/Paper/PaperBrokerageTests.cs +++ b/Tests/Brokerages/Paper/PaperBrokerageTests.cs @@ -90,7 +90,7 @@ public void AppliesDividendsOnce() var dataManager = new DataManager(feed, new UniverseSelection( algorithm, - new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio)), + new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio), algorithm: algorithm), dataPermissionManager, TestGlobals.DataProvider), algorithm, diff --git a/Tests/Common/Securities/SecurityPortfolioManagerTests.cs b/Tests/Common/Securities/SecurityPortfolioManagerTests.cs index 7e7dcb4a5a63..4e291114c9ae 100644 --- a/Tests/Common/Securities/SecurityPortfolioManagerTests.cs +++ b/Tests/Common/Securities/SecurityPortfolioManagerTests.cs @@ -228,7 +228,9 @@ public void ForexCashFills() securities.Add(usdJwbSecurity); securities.Add(mchUsdSecurity); - var securityService = new SecurityService(portfolio.CashBook, MarketHoursDatabase.FromDataFolder(), SymbolPropertiesDatabase.FromDataFolder(), dataManager.Algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(portfolio)); + var algorithm = new QCAlgorithm(); + algorithm.Securities = securities; + var securityService = new SecurityService(portfolio.CashBook, MarketHoursDatabase.FromDataFolder(), SymbolPropertiesDatabase.FromDataFolder(), dataManager.Algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(portfolio), algorithm: algorithm); portfolio.CashBook.EnsureCurrencyDataFeeds(securities, subscriptions, DefaultBrokerageModel.DefaultMarketMap, SecurityChanges.None, securityService); diff --git a/Tests/Common/Securities/SecurityServiceTests.cs b/Tests/Common/Securities/SecurityServiceTests.cs index 8b17a27f4452..9d70c5db9de6 100644 --- a/Tests/Common/Securities/SecurityServiceTests.cs +++ b/Tests/Common/Securities/SecurityServiceTests.cs @@ -264,7 +264,8 @@ public void AddPrimaryExchangeToSecurityObject() algorithm, new RegisteredSecurityDataTypesProvider(), new SecurityCacheProvider(algorithm.Portfolio), - mockedPrimaryExchangeProvider.Object); + mockedPrimaryExchangeProvider.Object, + algorithm: algorithm); var configs = _subscriptionManager.SubscriptionDataConfigService.Add(typeof(TradeBar), equitySymbol, Resolution.Second, false, false, false); diff --git a/Tests/Common/Util/ExtensionsTests.cs b/Tests/Common/Util/ExtensionsTests.cs index e0449b9ab968..2a42f9369ada 100644 --- a/Tests/Common/Util/ExtensionsTests.cs +++ b/Tests/Common/Util/ExtensionsTests.cs @@ -1745,7 +1745,8 @@ public void AppliesScalingToEquityTickQuotes() SymbolPropertiesDatabase.FromDataFolder(), algo, null, - null + null, + algorithm: algo ), new DataPermissionManager(), TestGlobals.DataProvider diff --git a/Tests/Engine/AlgorithmManagerTests.cs b/Tests/Engine/AlgorithmManagerTests.cs index b72c06b7c70e..c2d1d8beadf3 100644 --- a/Tests/Engine/AlgorithmManagerTests.cs +++ b/Tests/Engine/AlgorithmManagerTests.cs @@ -106,7 +106,8 @@ public void TestAlgorithmManagerSpeed() symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, - new SecurityCacheProvider(algorithm.Portfolio)), + new SecurityCacheProvider(algorithm.Portfolio), + algorithm: algorithm), dataPermissionManager, TestGlobals.DataProvider), algorithm, diff --git a/Tests/Engine/BrokerageTransactionHandlerTests/BrokerageTransactionHandlerTests.cs b/Tests/Engine/BrokerageTransactionHandlerTests/BrokerageTransactionHandlerTests.cs index 4bf3e01187de..64a3f6046e48 100644 --- a/Tests/Engine/BrokerageTransactionHandlerTests/BrokerageTransactionHandlerTests.cs +++ b/Tests/Engine/BrokerageTransactionHandlerTests/BrokerageTransactionHandlerTests.cs @@ -1624,7 +1624,7 @@ public void EmptyCashBalanceIsValid() var algorithm = new QCAlgorithm(); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); - var securityService = new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio)); + var securityService = new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio), algorithm: algorithm); algorithm.Securities.SetSecurityService(securityService); algorithm.SetLiveMode(true); algorithm.SetFinishedWarmingUp(); @@ -1672,7 +1672,7 @@ public void DoesNotLoopEndlesslyIfGetCashBalanceAlwaysThrows() var algorithm = new QCAlgorithm(); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); - var securityService = new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio)); + var securityService = new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio), algorithm: algorithm); algorithm.Securities.SetSecurityService(securityService); algorithm.SetLiveMode(true); algorithm.SetFinishedWarmingUp(); diff --git a/Tests/Engine/DataFeeds/DataManagerTests.cs b/Tests/Engine/DataFeeds/DataManagerTests.cs index a7a0e582c683..7ec844f007aa 100644 --- a/Tests/Engine/DataFeeds/DataManagerTests.cs +++ b/Tests/Engine/DataFeeds/DataManagerTests.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; +using System.Security.Cryptography; using NodaTime; using NUnit.Framework; using NUnit.Framework.Internal; @@ -48,7 +49,8 @@ public void SetUp() SymbolPropertiesDatabase.FromDataFolder(), _algorithm, new RegisteredSecurityDataTypesProvider(), - new SecurityCacheProvider(_algorithm.Portfolio)); + new SecurityCacheProvider(_algorithm.Portfolio), + algorithm: _algorithm); } [Test] diff --git a/Tests/Engine/DataFeeds/FileSystemDataFeedTests.cs b/Tests/Engine/DataFeeds/FileSystemDataFeedTests.cs index 7a878d3eebf2..469bbdf99cd3 100644 --- a/Tests/Engine/DataFeeds/FileSystemDataFeedTests.cs +++ b/Tests/Engine/DataFeeds/FileSystemDataFeedTests.cs @@ -49,7 +49,7 @@ public void TestsFileSystemDataFeedSpeed() var dataManager = new DataManager(feed, new UniverseSelection( algorithm, - new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio)), + new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio), algorithm: algorithm), dataPermissionManager, TestGlobals.DataProvider), algorithm, diff --git a/Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs b/Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs index 040eee5c743b..9a67eda84d12 100644 --- a/Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs +++ b/Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs @@ -395,7 +395,8 @@ private void SetupImpl(IDataQueueHandler dataQueueHandler, Synchronizer synchron SymbolPropertiesDatabase.FromDataFolder(), _algorithm, registeredTypesProvider, - new SecurityCacheProvider(_algorithm.Portfolio)); + new SecurityCacheProvider(_algorithm.Portfolio), + algorithm: _algorithm); var universeSelection = new UniverseSelection( _algorithm, securityService, diff --git a/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs b/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs index 12156533aaef..c0e0d59408ce 100644 --- a/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs +++ b/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs @@ -2357,7 +2357,8 @@ public void FastExitsDoNotThrowUnhandledExceptions() symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, - new SecurityCacheProvider(algorithm.Portfolio)); + new SecurityCacheProvider(algorithm.Portfolio), + algorithm: algorithm); algorithm.Securities.SetSecurityService(securityService); var dataPermissionManager = new DataPermissionManager(); var dataManager = new DataManager(_feed, @@ -2812,7 +2813,7 @@ private IDataFeed RunDataFeed(Resolution resolution = Resolution.Second, List( symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, - new SecurityCacheProvider(algorithm.Portfolio)); + new SecurityCacheProvider(algorithm.Portfolio), + algorithm: algorithm); algorithm.Securities.SetSecurityService(securityService); var dataPermissionManager = new DataPermissionManager(); var dataManager = new DataManager(_feed, @@ -3624,7 +3626,8 @@ public void HandlesFutureAndOptionChainUniverse(SecurityType securityType, int e symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, - new SecurityCacheProvider(algorithm.Portfolio)); + new SecurityCacheProvider(algorithm.Portfolio), + algorithm: algorithm); algorithm.Securities.SetSecurityService(securityService); var dataPermissionManager = new DataPermissionManager(); var dataManager = new DataManager(_feed, diff --git a/Tests/Engine/DataFeeds/SubscriptionSynchronizerTests.cs b/Tests/Engine/DataFeeds/SubscriptionSynchronizerTests.cs index 7d06db96fd07..f872cae6c472 100644 --- a/Tests/Engine/DataFeeds/SubscriptionSynchronizerTests.cs +++ b/Tests/Engine/DataFeeds/SubscriptionSynchronizerTests.cs @@ -59,7 +59,8 @@ private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm) symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, - new SecurityCacheProvider(algorithm.Portfolio)); + new SecurityCacheProvider(algorithm.Portfolio), + algorithm: algorithm); algorithm.Securities.SetSecurityService(securityService); var dataPermissionManager = new DataPermissionManager(); var dataManager = new DataManager(feed, diff --git a/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs b/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs index c62dbc440c4a..3bfcad648a6b 100644 --- a/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs +++ b/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs @@ -172,7 +172,7 @@ public void ExistingHoldingsAndOrdersResolution(Func> getHoldings, } [TestCaseSource(typeof(ExistingHoldingAndOrdersDataClass), nameof(ExistingHoldingAndOrdersDataClass.GetExistingHoldingsAndOrdersTestCaseData))] - public void ExistingHoldingsAndOrdersUniverseSettings(Func> getHoldings, Func> getOrders, bool expected) + public void ExistingHoldingsAndOrdersUniverseSettings(string testCase, Func> getHoldings, Func> getOrders, bool expected) { // Set our universe settings var hasCrypto = false; @@ -236,7 +236,7 @@ public void ExistingHoldingsAndOrdersUniverseSettings(Func> getHol } [TestCaseSource(typeof(ExistingHoldingAndOrdersDataClass),nameof(ExistingHoldingAndOrdersDataClass.GetExistingHoldingsAndOrdersTestCaseData))] - public void LoadsExistingHoldingsAndOrders(Func> getHoldings, Func> getOrders, bool expected) + public void LoadsExistingHoldingsAndOrders(string testCase, Func> getHoldings, Func> getOrders, bool expected) { var algorithm = new TestAlgorithm(); algorithm.SetHistoryProvider(new BrokerageTransactionHandlerTests.BrokerageTransactionHandlerTests.EmptyHistoryProvider()); @@ -255,7 +255,7 @@ public void LoadsExistingHoldingsAndOrders(Func> getHoldings, Func } [TestCaseSource(nameof(GetExistingHoldingsAndOrdersWithCustomDataTestCase))] - public void LoadsExistingHoldingsAndOrdersWithCustomData(Func> getHoldings, Func> getOrders) + public void LoadsExistingHoldingsAndOrdersWithCustomData(string testCase, Func> getHoldings, Func> getOrders) { var algorithm = new TestAlgorithm(); algorithm.AddData("BTC"); @@ -622,31 +622,39 @@ private void TestLoadExistingHoldingsAndOrders(IAlgorithm algorithm, Func>(() => new List { new Holding { Symbol = new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), Quantity = 1 }}), new Func>(() => new List())}, new object[] { + $"Test{_i++}", new Func>(() => new List { new Holding { Symbol = Symbols.SPY, Quantity = 1 }}), new Func>(() => new List())}, new object[] { + $"Test{_i++}", new Func>(() => new List()), new Func>(() => new List() { new LimitOrder(new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), 1, 1, DateTime.UtcNow)})}, new object[] { + $"Test{_i++}", new Func>(() => new List { new Holding { Symbol = new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), Quantity = 1 }}), new Func>(() => new List() { new LimitOrder(new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), 1, 1, DateTime.UtcNow)})}, new object[] { + $"Test{_i++}", new Func>(() => new List { new Holding { Symbol = new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), Quantity = 1 }, new Holding { Symbol = Symbols.SPY, Quantity = 1 }}), new Func>(() => new List() { new LimitOrder(new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), 1, 1, DateTime.UtcNow)})}, new object[] { + $"Test{_i++}", new Func>(() => new List { new Holding { Symbol = new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), Quantity = 1 }, new Holding { Symbol = Symbols.SPY, Quantity = 1 }}), @@ -661,11 +669,14 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData { get { + var i = 1; yield return new TestCaseData( + $"Test{i++}", new Func>(() => new List()), new Func>(() => new List()), true); yield return new TestCaseData( + $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.SPY, Quantity = 1 } @@ -676,6 +687,7 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( + $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.SPY_C_192_Feb19_2016, Quantity = 1 } @@ -686,6 +698,7 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( + $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.SPY, Quantity = 1 }, @@ -698,6 +711,7 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( + $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.SPY_C_192_Feb19_2016, Quantity = 1 }, @@ -710,6 +724,7 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( + $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.SPY_C_192_Feb19_2016, Quantity = 1 } @@ -720,6 +735,7 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( + $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.EURUSD, Quantity = 1 } @@ -730,6 +746,7 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( + $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.BTCUSD, Quantity = 1 } @@ -740,6 +757,7 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( + $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.Fut_SPY_Feb19_2016, Quantity = 1 } @@ -750,6 +768,7 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( + $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbol.Create("XYZ", SecurityType.Base, Market.USA), Quantity = 1 } @@ -762,6 +781,7 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), false); yield return new TestCaseData( + $"Test{i++}", new Func>(() => new List { new Holding { Symbol = new Symbol(SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), Quantity = 1 } @@ -772,10 +792,12 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), false); yield return new TestCaseData( + $"Test{i++}", new Func>(() => { throw new RegressionTestException(); }), new Func>(() => new List()), false); yield return new TestCaseData( + $"Test{i++}", new Func>(() => new List()), new Func>(() => { throw new RegressionTestException(); }), false); } diff --git a/Tests/Report/PortfolioLooperAlgorithmTests.cs b/Tests/Report/PortfolioLooperAlgorithmTests.cs index 816792640e54..ac9e01ce81b4 100644 --- a/Tests/Report/PortfolioLooperAlgorithmTests.cs +++ b/Tests/Report/PortfolioLooperAlgorithmTests.cs @@ -44,7 +44,8 @@ private PortfolioLooperAlgorithm CreateAlgorithm(IEnumerable orders, Algo symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, - new SecurityCacheProvider(algorithm.Portfolio)), + new SecurityCacheProvider(algorithm.Portfolio), + algorithm: algorithm), dataPermissionManager, TestGlobals.DataProvider), algorithm, @@ -59,7 +60,8 @@ private PortfolioLooperAlgorithm CreateAlgorithm(IEnumerable orders, Algo symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, - new SecurityCacheProvider(algorithm.Portfolio)); + new SecurityCacheProvider(algorithm.Portfolio), + algorithm: algorithm); // Initialize security services and other properties so that we // don't get null reference exceptions during our re-calculation diff --git a/Tests/ToolBox/RandomDataGenerator/RandomDataGeneratorTests.cs b/Tests/ToolBox/RandomDataGenerator/RandomDataGeneratorTests.cs index e5312c805520..f3df79120278 100644 --- a/Tests/ToolBox/RandomDataGenerator/RandomDataGeneratorTests.cs +++ b/Tests/ToolBox/RandomDataGenerator/RandomDataGeneratorTests.cs @@ -27,6 +27,7 @@ using QuantConnect.Securities.Option; using QuantConnect.Util; using static QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator; +using QuantConnect.Algorithm; namespace QuantConnect.Tests.ToolBox.RandomDataGenerator { @@ -121,6 +122,8 @@ public void RandomGeneratorProducesValuesBoundedForEquitiesWhenSplit(string star private static SecurityService GetSecurityService(RandomDataGeneratorSettings settings, SecurityManager securityManager) { + var algorithm = new QCAlgorithm(); + algorithm.Securities = securityManager; var securityService = new SecurityService( new CashBook(), MarketHoursDatabase.FromDataFolder(), @@ -144,7 +147,8 @@ private static SecurityService GetSecurityService(RandomDataGeneratorSettings se RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider( new SecurityPortfolioManager(securityManager, new SecurityTransactionManager(null, securityManager), new AlgorithmSettings())), - new MapFilePrimaryExchangeProvider(Composer.Instance.GetExportedValueByTypeName(Config.Get("map-file-provider", "LocalDiskMapFileProvider"))) + new MapFilePrimaryExchangeProvider(Composer.Instance.GetExportedValueByTypeName(Config.Get("map-file-provider", "LocalDiskMapFileProvider"))), + algorithm: algorithm ); securityManager.SetSecurityService(securityService); From 95de5116a8084f4ff3aa04305b1f6b085e5aeee9 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 28 Feb 2025 18:59:35 -0400 Subject: [PATCH 2/9] Minor fixes --- Common/Securities/SecurityService.cs | 11 +++++--- .../Setup/BrokerageSetupHandlerTests.cs | 28 ++----------------- 2 files changed, 10 insertions(+), 29 deletions(-) diff --git a/Common/Securities/SecurityService.cs b/Common/Securities/SecurityService.cs index 4f8d81cf2fdb..0ed9f0e6b909 100644 --- a/Common/Securities/SecurityService.cs +++ b/Common/Securities/SecurityService.cs @@ -71,12 +71,13 @@ private Security CreateSecurity(Symbol symbol, decimal leverage, bool addToSymbolCache, Security underlying, - bool initializeSecurity) + bool initializeSecurity, + bool reCreateSecurity) { var configList = new SubscriptionDataConfigList(symbol); configList.AddRange(subscriptionDataConfigList); - if (_algorithm != null && _algorithm.Securities.TryGetValue(symbol, out var existingSecurity)) + if (!reCreateSecurity && _algorithm != null && _algorithm.Securities.TryGetValue(symbol, out var existingSecurity)) { existingSecurity.AddData(configList); @@ -255,7 +256,8 @@ public Security CreateSecurity(Symbol symbol, bool addToSymbolCache = true, Security underlying = null) { - return CreateSecurity(symbol, subscriptionDataConfigList, leverage, addToSymbolCache, underlying, initializeSecurity: true); + return CreateSecurity(symbol, subscriptionDataConfigList, leverage, addToSymbolCache, underlying, + initializeSecurity: true, reCreateSecurity: false); } /// @@ -280,7 +282,8 @@ public Security CreateBenchmarkSecurity(Symbol symbol) leverage: 1, addToSymbolCache: false, underlying: null, - initializeSecurity: false); + initializeSecurity: false, + reCreateSecurity: true); } /// diff --git a/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs b/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs index 3bfcad648a6b..c62dbc440c4a 100644 --- a/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs +++ b/Tests/Engine/Setup/BrokerageSetupHandlerTests.cs @@ -172,7 +172,7 @@ public void ExistingHoldingsAndOrdersResolution(Func> getHoldings, } [TestCaseSource(typeof(ExistingHoldingAndOrdersDataClass), nameof(ExistingHoldingAndOrdersDataClass.GetExistingHoldingsAndOrdersTestCaseData))] - public void ExistingHoldingsAndOrdersUniverseSettings(string testCase, Func> getHoldings, Func> getOrders, bool expected) + public void ExistingHoldingsAndOrdersUniverseSettings(Func> getHoldings, Func> getOrders, bool expected) { // Set our universe settings var hasCrypto = false; @@ -236,7 +236,7 @@ public void ExistingHoldingsAndOrdersUniverseSettings(string testCase, Func> getHoldings, Func> getOrders, bool expected) + public void LoadsExistingHoldingsAndOrders(Func> getHoldings, Func> getOrders, bool expected) { var algorithm = new TestAlgorithm(); algorithm.SetHistoryProvider(new BrokerageTransactionHandlerTests.BrokerageTransactionHandlerTests.EmptyHistoryProvider()); @@ -255,7 +255,7 @@ public void LoadsExistingHoldingsAndOrders(string testCase, Func> } [TestCaseSource(nameof(GetExistingHoldingsAndOrdersWithCustomDataTestCase))] - public void LoadsExistingHoldingsAndOrdersWithCustomData(string testCase, Func> getHoldings, Func> getOrders) + public void LoadsExistingHoldingsAndOrdersWithCustomData(Func> getHoldings, Func> getOrders) { var algorithm = new TestAlgorithm(); algorithm.AddData("BTC"); @@ -622,39 +622,31 @@ private void TestLoadExistingHoldingsAndOrders(IAlgorithm algorithm, Func>(() => new List { new Holding { Symbol = new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), Quantity = 1 }}), new Func>(() => new List())}, new object[] { - $"Test{_i++}", new Func>(() => new List { new Holding { Symbol = Symbols.SPY, Quantity = 1 }}), new Func>(() => new List())}, new object[] { - $"Test{_i++}", new Func>(() => new List()), new Func>(() => new List() { new LimitOrder(new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), 1, 1, DateTime.UtcNow)})}, new object[] { - $"Test{_i++}", new Func>(() => new List { new Holding { Symbol = new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), Quantity = 1 }}), new Func>(() => new List() { new LimitOrder(new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), 1, 1, DateTime.UtcNow)})}, new object[] { - $"Test{_i++}", new Func>(() => new List { new Holding { Symbol = new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), Quantity = 1 }, new Holding { Symbol = Symbols.SPY, Quantity = 1 }}), new Func>(() => new List() { new LimitOrder(new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), 1, 1, DateTime.UtcNow)})}, new object[] { - $"Test{_i++}", new Func>(() => new List { new Holding { Symbol = new Symbol( SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), Quantity = 1 }, new Holding { Symbol = Symbols.SPY, Quantity = 1 }}), @@ -669,14 +661,11 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData { get { - var i = 1; yield return new TestCaseData( - $"Test{i++}", new Func>(() => new List()), new Func>(() => new List()), true); yield return new TestCaseData( - $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.SPY, Quantity = 1 } @@ -687,7 +676,6 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( - $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.SPY_C_192_Feb19_2016, Quantity = 1 } @@ -698,7 +686,6 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( - $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.SPY, Quantity = 1 }, @@ -711,7 +698,6 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( - $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.SPY_C_192_Feb19_2016, Quantity = 1 }, @@ -724,7 +710,6 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( - $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.SPY_C_192_Feb19_2016, Quantity = 1 } @@ -735,7 +720,6 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( - $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.EURUSD, Quantity = 1 } @@ -746,7 +730,6 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( - $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.BTCUSD, Quantity = 1 } @@ -757,7 +740,6 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( - $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbols.Fut_SPY_Feb19_2016, Quantity = 1 } @@ -768,7 +750,6 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), true); yield return new TestCaseData( - $"Test{i++}", new Func>(() => new List { new Holding { Symbol = Symbol.Create("XYZ", SecurityType.Base, Market.USA), Quantity = 1 } @@ -781,7 +762,6 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), false); yield return new TestCaseData( - $"Test{i++}", new Func>(() => new List { new Holding { Symbol = new Symbol(SecurityIdentifier.GenerateBase(typeof(Bitcoin), "BTC", Market.USA, false), "BTC"), Quantity = 1 } @@ -792,12 +772,10 @@ public static IEnumerable GetExistingHoldingsAndOrdersTestCaseData }), false); yield return new TestCaseData( - $"Test{i++}", new Func>(() => { throw new RegressionTestException(); }), new Func>(() => new List()), false); yield return new TestCaseData( - $"Test{i++}", new Func>(() => new List()), new Func>(() => { throw new RegressionTestException(); }), false); } From 7bc9ff45f09bbf8ba4047d972c135f1a596103e5 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 20 Mar 2025 18:33:39 -0400 Subject: [PATCH 3/9] Minor fixes --- ...sAlphaModelFrameworkRegressionAlgorithm.cs | 42 +++++++++---------- ...seCompositeDelistingRegressionAlgorithm.cs | 5 ++- ...seCompositeDelistingRegressionAlgorithm.py | 5 ++- ...istingRegressionAlgorithmNoAddEquityETF.py | 5 ++- ...sAlphaModelFrameworkRegressionAlgorithm.py | 2 +- 5 files changed, 31 insertions(+), 28 deletions(-) diff --git a/Algorithm.CSharp/HistoricalReturnsAlphaModelFrameworkRegressionAlgorithm.cs b/Algorithm.CSharp/HistoricalReturnsAlphaModelFrameworkRegressionAlgorithm.cs index 217e431a20f2..183ec9a5a947 100644 --- a/Algorithm.CSharp/HistoricalReturnsAlphaModelFrameworkRegressionAlgorithm.cs +++ b/Algorithm.CSharp/HistoricalReturnsAlphaModelFrameworkRegressionAlgorithm.cs @@ -32,7 +32,7 @@ public override void Initialize() public override void OnEndOfAlgorithm() { - const int expected = 78; + const int expected = 80; if (Insights.TotalCount != expected) { throw new RegressionTestException($"The total number of insights should be {expected}. Actual: {Insights.TotalCount}"); @@ -54,32 +54,32 @@ public override void OnEndOfAlgorithm() public override Dictionary ExpectedStatistics => new() { {"Total Orders", "69"}, - {"Average Win", "0.18%"}, + {"Average Win", "0.17%"}, {"Average Loss", "-0.15%"}, - {"Compounding Annual Return", "42.429%"}, + {"Compounding Annual Return", "46.315%"}, {"Drawdown", "0.900%"}, - {"Expectancy", "0.367"}, + {"Expectancy", "0.292"}, {"Start Equity", "100000"}, - {"End Equity", "102949.54"}, - {"Net Profit", "2.950%"}, - {"Sharpe Ratio", "5.164"}, - {"Sortino Ratio", "8.556"}, - {"Probabilistic Sharpe Ratio", "90.449%"}, - {"Loss Rate", "38%"}, - {"Win Rate", "62%"}, - {"Profit-Loss Ratio", "1.22"}, - {"Alpha", "0.306"}, - {"Beta", "-0.129"}, - {"Annual Standard Deviation", "0.055"}, + {"End Equity", "103177.61"}, + {"Net Profit", "3.178%"}, + {"Sharpe Ratio", "5.515"}, + {"Sortino Ratio", "9.319"}, + {"Probabilistic Sharpe Ratio", "91.936%"}, + {"Loss Rate", "40%"}, + {"Win Rate", "60%"}, + {"Profit-Loss Ratio", "1.15"}, + {"Alpha", "0.333"}, + {"Beta", "-0.138"}, + {"Annual Standard Deviation", "0.056"}, {"Annual Variance", "0.003"}, - {"Information Ratio", "1.181"}, - {"Tracking Error", "0.077"}, - {"Treynor Ratio", "-2.186"}, + {"Information Ratio", "1.488"}, + {"Tracking Error", "0.078"}, + {"Treynor Ratio", "-2.221"}, {"Total Fees", "$267.37"}, - {"Estimated Strategy Capacity", "$6000000.00"}, + {"Estimated Strategy Capacity", "$4000000.00"}, {"Lowest Capacity Asset", "AIG R735QTJ8XC9X"}, - {"Portfolio Turnover", "65.87%"}, - {"OrderListHash", "15bd0a959060b7ec5d77646e7e585f04"} + {"Portfolio Turnover", "65.85%"}, + {"OrderListHash", "a30e9c3a0143e548439788896d94a670"} }; } } diff --git a/Algorithm.CSharp/RegressionTests/Universes/ETFConstituentUniverseCompositeDelistingRegressionAlgorithm.cs b/Algorithm.CSharp/RegressionTests/Universes/ETFConstituentUniverseCompositeDelistingRegressionAlgorithm.cs index 4a812656bfbb..42df7832e752 100644 --- a/Algorithm.CSharp/RegressionTests/Universes/ETFConstituentUniverseCompositeDelistingRegressionAlgorithm.cs +++ b/Algorithm.CSharp/RegressionTests/Universes/ETFConstituentUniverseCompositeDelistingRegressionAlgorithm.cs @@ -116,8 +116,9 @@ public override void OnSecuritiesChanged(SecurityChanges changes) _universeAdded |= changes.AddedSecurities.Count == expectedChangesCount; } - // TODO: shouldn't be sending AAPL as a removed security since it was added by another universe - _universeRemoved |= changes.RemovedSecurities.Count == expectedChangesCount && + // AAPL was added by another universe, so it should be removed when this universe is removed, hence "- 1" + _universeRemoved |= changes.RemovedSecurities.Count == expectedChangesCount - 1 && + !changes.RemovedSecurities.Any(security => security.Symbol == _aapl) && UtcTime.Date >= _delistingDate && UtcTime.Date < EndDate; } diff --git a/Algorithm.Python/ETFConstituentUniverseCompositeDelistingRegressionAlgorithm.py b/Algorithm.Python/ETFConstituentUniverseCompositeDelistingRegressionAlgorithm.py index 915716e819ee..6826b23fbb40 100644 --- a/Algorithm.Python/ETFConstituentUniverseCompositeDelistingRegressionAlgorithm.py +++ b/Algorithm.Python/ETFConstituentUniverseCompositeDelistingRegressionAlgorithm.py @@ -66,9 +66,10 @@ def on_securities_changed(self, changes): # so AddedSecurities includes all ETF constituents (including APPL) plus GDVD self.universe_added = self.universe_added or len(changes.added_securities) == expected_changes_count - # TODO: shouldn't be sending AAPL as a removed security since it was added by another universe + # AAPL was added by another universe, so it should be removed when this universe is removed, hence "- 1" self.universe_removed = self.universe_removed or ( - len(changes.removed_securities) == expected_changes_count and + len(changes.removed_securities) == expected_changes_count - 1 and + not any(security.symbol == self.aapl for security in changes.removed_securities) and self.utc_time.date() >= self.delisting_date and self.utc_time.date() < self.end_date.date()) diff --git a/Algorithm.Python/ETFConstituentUniverseCompositeDelistingRegressionAlgorithmNoAddEquityETF.py b/Algorithm.Python/ETFConstituentUniverseCompositeDelistingRegressionAlgorithmNoAddEquityETF.py index 55ef4eec3926..deec2ec4fa73 100644 --- a/Algorithm.Python/ETFConstituentUniverseCompositeDelistingRegressionAlgorithmNoAddEquityETF.py +++ b/Algorithm.Python/ETFConstituentUniverseCompositeDelistingRegressionAlgorithmNoAddEquityETF.py @@ -61,9 +61,10 @@ def on_securities_changed(self, changes): if self.universe_selection_done: self.universe_added = self.universe_added or len(changes.added_securities) == self.universe_symbol_count - # TODO: shouldn't be sending AAPL as a removed security since it was added by another universe + # AAPL was added by another universe, so it should be removed when this universe is removed, hence "- 1" self.universe_removed = self.universe_removed or ( - len(changes.removed_securities) == self.universe_symbol_count and + len(changes.removed_securities) == self.universe_symbol_count - 1 and + not any(security.symbol == self.aapl for security in changes.removed_securities) and self.utc_time.date() >= self.delisting_date and self.utc_time.date() < self.end_date.date()) diff --git a/Algorithm.Python/HistoricalReturnsAlphaModelFrameworkRegressionAlgorithm.py b/Algorithm.Python/HistoricalReturnsAlphaModelFrameworkRegressionAlgorithm.py index 2104f75d5fa0..84b684a1e6b7 100644 --- a/Algorithm.Python/HistoricalReturnsAlphaModelFrameworkRegressionAlgorithm.py +++ b/Algorithm.Python/HistoricalReturnsAlphaModelFrameworkRegressionAlgorithm.py @@ -25,6 +25,6 @@ def initialize(self): self.set_alpha(HistoricalReturnsAlphaModel()) def on_end_of_algorithm(self): - expected = 78 + expected = 80 if self.insights.total_count != expected: raise Exception(f"The total number of insights should be {expected}. Actual: {self.insights.total_count}") From 4219116bf27f6159597aba0f00d95ff50cee6c82 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 21 Mar 2025 11:09:47 -0400 Subject: [PATCH 4/9] More regression algorithms fixes --- ...ChainUniverseRemovalRegressionAlgorithm.cs | 6 +-- ...airsTradingAlphaModelFrameworkAlgorithm.cs | 49 ++++++++++--------- ...iAlphaModelFrameworkRegressionAlgorithm.cs | 7 +-- ...airsTradingAlphaModelFrameworkAlgorithm.py | 5 +- ...iAlphaModelFrameworkRegressionAlgorithm.py | 5 +- Algorithm/QCAlgorithm.Universe.cs | 6 +-- Common/Extensions.cs | 20 +++++--- 7 files changed, 52 insertions(+), 46 deletions(-) diff --git a/Algorithm.CSharp/OptionChainUniverseRemovalRegressionAlgorithm.cs b/Algorithm.CSharp/OptionChainUniverseRemovalRegressionAlgorithm.cs index 6ad159c66ec5..9a81573a6646 100644 --- a/Algorithm.CSharp/OptionChainUniverseRemovalRegressionAlgorithm.cs +++ b/Algorithm.CSharp/OptionChainUniverseRemovalRegressionAlgorithm.cs @@ -131,7 +131,7 @@ public override void OnSecuritiesChanged(SecurityChanges changes) } } } - // We expect the equity to get Removed + // De equity is deselected, but it should not be removed since the options universe still selects it as the underlying else if (Time.Day == 7) { if (Time.Hour != 0) @@ -141,12 +141,12 @@ public override void OnSecuritiesChanged(SecurityChanges changes) // Options can be selected/deselected on this day, but the equity should be removed - if (changes.RemovedSecurities.Count == 0 || !changes.RemovedSecurities.Any(x => x.Symbol == _aapl)) + if (changes.RemovedSecurities.Count == 0 || changes.RemovedSecurities.Any(x => x.Symbol == _aapl)) { throw new RegressionTestException($"Unexpected SecurityChanges: {changes}"); } } - // We expect the options to get Removed, happens in the next loop after removing the equity + // We expect the options to get Removed, as well as the underlying, happens in the next loop after removing the equity else if (Time.Day == 9) { if (Time.Hour != 0) diff --git a/Algorithm.CSharp/PearsonCorrelationPairsTradingAlphaModelFrameworkAlgorithm.cs b/Algorithm.CSharp/PearsonCorrelationPairsTradingAlphaModelFrameworkAlgorithm.cs index 0c20fa6608e7..5bf605cb57e2 100644 --- a/Algorithm.CSharp/PearsonCorrelationPairsTradingAlphaModelFrameworkAlgorithm.cs +++ b/Algorithm.CSharp/PearsonCorrelationPairsTradingAlphaModelFrameworkAlgorithm.cs @@ -60,9 +60,10 @@ public override void OnEndOfAlgorithm() { // We have removed all securities from the universe. The Alpha Model should remove the consolidator var consolidatorCount = SubscriptionManager.Subscriptions.Sum(s => s.Consolidators.Count); - if (consolidatorCount > 0) + // Expect 2 consolidators for SPY and AIG, which where manually added at the start + if (consolidatorCount != 2) { - throw new RegressionTestException($"The number of consolidator is should be zero. Actual: {consolidatorCount}"); + throw new RegressionTestException($"The number of consolidator is should be 2. Actual: {consolidatorCount}"); } } @@ -84,7 +85,7 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 1008; + public int AlgorithmHistoryDataPoints => 1512; /// /// Final status of the algorithm @@ -97,32 +98,32 @@ public override void OnEndOfAlgorithm() public Dictionary ExpectedStatistics => new Dictionary { {"Total Orders", "6"}, - {"Average Win", "0.99%"}, - {"Average Loss", "-0.84%"}, - {"Compounding Annual Return", "24.021%"}, - {"Drawdown", "0.800%"}, - {"Expectancy", "0.089"}, + {"Average Win", "0.82%"}, + {"Average Loss", "-0.39%"}, + {"Compounding Annual Return", "43.360%"}, + {"Drawdown", "0.700%"}, + {"Expectancy", "0.546"}, {"Start Equity", "100000"}, - {"End Equity", "100295.35"}, - {"Net Profit", "0.295%"}, - {"Sharpe Ratio", "4.205"}, + {"End Equity", "100494.63"}, + {"Net Profit", "0.495%"}, + {"Sharpe Ratio", "10.477"}, {"Sortino Ratio", "0"}, - {"Probabilistic Sharpe Ratio", "61.706%"}, + {"Probabilistic Sharpe Ratio", "85.907%"}, {"Loss Rate", "50%"}, {"Win Rate", "50%"}, - {"Profit-Loss Ratio", "1.18"}, - {"Alpha", "0.08"}, - {"Beta", "0.06"}, - {"Annual Standard Deviation", "0.047"}, - {"Annual Variance", "0.002"}, - {"Information Ratio", "-8.305"}, - {"Tracking Error", "0.214"}, - {"Treynor Ratio", "3.313"}, - {"Total Fees", "$31.60"}, - {"Estimated Strategy Capacity", "$3200000.00"}, + {"Profit-Loss Ratio", "2.09"}, + {"Alpha", "0.227"}, + {"Beta", "0.066"}, + {"Annual Standard Deviation", "0.034"}, + {"Annual Variance", "0.001"}, + {"Information Ratio", "-7.7"}, + {"Tracking Error", "0.21"}, + {"Treynor Ratio", "5.41"}, + {"Total Fees", "$25.78"}, + {"Estimated Strategy Capacity", "$3300000.00"}, {"Lowest Capacity Asset", "AIG R735QTJ8XC9X"}, - {"Portfolio Turnover", "80.47%"}, - {"OrderListHash", "476d54ac7295563a79add3a80310a0a8"} + {"Portfolio Turnover", "60.55%"}, + {"OrderListHash", "0eb251234d0fa772130bb341457091b4"} }; } } diff --git a/Algorithm.CSharp/RsiAlphaModelFrameworkRegressionAlgorithm.cs b/Algorithm.CSharp/RsiAlphaModelFrameworkRegressionAlgorithm.cs index b2db834c8a4d..d69c5e181215 100644 --- a/Algorithm.CSharp/RsiAlphaModelFrameworkRegressionAlgorithm.cs +++ b/Algorithm.CSharp/RsiAlphaModelFrameworkRegressionAlgorithm.cs @@ -35,9 +35,10 @@ public override void OnEndOfAlgorithm() { // We have removed all securities from the universe. The Alpha Model should remove the consolidator var consolidatorCount = SubscriptionManager.Subscriptions.Sum(s => s.Consolidators.Count); - if (consolidatorCount > 0) - { - throw new RegressionTestException($"The number of consolidators should be zero. Actual: {consolidatorCount}"); + // Expect 2 consolidators for AAPL and AIG, which where manually added at the start + if (consolidatorCount != 2) + { + throw new RegressionTestException($"The number of consolidators should be 2. Actual: {consolidatorCount}"); } } diff --git a/Algorithm.Python/PearsonCorrelationPairsTradingAlphaModelFrameworkAlgorithm.py b/Algorithm.Python/PearsonCorrelationPairsTradingAlphaModelFrameworkAlgorithm.py index e3200d7d2eef..b5b1529d7922 100644 --- a/Algorithm.Python/PearsonCorrelationPairsTradingAlphaModelFrameworkAlgorithm.py +++ b/Algorithm.Python/PearsonCorrelationPairsTradingAlphaModelFrameworkAlgorithm.py @@ -49,5 +49,6 @@ def initialize(self): def on_end_of_algorithm(self) -> None: # We have removed all securities from the universe. The Alpha Model should remove the consolidator consolidator_count = sum(s.consolidators.count for s in self.subscription_manager.subscriptions) - if consolidator_count > 0: - raise Exception(f"The number of consolidator should be zero. Actual: {consolidator_count}") + # Expect 2 consolidators for SPY and AIG, which where manually added at the start + if consolidator_count != 2: + raise Exception(f"The number of consolidator should be 2. Actual: {consolidator_count}") diff --git a/Algorithm.Python/RsiAlphaModelFrameworkRegressionAlgorithm.py b/Algorithm.Python/RsiAlphaModelFrameworkRegressionAlgorithm.py index 4c04cc4fe267..86123c214788 100644 --- a/Algorithm.Python/RsiAlphaModelFrameworkRegressionAlgorithm.py +++ b/Algorithm.Python/RsiAlphaModelFrameworkRegressionAlgorithm.py @@ -27,5 +27,6 @@ def initialize(self): def on_end_of_algorithm(self): # We have removed all securities from the universe. The Alpha Model should remove the consolidator consolidator_count = sum([s.consolidators.count for s in self.subscription_manager.subscriptions]) - if consolidator_count > 0: - raise Exception(f"The number of consolidators should be zero. Actual: {consolidator_count}") + # Expect 2 consolidators for AAPL and AIG, which where manually added at the start + if consolidator_count != 2: + raise Exception(f"The number of consolidators should be 2. Actual: {consolidator_count}") diff --git a/Algorithm/QCAlgorithm.Universe.cs b/Algorithm/QCAlgorithm.Universe.cs index 30455190d504..69fb926a4a42 100644 --- a/Algorithm/QCAlgorithm.Universe.cs +++ b/Algorithm/QCAlgorithm.Universe.cs @@ -591,11 +591,7 @@ private Security AddToUserDefinedUniverse( } else { - var isTradable = security.IsTradable; - // We will reuse existing so we return it to the user. - // We will use the IsTradable flag of the new security, since existing could of been set to false when removed - security = existingSecurity; - security.IsTradable = isTradable; + security.MakeTradable(); } } else diff --git a/Common/Extensions.cs b/Common/Extensions.cs index f0f20c12cfc9..06ce63fee2e0 100644 --- a/Common/Extensions.cs +++ b/Common/Extensions.cs @@ -3894,13 +3894,7 @@ public static void ProcessSecurityChanges(this IAlgorithm algorithm, SecurityCha { // uses TryAdd, so don't need to worry about duplicates here algorithm.Securities.Add(security); - - if (security.Type == SecurityType.Index && !(security as Securities.Index.Index).ManualSetIsTradable) - { - continue; - } - - security.IsTradable = true; + security.MakeTradable(); } var activeSecurities = algorithm.UniverseManager.ActiveSecurities; @@ -3913,6 +3907,18 @@ public static void ProcessSecurityChanges(this IAlgorithm algorithm, SecurityCha } } + /// + /// Helper method to set the property to true + /// for the given security when possible + /// + public static void MakeTradable(this Security security) + { + if (security.Type != SecurityType.Index || (security as Securities.Index.Index).ManualSetIsTradable) + { + security.IsTradable = true; + } + } + /// /// Helper method to set an algorithm runtime exception in a normalized fashion /// From b3a554b20dcc3bd482a276ee2929af01a49ddd6d Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 21 Mar 2025 11:51:39 -0400 Subject: [PATCH 5/9] Minor fixes --- Common/Extensions.cs | 2 +- Common/Securities/Index/Index.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Common/Extensions.cs b/Common/Extensions.cs index 06ce63fee2e0..914e64c35562 100644 --- a/Common/Extensions.cs +++ b/Common/Extensions.cs @@ -3915,7 +3915,7 @@ public static void MakeTradable(this Security security) { if (security.Type != SecurityType.Index || (security as Securities.Index.Index).ManualSetIsTradable) { - security.IsTradable = true; + security.IsTradable = security.Subscriptions.Any(config => !config.IsInternalFeed); } } diff --git a/Common/Securities/Index/Index.cs b/Common/Securities/Index/Index.cs index d00d354e3512..a962e24da008 100644 --- a/Common/Securities/Index/Index.cs +++ b/Common/Securities/Index/Index.cs @@ -35,7 +35,7 @@ public override bool IsTradable { get => _isTradable; set { - if (value) ManualSetIsTradable = true; + ManualSetIsTradable = value; _isTradable = value; } } From ba132b910d6c5cae36e169212fb99f43a4838833 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 21 Mar 2025 14:57:43 -0400 Subject: [PATCH 6/9] Minor fix --- Common/Extensions.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Common/Extensions.cs b/Common/Extensions.cs index 914e64c35562..7dfa63eb5e7b 100644 --- a/Common/Extensions.cs +++ b/Common/Extensions.cs @@ -3894,7 +3894,7 @@ public static void ProcessSecurityChanges(this IAlgorithm algorithm, SecurityCha { // uses TryAdd, so don't need to worry about duplicates here algorithm.Securities.Add(security); - security.MakeTradable(); + security.MakeTradable(forceInternal: true); } var activeSecurities = algorithm.UniverseManager.ActiveSecurities; @@ -3911,11 +3911,11 @@ public static void ProcessSecurityChanges(this IAlgorithm algorithm, SecurityCha /// Helper method to set the property to true /// for the given security when possible /// - public static void MakeTradable(this Security security) + public static void MakeTradable(this Security security, bool forceInternal = false) { if (security.Type != SecurityType.Index || (security as Securities.Index.Index).ManualSetIsTradable) { - security.IsTradable = security.Subscriptions.Any(config => !config.IsInternalFeed); + security.IsTradable = forceInternal || security.Subscriptions.Any(config => !config.IsInternalFeed); } } From 78e19b3781758c21ce36203b815432ef18e10a9b Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 21 Mar 2025 18:13:38 -0400 Subject: [PATCH 7/9] Add regression algorithms --- ...tyReAdditionBehaviorRegressionAlgorithm.cs | 23 +- ...onReAdditionBehaviorRegressionAlgorithm.cs | 189 ++++++++++++++++ ...onReAdditionBehaviorRegressionAlgorithm.cs | 208 ++++++++++++++++++ Common/Securities/SecurityService.cs | 1 + 4 files changed, 410 insertions(+), 11 deletions(-) create mode 100644 Algorithm.CSharp/SecurityManuallyAddedOptionReAdditionBehaviorRegressionAlgorithm.cs create mode 100644 Algorithm.CSharp/SecurityOptionReAdditionBehaviorRegressionAlgorithm.cs diff --git a/Algorithm.CSharp/SecurityEquityReAdditionBehaviorRegressionAlgorithm.cs b/Algorithm.CSharp/SecurityEquityReAdditionBehaviorRegressionAlgorithm.cs index 0f26e3ca65af..dd4cb937f759 100644 --- a/Algorithm.CSharp/SecurityEquityReAdditionBehaviorRegressionAlgorithm.cs +++ b/Algorithm.CSharp/SecurityEquityReAdditionBehaviorRegressionAlgorithm.cs @@ -35,16 +35,14 @@ public override void Initialize() SetStartDate(2013, 10, 04); SetEndDate(2013, 10, 30); + // TODO: Assert that securities are initialized on re-addition var seeder = new FuncSecuritySeeder((security) => { Debug($"[{Time}] Seeding {security.Symbol}"); - return GetLastKnownPrices(security); }); - SetSecurityInitializer(security => - { - seeder.SeedSecurity(security); - }); + + SetSecurityInitializer(security => seeder.SeedSecurity(security)); _equity = AddEquity("SPY"); @@ -112,7 +110,10 @@ public override void OnSecuritiesChanged(SecurityChanges changes) public override void OnEndOfAlgorithm() { - + if (_tradableDates.Count > 0) + { + throw new RegressionTestException($"Expected no more tradable dates. Still have {_tradableDates.Count}"); + } } /// @@ -128,17 +129,17 @@ public override void OnEndOfAlgorithm() /// /// Data Points count of all timeslices of algorithm /// - public long DataPoints => 774; + public long DataPoints => 5581; /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 10; + public int AlgorithmHistoryDataPoints => 4040; /// /// Final status of the algorithm /// - public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.RuntimeError; + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; /// /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm @@ -164,8 +165,8 @@ public override void OnEndOfAlgorithm() {"Beta", "0"}, {"Annual Standard Deviation", "0"}, {"Annual Variance", "0"}, - {"Information Ratio", "0"}, - {"Tracking Error", "0"}, + {"Information Ratio", "-4.884"}, + {"Tracking Error", "0.108"}, {"Treynor Ratio", "0"}, {"Total Fees", "$0.00"}, {"Estimated Strategy Capacity", "$0"}, diff --git a/Algorithm.CSharp/SecurityManuallyAddedOptionReAdditionBehaviorRegressionAlgorithm.cs b/Algorithm.CSharp/SecurityManuallyAddedOptionReAdditionBehaviorRegressionAlgorithm.cs new file mode 100644 index 000000000000..4b51a1369f74 --- /dev/null +++ b/Algorithm.CSharp/SecurityManuallyAddedOptionReAdditionBehaviorRegressionAlgorithm.cs @@ -0,0 +1,189 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Interfaces; +using QuantConnect.Securities; +using QuantConnect.Securities.Option; + +namespace QuantConnect.Algorithm.CSharp +{ + public class SecurityManuallyAddedOptionReAdditionBehaviorRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + private Option _manuallyAddedContract; + + private bool _securityWasRemoved; + + private Queue _tradableDates; + + public override void Initialize() + { + SetStartDate(2014, 06, 04); + SetEndDate(2014, 06, 20); + SetCash(100000); + + // TODO: Assert that securities are initialized on re-addition + var seeder = new FuncSecuritySeeder((security) => + { + Debug($"[{Time}] Seeding {security.Symbol}"); + return GetLastKnownPrices(security); + }); + + SetSecurityInitializer(security => seeder.SeedSecurity(security)); + + var equitySymbol = QuantConnect.Symbol.Create("AAPL", SecurityType.Equity, Market.USA); + var optionContractSymbol = QuantConnect.Symbol.CreateOption(equitySymbol, Market.USA, + OptionStyle.American, OptionRight.Call, 342.9m, new DateTime(2014, 07, 19)); + + _manuallyAddedContract = AddOptionContract(optionContractSymbol, Resolution.Daily); + + _tradableDates = new(QuantConnect.Time.EachTradeableDay(_manuallyAddedContract.Exchange.Hours, StartDate, EndDate)); + } + + public override void OnEndOfDay(Symbol symbol) + { + if (symbol != _manuallyAddedContract.Symbol) + { + return; + } + + var currentTradableDate = _tradableDates.Dequeue(); + if (currentTradableDate != Time.Date) + { + throw new RegressionTestException($"Expected the current tradable date to be {Time.Date}. Got {currentTradableDate}"); + } + + // Remove the security every day + Debug($"[{Time}] Removing the option contract"); + _securityWasRemoved = RemoveSecurity(symbol); + + if (!_securityWasRemoved) + { + throw new RegressionTestException($"Expected the option contract to be removed"); + } + } + + public override void OnSecuritiesChanged(SecurityChanges changes) + { + if (_securityWasRemoved) + { + if (changes.AddedSecurities.Count > 0) + { + throw new RegressionTestException($"Expected no securities to be added. Got {changes.AddedSecurities.Count}"); + } + + if (!changes.RemovedSecurities.Contains(_manuallyAddedContract)) + { + throw new RegressionTestException($"Expected the option contract to be removed. Got {changes.RemovedSecurities.Count}"); + } + + _securityWasRemoved = false; + + if (Time.Date >= EndDate.Date) + { + return; + } + + // Add the security back + Debug($"[{Time}] Re-adding the option contract"); + var reAddedContract = AddOptionContract(_manuallyAddedContract.Symbol, Resolution.Daily); + + if (reAddedContract != _manuallyAddedContract) + { + throw new RegressionTestException($"Expected the re-added option contract to be the same as the original option contract"); + } + + if (!reAddedContract.IsTradable) + { + throw new RegressionTestException($"Expected the re-added option contract to be tradable"); + } + } + else if (!changes.AddedSecurities.Contains(_manuallyAddedContract)) + { + throw new RegressionTestException($"Expected the option contract to be added back"); + } + } + + public override void OnEndOfAlgorithm() + { + if (_tradableDates.Count > 0) + { + throw new RegressionTestException($"Expected no more tradable dates. Still have {_tradableDates.Count}"); + } + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 115; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 5; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "0"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100000"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "-6.27"}, + {"Tracking Error", "0.056"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "$0.00"}, + {"Estimated Strategy Capacity", "$0"}, + {"Lowest Capacity Asset", ""}, + {"Portfolio Turnover", "0%"}, + {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"} + }; + } +} diff --git a/Algorithm.CSharp/SecurityOptionReAdditionBehaviorRegressionAlgorithm.cs b/Algorithm.CSharp/SecurityOptionReAdditionBehaviorRegressionAlgorithm.cs new file mode 100644 index 000000000000..4db2a7d4bbf1 --- /dev/null +++ b/Algorithm.CSharp/SecurityOptionReAdditionBehaviorRegressionAlgorithm.cs @@ -0,0 +1,208 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Interfaces; +using QuantConnect.Securities; +using QuantConnect.Securities.Option; + +namespace QuantConnect.Algorithm.CSharp +{ + public class SecurityOptionReAdditionBehaviorRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + private List _contractsToSelect; + private HashSet