Python fundamental history request fix (QuantConnect#7905)

- Fix python fundamental pandas history request, where properties
  wouldn't change due to cache. Expanding regression test to assert
  issue

Algorithm.CSharp/FundamentalRegressionAlgorithm.cs

@@ -69,10 +69,10 @@ public override void Initialize()
             }
 
             // Request historical fundamental data for symbols
-            var history = History<Fundamental>(Securities.Keys, new TimeSpan(1, 0, 0, 0)).ToList();
-            if(history.Count != 1)
+            var history = History<Fundamental>(Securities.Keys, new TimeSpan(2, 0, 0, 0)).ToList();
+            if(history.Count != 2)
             {
-                throw new Exception($"Unexpected {nameof(Fundamental)} history count {history.Count}! Expected 1");
+                throw new Exception($"Unexpected {nameof(Fundamental)} history count {history.Count}! Expected 2");
             }
 
             if (history[0].Values.Count != 2)
@@ -195,7 +195,7 @@ public override void OnSecuritiesChanged(SecurityChanges changes)
         /// <summary>
         /// Data Points count of the algorithm history
         /// </summary>
-        public virtual int AlgorithmHistoryDataPoints => 14;
+        public virtual int AlgorithmHistoryDataPoints => 16;
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm

Algorithm.Python/FundamentalRegressionAlgorithm.py

@@ -49,14 +49,18 @@ def Initialize(self):
             raise ValueError(f"Unexpected Fundamental count {len(fundamentals)}! Expected 2")
 
         # Request historical fundamental data for symbols
-        history = self.History(Fundamental, TimeSpan(1, 0, 0, 0))
-        if len(history) != 2:
-            raise ValueError(f"Unexpected Fundamental history count {len(history)}! Expected 2")
+        history = self.History(Fundamental, TimeSpan(2, 0, 0, 0))
+        if len(history) != 4:
+            raise ValueError(f"Unexpected Fundamental history count {len(history)}! Expected 4")
 
         for ticker in [ "AAPL", "SPY" ]:
             data = history.loc[ticker]
             if data["value"][0] == 0:
                 raise ValueError(f"Unexpected {data} fundamental data")
+        if Object.ReferenceEquals(data.earningreports.iloc[0], data.earningreports.iloc[1]):
+            raise ValueError(f"Unexpected fundamental data instance duplication")
+        if data.earningreports.iloc[0]._timeProvider.GetUtcNow() == data.earningreports.iloc[1]._timeProvider.GetUtcNow():
+            raise ValueError(f"Unexpected fundamental data instance duplication")
 
         self.AssertFundamentalUniverseData()
 

Common/Data/Fundamental/FundamentalTimeDependentProperty.cs

@@ -0,0 +1,50 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2023 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using Python.Runtime;
+
+namespace QuantConnect.Data.Fundamental
+{
+    /// <summary>
+    /// Simple base class shared by top layer fundamental properties which depend on a time provider
+    /// </summary>
+    public abstract class FundamentalTimeDependentProperty : ReusuableCLRObject
+    {
+        /// <summary>
+        /// The time provider instance to use
+        /// </summary>
+        protected ITimeProvider _timeProvider { get; }
+
+        /// <summary>
+        /// The SID instance to use
+        /// </summary>
+        protected SecurityIdentifier _securityIdentifier { get; }
+
+        /// <summary>
+        /// Creates a new instance for the given time and security
+        /// </summary>
+        public FundamentalTimeDependentProperty(ITimeProvider timeProvider, SecurityIdentifier securityIdentifier)
+        {
+            _timeProvider = timeProvider;
+            _securityIdentifier = securityIdentifier;
+        }
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public abstract FundamentalTimeDependentProperty Clone(ITimeProvider timeProvider);
+    }
+}

Common/Data/Fundamental/Generated/AssetClassification.cs

@@ -26,7 +26,7 @@ namespace QuantConnect.Data.Fundamental
     /// <summary>
     /// Definition of the AssetClassification class
     /// </summary>
-    public class AssetClassification : ReusuableCLRObject
+    public class AssetClassification : FundamentalTimeDependentProperty
     {
         /// <summary>
         /// The purpose of the Stock Types is to group companies according to the underlying fundamentals of their business. They answer the question: If I buy this stock, what kind of company am I buying? Unlike the style box, the emphasis with the Stock Types is on income statement, balance sheet, and cash-flow data-not price data or valuation multiples. We focus on the company, not the stock. Morningstar calculates this figure in-house on a monthly basis.
@@ -181,16 +181,20 @@ public class AssetClassification : ReusuableCLRObject
         [JsonProperty("3016")]
         public int CANNAICS => FundamentalService.Get<int>(_timeProvider.GetUtcNow(), _securityIdentifier, FundamentalProperty.AssetClassification_CANNAICS);
 
-        private readonly ITimeProvider _timeProvider;
-        private readonly SecurityIdentifier _securityIdentifier;
-
         /// <summary>
         /// Creates a new instance for the given time and security
         /// </summary>
         public AssetClassification(ITimeProvider timeProvider, SecurityIdentifier securityIdentifier)
+            : base(timeProvider, securityIdentifier)
+        {
+        }
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override FundamentalTimeDependentProperty Clone(ITimeProvider timeProvider)
         {
-            _timeProvider = timeProvider;
-            _securityIdentifier = securityIdentifier;
+            return new AssetClassification(timeProvider, _securityIdentifier);
         }
     }
 }

Common/Data/Fundamental/Generated/CompanyProfile.cs

@@ -26,7 +26,7 @@ namespace QuantConnect.Data.Fundamental
     /// <summary>
     /// Definition of the CompanyProfile class
     /// </summary>
-    public class CompanyProfile : ReusuableCLRObject
+    public class CompanyProfile : FundamentalTimeDependentProperty
     {
         /// <summary>
         /// The headquarter address as given in the latest report
@@ -316,16 +316,20 @@ public class CompanyProfile : ReusuableCLRObject
         [JsonProperty("40010")]
         public string ReasonofSharesChange => FundamentalService.Get<string>(_timeProvider.GetUtcNow(), _securityIdentifier, FundamentalProperty.CompanyProfile_ReasonofSharesChange);
 
-        private readonly ITimeProvider _timeProvider;
-        private readonly SecurityIdentifier _securityIdentifier;
-
         /// <summary>
         /// Creates a new instance for the given time and security
         /// </summary>
         public CompanyProfile(ITimeProvider timeProvider, SecurityIdentifier securityIdentifier)
+            : base(timeProvider, securityIdentifier)
+        {
+        }
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override FundamentalTimeDependentProperty Clone(ITimeProvider timeProvider)
         {
-            _timeProvider = timeProvider;
-            _securityIdentifier = securityIdentifier;
+            return new CompanyProfile(timeProvider, _securityIdentifier);
         }
     }
 }

Common/Data/Fundamental/Generated/CompanyReference.cs

@@ -26,7 +26,7 @@ namespace QuantConnect.Data.Fundamental
     /// <summary>
     /// Definition of the CompanyReference class
     /// </summary>
-    public class CompanyReference : ReusuableCLRObject
+    public class CompanyReference : FundamentalTimeDependentProperty
     {
         /// <summary>
         /// 10-digit unique and unchanging Morningstar identifier assigned to every company.
@@ -253,16 +253,20 @@ public class CompanyReference : ReusuableCLRObject
         [JsonProperty("27")]
         public DateTime ExpectedFiscalYearEnd => FundamentalService.Get<DateTime>(_timeProvider.GetUtcNow(), _securityIdentifier, FundamentalProperty.CompanyReference_ExpectedFiscalYearEnd);
 
-        private readonly ITimeProvider _timeProvider;
-        private readonly SecurityIdentifier _securityIdentifier;
-
         /// <summary>
         /// Creates a new instance for the given time and security
         /// </summary>
         public CompanyReference(ITimeProvider timeProvider, SecurityIdentifier securityIdentifier)
+            : base(timeProvider, securityIdentifier)
+        {
+        }
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override FundamentalTimeDependentProperty Clone(ITimeProvider timeProvider)
         {
-            _timeProvider = timeProvider;
-            _securityIdentifier = securityIdentifier;
+            return new CompanyReference(timeProvider, _securityIdentifier);
         }
     }
 }

Common/Data/Fundamental/Generated/EarningRatios.cs

@@ -26,7 +26,7 @@ namespace QuantConnect.Data.Fundamental
     /// <summary>
     /// Definition of the EarningRatios class
     /// </summary>
-    public class EarningRatios : ReusuableCLRObject
+    public class EarningRatios : FundamentalTimeDependentProperty
     {
         /// <summary>
         /// The growth in the company's diluted earnings per share (EPS) on a percentage basis. Morningstar calculates the annualized growth percentage based on the underlying diluted EPS reported in the Income Statement within the company filings or reports.
@@ -118,16 +118,20 @@ public class EarningRatios : ReusuableCLRObject
         public NormalizedBasicEPSGrowth NormalizedBasicEPSGrowth => _normalizedBasicEPSGrowth ??= new(_timeProvider, _securityIdentifier);
         private NormalizedBasicEPSGrowth _normalizedBasicEPSGrowth;
 
-        private readonly ITimeProvider _timeProvider;
-        private readonly SecurityIdentifier _securityIdentifier;
-
         /// <summary>
         /// Creates a new instance for the given time and security
         /// </summary>
         public EarningRatios(ITimeProvider timeProvider, SecurityIdentifier securityIdentifier)
+            : base(timeProvider, securityIdentifier)
+        {
+        }
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override FundamentalTimeDependentProperty Clone(ITimeProvider timeProvider)
         {
-            _timeProvider = timeProvider;
-            _securityIdentifier = securityIdentifier;
+            return new EarningRatios(timeProvider, _securityIdentifier);
         }
     }
 }

Common/Data/Fundamental/Generated/EarningReports.cs

@@ -26,7 +26,7 @@ namespace QuantConnect.Data.Fundamental
     /// <summary>
     /// Definition of the EarningReports class
     /// </summary>
-    public class EarningReports : ReusuableCLRObject
+    public class EarningReports : FundamentalTimeDependentProperty
     {
         /// <summary>
         /// The exact date that is given in the financial statements for each quarter's end.
@@ -328,16 +328,20 @@ public class EarningReports : ReusuableCLRObject
         public DividendCoverageRatio DividendCoverageRatio => _dividendCoverageRatio ??= new(_timeProvider, _securityIdentifier);
         private DividendCoverageRatio _dividendCoverageRatio;
 
-        private readonly ITimeProvider _timeProvider;
-        private readonly SecurityIdentifier _securityIdentifier;
-
         /// <summary>
         /// Creates a new instance for the given time and security
         /// </summary>
         public EarningReports(ITimeProvider timeProvider, SecurityIdentifier securityIdentifier)
+            : base(timeProvider, securityIdentifier)
+        {
+        }
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override FundamentalTimeDependentProperty Clone(ITimeProvider timeProvider)
         {
-            _timeProvider = timeProvider;
-            _securityIdentifier = securityIdentifier;
+            return new EarningReports(timeProvider, _securityIdentifier);
         }
     }
 }

Common/Data/Fundamental/Generated/FinancialStatements.cs

@@ -26,7 +26,7 @@ namespace QuantConnect.Data.Fundamental
     /// <summary>
     /// Definition of the FinancialStatements class
     /// </summary>
-    public class FinancialStatements : ReusuableCLRObject
+    public class FinancialStatements : FundamentalTimeDependentProperty
     {
         /// <summary>
         /// The exact date that is given in the financial statements for each quarter's end.
@@ -149,16 +149,20 @@ public class FinancialStatements : ReusuableCLRObject
         public CashFlowStatement CashFlowStatement => _cashFlowStatement ??= new(_timeProvider, _securityIdentifier);
         private CashFlowStatement _cashFlowStatement;
 
-        private readonly ITimeProvider _timeProvider;
-        private readonly SecurityIdentifier _securityIdentifier;
-
         /// <summary>
         /// Creates a new instance for the given time and security
         /// </summary>
         public FinancialStatements(ITimeProvider timeProvider, SecurityIdentifier securityIdentifier)
+            : base(timeProvider, securityIdentifier)
+        {
+        }
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override FundamentalTimeDependentProperty Clone(ITimeProvider timeProvider)
         {
-            _timeProvider = timeProvider;
-            _securityIdentifier = securityIdentifier;
+            return new FinancialStatements(timeProvider, _securityIdentifier);
         }
     }
 }

Common/Data/Fundamental/Generated/OperationRatios.cs

@@ -26,7 +26,7 @@ namespace QuantConnect.Data.Fundamental
     /// <summary>
     /// Definition of the OperationRatios class
     /// </summary>
-    public class OperationRatios : ReusuableCLRObject
+    public class OperationRatios : FundamentalTimeDependentProperty
     {
         /// <summary>
         /// The growth in the company's revenue on a percentage basis. Morningstar calculates the growth percentage based on the underlying revenue data reported in the Income Statement within the company filings or reports.
@@ -728,16 +728,20 @@ public class OperationRatios : ReusuableCLRObject
         public LossRatio LossRatio => _lossRatio ??= new(_timeProvider, _securityIdentifier);
         private LossRatio _lossRatio;
 
-        private readonly ITimeProvider _timeProvider;
-        private readonly SecurityIdentifier _securityIdentifier;
-
         /// <summary>
         /// Creates a new instance for the given time and security
         /// </summary>
         public OperationRatios(ITimeProvider timeProvider, SecurityIdentifier securityIdentifier)
+            : base(timeProvider, securityIdentifier)
+        {
+        }
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override FundamentalTimeDependentProperty Clone(ITimeProvider timeProvider)
         {
-            _timeProvider = timeProvider;
-            _securityIdentifier = securityIdentifier;
+            return new OperationRatios(timeProvider, _securityIdentifier);
         }
     }
 }

Common/Data/Fundamental/Generated/SecurityReference.cs

@@ -26,7 +26,7 @@ namespace QuantConnect.Data.Fundamental
     /// <summary>
     /// Definition of the SecurityReference class
     /// </summary>
-    public class SecurityReference : ReusuableCLRObject
+    public class SecurityReference : FundamentalTimeDependentProperty
     {
         /// <summary>
         /// An arrangement of characters (often letters) representing a particular security listed on an exchange or otherwise traded publicly. Note: Morningstar's multi-share class symbols will often contain a "period" within the symbol; e.g. BRK.B for Berkshire Hathaway Class B.
@@ -244,16 +244,20 @@ public class SecurityReference : ReusuableCLRObject
         [JsonProperty("1029")]
         public string MarketDataID => FundamentalService.Get<string>(_timeProvider.GetUtcNow(), _securityIdentifier, FundamentalProperty.SecurityReference_MarketDataID);
 
-        private readonly ITimeProvider _timeProvider;
-        private readonly SecurityIdentifier _securityIdentifier;
-
         /// <summary>
         /// Creates a new instance for the given time and security
         /// </summary>
         public SecurityReference(ITimeProvider timeProvider, SecurityIdentifier securityIdentifier)
+            : base(timeProvider, securityIdentifier)
+        {
+        }
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override FundamentalTimeDependentProperty Clone(ITimeProvider timeProvider)
         {
-            _timeProvider = timeProvider;
-            _securityIdentifier = securityIdentifier;
+            return new SecurityReference(timeProvider, _securityIdentifier);
         }
     }
 }