Skip to content

Commit

Permalink
Historical data when symbol was restructured (renamed) (#10)
Browse files Browse the repository at this point in the history 
* feat: `volatile` for prevent spamming flags

* test:remove: extra param in test data
feat: add overloading of CreateHistoryRequest

* refactor: DataDownloader provider to general structure

* fix: convert DateTime to TZ in GetHistory
feat: prevent spamming flag for limit of minute data
refactor: reduce some part of code
test:feat: get ancient historical data (renamed symbol test)

* feat: get relevant ticker for each historical request
  • Loading branch information
@Romazes
Romazes committed Mar 18, 2024
1 parent 1d47274 commit 6d3d9f6
Show file tree
Hide file tree
Showing 3 changed files with 147 additions and 112 deletions.
52 changes: 43 additions & 9 deletions QuantConnect.IEX.Tests/IEXDataHistoryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
*/

using System;
using NodaTime;
using System.Linq;
using NUnit.Framework;
using QuantConnect.Data;
Expand Down Expand Up @@ -65,7 +66,7 @@ internal static IEnumerable<TestCaseData> ValidDataTestParameters
.SetDescription("Valid parameters - Minute resolution, 5 days period.")
.SetCategory("Valid");

yield return new TestCaseData(Symbols.SPY, Resolution.Minute, TickType.Trade, TimeSpan.FromDays(45), true)
yield return new TestCaseData(Symbols.SPY, Resolution.Minute, TickType.Trade, TimeSpan.FromDays(45))
.SetDescription("Valid parameters - Beyond 45 days, Minute resolution.")
.SetCategory("Valid");
}
Expand Down Expand Up @@ -159,6 +160,22 @@ public void IEXCloudGetHistoryWithValidDataTestParameters(Symbol symbol, Resolut
Assert.That(slices, Is.Ordered.By("Time"));
}

[Explicit("This tests require a iexcloud.io api key")]
[TestCase("GOOGL", Resolution.Daily, "2013/1/3", "2015/12/29", Description = "October 2, 2015. [GOOG -> GOOGL]")]
[TestCase("META", Resolution.Daily, "2020/1/3", "2023/12/29", Description = "October 28, 2021. [FB -> META]")]
public void GetAncientEquityHistoricalData(string ticker, Resolution resolution, DateTime startDate, DateTime endDate)
{
var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA);

var request = CreateHistoryRequest(symbol, resolution, TickType.Trade, startDate, endDate);

var slices = iexDataProvider.GetHistory(new[] { request }, TimeZones.NewYork)?.ToList();

Assert.Greater(slices.Count, 1);
Assert.That(slices.First().Time.Date, Is.EqualTo(startDate));
Assert.That(slices.Last().Time.Date, Is.LessThanOrEqualTo(endDate));
}

internal static void AssertTradeBar(Symbol expectedSymbol, Resolution resolution, BaseData baseData, Symbol actualSymbol = null)
{
if (actualSymbol != null)
Expand Down Expand Up @@ -235,25 +252,42 @@ private Slice[] GetHistory(Symbol symbol, Resolution resolution, TickType tickTy

internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period)
{
var utcNow = DateTime.UtcNow;
var end = new DateTime(2024, 3, 15, 16, 0, 0);

var dataType = LeanData.GetDataType(resolution, tickType);
if (resolution == Resolution.Daily)
{
end = end.Date.AddDays(1);
}

return CreateHistoryRequest(symbol, resolution, tickType, end.Subtract(period), end);
}

internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateTime, DateTime endDateTime,
SecurityExchangeHours exchangeHours = null, DateTimeZone dataTimeZone = null)
{
if (exchangeHours == null)
{
exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork);
}

var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
if (dataTimeZone == null)
{
dataTimeZone = TimeZones.NewYork;
}

var dataType = LeanData.GetDataType(resolution, tickType);
return new HistoryRequest(
startTimeUtc: utcNow.Add(-period),
endTimeUtc: utcNow,
startTimeUtc: startDateTime,
endTimeUtc: endDateTime,
dataType: dataType,
symbol: symbol,
resolution: resolution,
exchangeHours: exchangeHours,
dataTimeZone: dataTimeZone,
fillForwardResolution: resolution,
fillForwardResolution: null,
includeExtendedMarketHours: true,
isCustomData: false,
DataNormalizationMode.Raw,
dataNormalizationMode: DataNormalizationMode.Adjusted,
tickType: tickType
);
}
Expand Down
35 changes: 15 additions & 20 deletions QuantConnect.IEX/IEXDataDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
*/

using QuantConnect.Data;
using QuantConnect.Util;
using QuantConnect.Securities;
using QuantConnect.Data.Market;

namespace QuantConnect.Lean.DataSource.IEX
{
Expand Down Expand Up @@ -44,30 +44,25 @@ public void Dispose()
public IEnumerable<BaseData>? Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;

var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);

var historyRequests = new[] {
new HistoryRequest(startUtc,
endUtc,
typeof(TradeBar),
symbol,
resolution,
exchangeHours,
dataTimeZone,
resolution,
true,
false,
DataNormalizationMode.Raw,
TickType.Trade)
};
var historyRequests = new HistoryRequest(
dataDownloaderGetParameters.StartUtc,
dataDownloaderGetParameters.EndUtc,
LeanData.GetDataType(dataDownloaderGetParameters.Resolution, dataDownloaderGetParameters.TickType),
symbol,
dataDownloaderGetParameters.Resolution,
exchangeHours,
dataTimeZone,
dataDownloaderGetParameters.Resolution,
true,
false,
DataNormalizationMode.Raw,
dataDownloaderGetParameters.TickType);

return _handler.GetHistory(historyRequests, TimeZones.EasternStandard)?.Select(slice => (BaseData)slice[symbol]);
return _handler.ProcessHistoryRequests(historyRequests);
}
}
}
172 changes: 89 additions & 83 deletions QuantConnect.IEX/IEXDataProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,22 +56,28 @@ public class IEXDataProvider : SynchronizingHistoryProvider, IDataQueueHandler
/// <summary>
/// Flag indicating whether a warning about unsupported data types in user history should be suppressed to prevent spam.
/// </summary>
private static bool _invalidHistoryDataTypeWarningFired;
private volatile bool _invalidHistoryDataTypeWarningFired;

/// <summary>
/// Indicates whether the warning for invalid <see cref="SecurityType"/> has been fired.
/// </summary>
private bool _invalidSecurityTypeWarningFired;
private volatile bool _invalidSecurityTypeWarningFired;

/// <summary>
/// Indicates whether a warning for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC.
/// </summary>
private bool _invalidStartTimeWarningFired;
private volatile bool _invalidStartTimeWarningFired;

/// <summary>
/// Indicates whether a warning for an invalid <see cref="Resolution"/> has been fired, where the resolution is neither daily nor minute-based.
/// </summary>
private bool _invalidResolutionWarningFired;
private volatile bool _invalidResolutionWarningFired;

/// <summary>
/// Indicates whether a warning has been triggered for reaching the limit of <see cref="Resolution.Minute"/> resolution,
/// where the startDateTime is not greater than 2 years.
/// </summary>
private volatile bool _limitMinuteResolutionWarningFired;

/// <summary>
/// Represents two clients: one for the trade channel and another for the top-of-book channel.
Expand Down Expand Up @@ -512,49 +518,13 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
var subscriptions = new List<Subscription>();
foreach (var request in requests)
{
// IEX does return historical TradeBar - give one time warning if inconsistent data type was requested
if (request.TickType != TickType.Trade)
{
if (!_invalidHistoryDataTypeWarningFired)
{
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Not supported data type - {request.DataType.Name}. " +
"Currently available support only for historical of type - TradeBar");
_invalidHistoryDataTypeWarningFired = true;
}
continue;
}

if (!CanSubscribe(request.Symbol))
{
if (!_invalidSecurityTypeWarningFired)
{
Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'");
_invalidSecurityTypeWarningFired = true;
}
continue;
}

if (request.StartTimeUtc >= request.EndTimeUtc)
{
if (!_invalidStartTimeWarningFired)
{
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned.");
_invalidStartTimeWarningFired = true;
}
continue;
}
var history = ProcessHistoryRequests(request);

if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute)
if (history == null)
{
if (!_invalidResolutionWarningFired)
{
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls for IEX only support daily & minute resolution.");
_invalidResolutionWarningFired = true;
}
continue;
}

var history = ProcessHistoryRequests(request);
var subscription = CreateSubscription(request, history);
subscriptions.Add(subscription);
}
Expand All @@ -570,23 +540,75 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
/// <summary>
/// Populate request data
/// </summary>
private IEnumerable<BaseData>? ProcessHistoryRequests(Data.HistoryRequest request)
public IEnumerable<BaseData>? ProcessHistoryRequests(Data.HistoryRequest request)
{
var ticker = request.Symbol.ID.Symbol;
var start = ConvertTickTimeBySymbol(request.Symbol, request.StartTimeUtc);
var end = ConvertTickTimeBySymbol(request.Symbol, request.EndTimeUtc);
// IEX does return historical TradeBar - give one time warning if inconsistent data type was requested
if (request.TickType != TickType.Trade)
{
if (!_invalidHistoryDataTypeWarningFired)
{
_invalidHistoryDataTypeWarningFired = true;
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Not supported data type - {request.DataType.Name}. " +
"Currently available support only for historical of type - TradeBar");
}
return null;
}

if (!CanSubscribe(request.Symbol))
{
if (!_invalidSecurityTypeWarningFired)
{
_invalidSecurityTypeWarningFired = true;
Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'");
}
return null;
}

Log.Trace($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {request.Symbol.SecurityType}.{ticker}, Resolution: {request.Resolution}, DateTime: [{start} - {end}].");
if (request.StartTimeUtc >= request.EndTimeUtc)
{
if (!_invalidStartTimeWarningFired)
{
_invalidStartTimeWarningFired = true;
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned.");
}
return null;
}

if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute)
{
if (!_invalidResolutionWarningFired)
{
_invalidResolutionWarningFired = true;
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls for IEX only support daily & minute resolution.");
}
return null;
}

// Always obtain the most relevant ticker symbol based on the current time.
var ticker = SecurityIdentifier.Ticker(request.Symbol, DateTime.UtcNow);
var startExchangeDateTime = ConvertTickTimeBySymbol(request.Symbol, request.StartTimeUtc);
var endExchangeDateTime = ConvertTickTimeBySymbol(request.Symbol, request.EndTimeUtc);

if (request.Resolution == Resolution.Minute && startExchangeDateTime <= DateTime.Today.AddYears(-2))
{
if (!_limitMinuteResolutionWarningFired)
{
_limitMinuteResolutionWarningFired = true;
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls with minute resolution for IEX available only not more 2 years.");
}
return null;
}

Log.Trace($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {request.Symbol.SecurityType}.{ticker}, Resolution: {request.Resolution}, DateTime: [{startExchangeDateTime} - {endExchangeDateTime}].");

var span = end - start;
var urls = new List<string>();

switch (request.Resolution)
{
case Resolution.Minute:
{
var begin = start;
while (begin < end)
var begin = startExchangeDateTime;
while (begin < endExchangeDateTime)
{
var url = $"{BaseUrl}/{ticker}/chart/date/{begin.ToStringInvariant("yyyyMMdd")}?token={_apiKey}";
urls.Add(url);
Expand All @@ -597,35 +619,19 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
}
case Resolution.Daily:
{
string suffix;
if (span.Days < 30)
{
suffix = "1m";
}
else if (span.Days < 3 * 30)
{
suffix = "3m";
}
else if (span.Days < 6 * 30)
{
suffix = "6m";
}
else if (span.Days < 12 * 30)
{
suffix = "1y";
}
else if (span.Days < 24 * 30)
{
suffix = "2y";
}
else if (span.Days < 60 * 30)
{
suffix = "5y";
}
else
// To retrieve a specific start-to-end dateTime range, calculate the duration between the current time and the startExchangeDateTime.
var span = DateTime.Now - startExchangeDateTime;

string suffix = span.Days switch
{
suffix = "max"; // max is 15 years
}
< 30 => "1m",
< 3 * 30 => "3m",
< 6 * 30 => "6m",
< 12 * 30 => "1y",
< 24 * 30 => "2y",
< 60 * 30 => "5y",
_ => "max" // max is 15 years
};

var url = $"{BaseUrl}/{ticker}/chart/{suffix}?token={_apiKey}";
urls.Add(url);
Expand All @@ -634,7 +640,9 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
}
}

return GetHistoryRequestByUrls(urls, start, end, request.Resolution, request.DataNormalizationMode, request.Symbol);
Log.Debug($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {string.Join("\n", urls)}");

return GetHistoryRequestByUrls(urls, startExchangeDateTime, endExchangeDateTime, request.Resolution, request.DataNormalizationMode, request.Symbol);
}

/// <summary>
Expand Down Expand Up @@ -678,7 +686,7 @@ private IEnumerable<BaseData> GetHistoryRequestByUrls(List<string> urls, DateTim
period = TimeSpan.FromDays(1);
}

if (date < startDateTime || date > endDateTime)
if (!(date >= startDateTime && date < endDateTime))
{
continue;
}
Expand Down Expand Up @@ -710,9 +718,7 @@ private IEnumerable<BaseData> GetHistoryRequestByUrls(List<string> urls, DateTim
volume = item["volume"].Value<int>();
}

var tradeBar = new TradeBar(date, symbol, open, high, low, close, volume, period);

yield return tradeBar;
yield return new TradeBar(ConvertTickTimeBySymbol(symbol, date), symbol, open, high, low, close, volume, period);
}
}
}
Expand Down

0 comments on commit 6d3d9f6

Please sign in to comment.