Skip to content

Commit

Permalink
Return null from GetHistory and Downloader (#8)
Browse files Browse the repository at this point in the history 
* feat: return null from History and Downloader methods

* feat: missed flags to prevent spamming for user
  • Loading branch information
@Romazes
Romazes committed Feb 27, 2024
1 parent 0d365ad commit 1f11e07
Show file tree
Hide file tree
Showing 4 changed files with 80 additions and 34 deletions.
6 changes: 3 additions & 3 deletions QuantConnect.IEX.Tests/IEXDataDownloaderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ public void DownloadsHistoricalDataWithInvalidDataTestParameters(Symbol symbol,

var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType);

var downloadResponse = _downloader.Get(parameters).ToList();
var downloadResponse = _downloader.Get(parameters)?.ToList();

Assert.IsEmpty(downloadResponse);
Assert.IsNull(downloadResponse);
}

[Explicit("This tests require a iexcloud.io api key")]
Expand All @@ -54,7 +54,7 @@ public void DownloadsHistoricalDataWithValidDataTestParameters(Symbol symbol, Re

var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType);

var downloadResponse = _downloader.Get(parameters).ToList();
var downloadResponse = _downloader.Get(parameters)?.ToList();

Assert.IsNotEmpty(downloadResponse);

Expand Down
5 changes: 2 additions & 3 deletions QuantConnect.IEX.Tests/IEXDataHistoryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,7 @@ public void IEXCloudGetHistoryWithInvalidDataTestParameters(Symbol symbol, Resol
{
var slices = GetHistory(symbol, resolution, tickType, period);

Assert.IsEmpty(slices);
return;
Assert.IsNull(slices);
}

[Explicit("This tests require a iexcloud.io api key")]
Expand Down Expand Up @@ -228,7 +227,7 @@ private Slice[] GetHistory(Symbol symbol, Resolution resolution, TickType tickTy
{
var requests = new[] { CreateHistoryRequest(symbol, resolution, tickType, period) };

var slices = iexDataProvider.GetHistory(requests, TimeZones.Utc).ToArray();
var slices = iexDataProvider.GetHistory(requests, TimeZones.Utc)?.ToArray();
Log.Trace("Data points retrieved: " + iexDataProvider.DataPointCount);
Log.Trace("tick Type: " + tickType);
return slices;
Expand Down
20 changes: 10 additions & 10 deletions QuantConnect.IEX/IEXDataDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public void Dispose()
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
public IEnumerable<BaseData>? Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
Expand All @@ -52,19 +52,21 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa

if (tickType != TickType.Trade)
{
yield break;
Log.Error($"{nameof(IEXDataDownloader)}.{nameof(Get)}: Not supported data type - {tickType}. " +
"Currently available support only for historical of type - TradeBar");
return null;
}

if (resolution != Resolution.Daily && resolution != Resolution.Minute)
if (resolution != Resolution.Daily && resolution != Resolution.Minute)
{
Log.Error($"{nameof(IEXDataDownloader)}.{nameof(Get)}: InvalidResolution. {resolution} resolution not supported, no history returned");
yield break;
return null;
}

if (endUtc < startUtc)
{
Log.Error($"{nameof(IEXDataDownloader)}.{nameof(Get)}:InvalidDateRange. The history request start date must precede the end date, no history returned");
yield break;
return null;
}

var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
Expand All @@ -75,7 +77,7 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
endUtc,
typeof(TradeBar),
symbol,
resolution,
resolution,
exchangeHours,
dataTimeZone,
resolution,
Expand All @@ -85,10 +87,8 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
TickType.Trade)
};

foreach (var slice in _handler.GetHistory(historyRequests, TimeZones.EasternStandard))
{
yield return slice[symbol];
}

return _handler.GetHistory(historyRequests, TimeZones.EasternStandard)?.Select(slice => (BaseData)slice[symbol]);
}
}
}
83 changes: 65 additions & 18 deletions QuantConnect.IEX/IEXDataProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,21 @@ public class IEXDataProvider : SynchronizingHistoryProvider, IDataQueueHandler
/// </summary>
private static bool _invalidHistoryDataTypeWarningFired;

/// <summary>
/// Indicates whether the warning for invalid <see cref="SecurityType"/> has been fired.
/// </summary>
private bool _invalidSecurityTypeWarningFired;

/// <summary>
/// Indicates whether a warning for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC.
/// </summary>
private bool _invalidStartTimeWarningFired;

/// <summary>
/// Indicates whether a warning for an invalid <see cref="Resolution"/> has been fired, where the resolution is neither daily nor minute-based.
/// </summary>
private bool _invalidResolutionWarningFired;

/// <summary>
/// Represents two clients: one for the trade channel and another for the top-of-book channel.
/// </summary>
Expand Down Expand Up @@ -491,7 +506,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
/// <param name="requests">The historical data requests</param>
/// <param name="sliceTimeZone">The time zone used when time stamping the slice instances</param>
/// <returns>An enumerable of the slices of data covering the span specified in each request</returns>
public override IEnumerable<Slice> GetHistory(IEnumerable<Data.HistoryRequest> requests, DateTimeZone sliceTimeZone)
public override IEnumerable<Slice>? GetHistory(IEnumerable<Data.HistoryRequest> requests, DateTimeZone sliceTimeZone)
{
// Create subscription objects from the configs
var subscriptions = new List<Subscription>();
Expand All @@ -506,44 +521,61 @@ public override IEnumerable<Slice> GetHistory(IEnumerable<Data.HistoryRequest> r
"Currently available support only for historical of type - TradeBar");
_invalidHistoryDataTypeWarningFired = true;
}
return Enumerable.Empty<Slice>();
continue;
}

if (request.Symbol.SecurityType != SecurityType.Equity)
if (!CanSubscribe(request.Symbol))
{
Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'");
return Enumerable.Empty<Slice>();
if (!_invalidSecurityTypeWarningFired)
{
Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'");
_invalidSecurityTypeWarningFired = true;
}
continue;
}

if (request.StartTimeUtc >= request.EndTimeUtc)
{
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned.");
return Enumerable.Empty<Slice>();
if (!_invalidStartTimeWarningFired)
{
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned.");
_invalidStartTimeWarningFired = true;
}
continue;
}

if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute)
{
if (!_invalidResolutionWarningFired)
{
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls for IEX only support daily & minute resolution.");
_invalidResolutionWarningFired = true;
}
continue;
}

var history = ProcessHistoryRequests(request);
var subscription = CreateSubscription(request, history);
subscriptions.Add(subscription);
}

if (subscriptions.Count == 0)
{
return null;
}

return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone);
}

/// <summary>
/// Populate request data
/// </summary>
private IEnumerable<BaseData> ProcessHistoryRequests(Data.HistoryRequest request)
private IEnumerable<BaseData>? ProcessHistoryRequests(Data.HistoryRequest request)
{
var ticker = request.Symbol.ID.Symbol;
var start = ConvertTickTimeBySymbol(request.Symbol, request.StartTimeUtc);
var end = ConvertTickTimeBySymbol(request.Symbol, request.EndTimeUtc);

if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute)
{
Log.Error("IEXDataProvider.GetHistory(): History calls for IEX only support daily & minute resolution.");
yield break;
}

Log.Trace($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {request.Symbol.SecurityType}.{ticker}, Resolution: {request.Resolution}, DateTime: [{start} - {end}].");

var span = end - start;
Expand Down Expand Up @@ -602,6 +634,21 @@ private IEnumerable<BaseData> ProcessHistoryRequests(Data.HistoryRequest request
}
}

return GetHistoryRequestByUrls(urls, start, end, request.Resolution, request.DataNormalizationMode, request.Symbol);
}

/// <summary>
/// Retrieves historical data by making requests to the specified URLs.
/// </summary>
/// <param name="urls">The list of URLs to retrieve historical data from.</param>
/// <param name="startDateTime">The start date and time for the historical data. (use to skip data which not passed condition)</param>
/// <param name="endDateTime">The end date and time for the historical data. (use to skip data which not passed condition)</param>
/// <param name="resolution">The <see cref="Resolution"/> of the historical data.</param>
/// <param name="dataNormalizationMode">The <seealso cref="DataNormalizationMode"/> the historical data.</param>
/// <param name="symbol">The <seealso cref="Symbol"/> for which historical data is being retrieved.</param>
/// <returns>An enumerable collection of <see cref="BaseData"/> containing the retrieved historical data.</returns>
private IEnumerable<BaseData> GetHistoryRequestByUrls(List<string> urls, DateTime startDateTime, DateTime endDateTime, Resolution resolution, DataNormalizationMode dataNormalizationMode, Symbol symbol)
{
foreach (var url in urls)
{
var response = ExecuteGetRequest(url);
Expand Down Expand Up @@ -631,7 +678,7 @@ private IEnumerable<BaseData> ProcessHistoryRequests(Data.HistoryRequest request
period = TimeSpan.FromDays(1);
}

if (date < start || date > end)
if (date < startDateTime || date > endDateTime)
{
continue;
}
Expand All @@ -645,8 +692,8 @@ private IEnumerable<BaseData> ProcessHistoryRequests(Data.HistoryRequest request

decimal open, high, low, close, volume;

if (request.Resolution == Resolution.Daily &&
request.DataNormalizationMode == DataNormalizationMode.Raw)
if (resolution == Resolution.Daily &&
dataNormalizationMode == DataNormalizationMode.Raw)
{
open = item["uOpen"].Value<decimal>();
high = item["uHigh"].Value<decimal>();
Expand All @@ -663,7 +710,7 @@ private IEnumerable<BaseData> ProcessHistoryRequests(Data.HistoryRequest request
volume = item["volume"].Value<int>();
}

var tradeBar = new TradeBar(date, request.Symbol, open, high, low, close, volume, period);
var tradeBar = new TradeBar(date, symbol, open, high, low, close, volume, period);

yield return tradeBar;
}
Expand Down

0 comments on commit 1f11e07

Please sign in to comment.