Skip to content

Commit

Permalink
refactor: return null in wrong request of GetHistory()
Browse files Browse the repository at this point in the history
  • Loading branch information
@Romazes
Romazes committed Feb 22, 2024
1 parent 0de9587 commit ff7e0f4
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 30 deletions.
9 changes: 5 additions & 4 deletions QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,9 @@ public void DownloadsHistoricalDataWithValidDataTestParameters(Symbol symbol, Re
{
var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateTimeUtc, endDateTimeUtc, TickType.Trade);

var downloadResponse = _downloader.Get(parameters).ToList();
var downloadResponse = _downloader.Get(parameters)?.ToList();

Assert.IsNotNull(downloadResponse);
Assert.IsNotEmpty(downloadResponse);

Log.Trace($"{symbol}.{resolution}.[{startDateTimeUtc} - {endDateTimeUtc}]: Amount = {downloadResponse.Count}");
Expand Down Expand Up @@ -84,9 +85,9 @@ public void DownloadsHistoricalDataWithInvalidDataTestParameters(Symbol symbol,
{
var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateTimeUtc, endDateTimeUtc, tickType);

var downloadResponse = _downloader.Get(parameters).ToList();
var downloadResponse = _downloader.Get(parameters)?.ToList();

Assert.IsEmpty(downloadResponse);
Assert.IsNull(downloadResponse);
}

private static IEnumerable<TestCaseData> HistoricalInvalidDataThrowExceptionTestCases
Expand All @@ -106,7 +107,7 @@ public void DownloadsHistoricalDataWithInvalidDataTestParametersThrowException(S
{
var parameters = new DataDownloaderGetParameters(symbol, Resolution.Minute, new DateTime(2024, 1, 1), new DateTime(2024, 2, 1), TickType.Trade);

Assert.That(() => _downloader.Get(parameters).ToList(), Throws.Exception);
Assert.That(() => _downloader.Get(parameters)?.ToList(), Throws.Exception);
}
}
}
22 changes: 11 additions & 11 deletions QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public void OneTimeSetUp()

[Test]
[TestCaseSource(nameof(TestData))]
public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, int period, bool isNonEmptyResult)
public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, int period, bool isNotNullResult)
{
_coinApiDataQueueHandler.SetUpHistDataLimit(100);

Expand All @@ -67,14 +67,15 @@ public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, i
resolution, true, false, DataNormalizationMode.Raw, TickType.Trade)
};

var slices = _coinApiDataQueueHandler.GetHistory(historyRequests, TimeZones.Utc).ToArray();
var slices = _coinApiDataQueueHandler.GetHistory(historyRequests, TimeZones.Utc)?.ToArray();

if (isNonEmptyResult)
if (isNotNullResult)
{
Assert.IsNotNull(slices);
// For resolution larger than second do more tests
if (resolution > Resolution.Second)
{
Assert.AreEqual(period, slices.Length);
Assert.That(slices.Length, Is.EqualTo(period));

var firstSliceTradeBars = slices.First().Bars.Values;

Expand All @@ -83,34 +84,33 @@ public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, i
firstSliceTradeBars.DoForEach(tb =>
{
var resTimeSpan = resolution.ToTimeSpan();
Assert.AreEqual(resTimeSpan, tb.Period);
Assert.AreEqual(startTimeUtc.RoundUp(resTimeSpan), tb.Time);
Assert.That(tb.Period, Is.EqualTo(resTimeSpan));
Assert.That(tb.Time, Is.EqualTo(startTimeUtc.RoundUp(resTimeSpan)));
});

var lastSliceTradeBars = slices.Last().Bars.Values;

lastSliceTradeBars.DoForEach(tb =>
{
var resTimeSpan = resolution.ToTimeSpan();
Assert.AreEqual(resTimeSpan, tb.Period);
Assert.AreEqual(nowUtc.RoundDown(resTimeSpan), tb.Time);
Assert.That(tb.Period, Is.EqualTo(resTimeSpan));
Assert.That(tb.Time, Is.EqualTo(nowUtc.RoundDown(resTimeSpan)));
});
}
// For res. second data counts, start/end dates may slightly vary from historical request's
// Make sure just that resolution is correct and amount is positive numb.
else
{
Assert.IsTrue(slices.Length > 0);
Assert.AreEqual(resolution.ToTimeSpan(), slices.First().Bars.Values.FirstOrDefault()?.Period);
Assert.That(slices.First().Bars.Values.FirstOrDefault()?.Period, Is.EqualTo(resolution.ToTimeSpan()));
}

// Slices are ordered by time
Assert.That(slices, Is.Ordered.By("Time"));
}
else
{
// Empty
Assert.IsEmpty(slices);
Assert.IsNull(slices);
}
}

Expand Down
21 changes: 17 additions & 4 deletions QuantConnect.CoinAPI/CoinAPIDataDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,19 @@ public CoinAPIDataDownloader()
_marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
}

public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
public IEnumerable<BaseData>? Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
if (dataDownloaderGetParameters.TickType != TickType.Trade)
{
Log.Error($"{nameof(CoinAPIDataDownloader)}.{nameof(Get)}: Not supported data type - {dataDownloaderGetParameters.TickType}. " +
$"Currently available support only for historical of type - {nameof(TickType.Trade)}");
yield break;
return null;
}

if (dataDownloaderGetParameters.EndUtc < dataDownloaderGetParameters.StartUtc)
{
Log.Error($"{nameof(CoinAPIDataDownloader)}.{nameof(Get)}:InvalidDateRange. The history request start date must precede the end date, no history returned");
yield break;
return null;
}

var symbol = dataDownloaderGetParameters.Symbol;
Expand All @@ -64,7 +64,20 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
dataNormalizationMode: DataNormalizationMode.Raw,
tickType: TickType.Trade);

foreach (var slice in _historyProvider.GetHistory(historyRequests))
var history = _historyProvider.GetHistory(historyRequests);

// historyRequest contains wrong data request
if (history == null)
{
return null;
}

return GetHistoryInSlice(history);
}

private IEnumerable<BaseData> GetHistoryInSlice(IEnumerable<BaseData> history)
{
foreach (var slice in history)
{
yield return slice;
}
Expand Down
42 changes: 31 additions & 11 deletions QuantConnect.CoinAPI/CoinApiDataHistoryProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,20 @@ public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> reques
foreach (var request in requests)
{
var history = GetHistory(request);

if (history == null)
{
continue;
}

var subscription = CreateSubscription(request, history);
subscriptions.Add(subscription);
}

if (subscriptions.Count == 0)
{
return null;
}
return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone);
}

Expand All @@ -59,13 +70,13 @@ public IEnumerable<BaseData> GetHistory(HistoryRequest historyRequest)
if (!CanSubscribe(historyRequest.Symbol))
{
Log.Error($"CoinApiDataProvider.GetHistory(): Invalid security type {historyRequest.Symbol.SecurityType}");
yield break;
return null;
}

if (historyRequest.Resolution == Resolution.Tick)
{
Log.Error($"CoinApiDataProvider.GetHistory(): No historical ticks, only OHLCV timeseries");
yield break;
return null;
}

if (historyRequest.DataType == typeof(QuoteBar))
Expand All @@ -75,12 +86,21 @@ public IEnumerable<BaseData> GetHistory(HistoryRequest historyRequest)
Log.Error("CoinApiDataProvider.GetHistory(): No historical QuoteBars , only TradeBars");
_invalidHistoryDataTypeWarningFired = true;
}
yield break;
return null;
}

var resolutionTimeSpan = historyRequest.Resolution.ToTimeSpan();
var lastRequestedBarStartTime = historyRequest.EndTimeUtc.RoundDown(resolutionTimeSpan);
var currentStartTime = historyRequest.StartTimeUtc.RoundUp(resolutionTimeSpan);
return GetHistory(historyRequest.Symbol,
historyRequest.Resolution,
historyRequest.StartTimeUtc,
historyRequest.EndTimeUtc
);
}

private IEnumerable<BaseData> GetHistory(Symbol symbol, Resolution resolution, DateTime startDateTimeUtc, DateTime endDateTimeUtc)
{
var resolutionTimeSpan = resolution.ToTimeSpan();
var lastRequestedBarStartTime = endDateTimeUtc.RoundDown(resolutionTimeSpan);
var currentStartTime = startDateTimeUtc.RoundUp(resolutionTimeSpan);
var currentEndTime = lastRequestedBarStartTime;

// Perform a check of the number of bars requested, this must not exceed a static limit
Expand All @@ -95,8 +115,8 @@ public IEnumerable<BaseData> GetHistory(HistoryRequest historyRequest)

while (currentStartTime < lastRequestedBarStartTime)
{
var coinApiSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol);
var coinApiPeriod = _ResolutionToCoinApiPeriodMappings[historyRequest.Resolution];
var coinApiSymbol = _symbolMapper.GetBrokerageSymbol(symbol);
var coinApiPeriod = _ResolutionToCoinApiPeriodMappings[resolution];

// Time must be in ISO 8601 format
var coinApiStartTime = currentStartTime.ToStringInvariant("s");
Expand Down Expand Up @@ -128,15 +148,15 @@ public IEnumerable<BaseData> GetHistory(HistoryRequest historyRequest)
// Can be no historical data for a short period interval
if (!coinApiHistoryBars.Any())
{
Log.Error($"CoinApiDataProvider.GetHistory(): API returned no data for the requested period [{coinApiStartTime} - {coinApiEndTime}] for symbol [{historyRequest.Symbol}]");
Log.Error($"CoinApiDataProvider.GetHistory(): API returned no data for the requested period [{coinApiStartTime} - {coinApiEndTime}] for symbol [{symbol}]");
continue;
}

foreach (var ohlcv in coinApiHistoryBars)
{
yield return
new TradeBar(ohlcv.TimePeriodStart, historyRequest.Symbol, ohlcv.PriceOpen, ohlcv.PriceHigh,
ohlcv.PriceLow, ohlcv.PriceClose, ohlcv.VolumeTraded, historyRequest.Resolution.ToTimeSpan());
new TradeBar(ohlcv.TimePeriodStart, symbol, ohlcv.PriceOpen, ohlcv.PriceHigh,
ohlcv.PriceLow, ohlcv.PriceClose, ohlcv.VolumeTraded, resolutionTimeSpan);
}

currentStartTime = currentEndTime;
Expand Down

0 comments on commit ff7e0f4

Please sign in to comment.