Skip to content

Commit

Permalink
Initialization via Job Packet and Rename DataQueueHandler to DataProv…
Browse files Browse the repository at this point in the history 
…ider (#7)

* feat: initialization through Job packet

* rename: DataQueueHandler to DataProvider

* rename: projects, solution
fix: path in workflow file

* fix: tryGetConfig of iex-price-plan

* feat: handle "parsing to enum" excpetion

* fix: additional validation to enum parsed value

* feat: use global instead of Config in ValidateSubscription
  • Loading branch information
@Romazes
Romazes committed Feb 21, 2024
1 parent acb9c55 commit 0d365ad
Show file tree
Hide file tree
Showing 16 changed files with 221 additions and 121 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@ jobs:
options: -v /home/runner/work:/__w --workdir /__w/Lean.DataSource.IEX/Lean.DataSource.IEX -e QC_JOB_USER_ID=${{ secrets.QC_JOB_USER_ID }} -e QC_API_ACCESS_TOKEN=${{ secrets.QC_API_ACCESS_TOKEN }} -e QC_JOB_ORGANIZATION_ID=${{ secrets.QC_JOB_ORGANIZATION_ID }} -e QC_IEX_CLOUD_API_KEY=${{ secrets.QC_IEX_CLOUD_API_KEY }}

- name: BuildDataSource
run: dotnet build ./QuantConnect.IEX/QuantConnect.IEX.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
run: dotnet build ./QuantConnect.IEX/QuantConnect.DataSource.IEX.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1

- name: BuildDataSourceTests
run: dotnet build ./QuantConnect.IEX.Tests/QuantConnect.IEX.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
run: dotnet build ./QuantConnect.IEX.Tests/QuantConnect.DataSource.IEX.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1

- name: Run Tests
run: dotnet test ./QuantConnect.IEX.Tests/bin/Release/QuantConnect.IEX.Tests.dll
run: dotnet test ./QuantConnect.IEX.Tests/bin/Release/QuantConnect.Lean.DataSource.IEX.Tests.dll
4 changes: 2 additions & 2 deletions Lean.DataSource.IEX.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.5.002.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.IEX.Tests", "QuantConnect.IEX.Tests\QuantConnect.IEX.Tests.csproj", "{38BD8123-9838-4984-A4D1-B8D1062197B4}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.DataSource.IEX.Tests", "QuantConnect.IEX.Tests\QuantConnect.DataSource.IEX.Tests.csproj", "{38BD8123-9838-4984-A4D1-B8D1062197B4}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.IEX", "QuantConnect.IEX\QuantConnect.IEX.csproj", "{6324C205-9693-4FA1-9EC1-9662E569D046}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.DataSource.IEX", "QuantConnect.IEX\QuantConnect.DataSource.IEX.csproj", "{6324C205-9693-4FA1-9EC1-9662E569D046}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.Tests", "..\Lean\Tests\QuantConnect.Tests.csproj", "{D6772C71-231E-44D1-9DD1-E796D393AAAB}"
EndProject
Expand Down
2 changes: 1 addition & 1 deletion QuantConnect.IEX.Tests/IEXDataDownloaderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
using NUnit.Framework;
using System.Collections.Generic;

namespace QuantConnect.IEX.Tests
namespace QuantConnect.Lean.DataSource.IEX.Tests
{
public class IEXDataDownloaderTests
{
Expand Down
12 changes: 6 additions & 6 deletions QuantConnect.IEX.Tests/IEXDataHistoryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,25 +25,25 @@
using QuantConnect.Data.Market;
using System.Collections.Generic;

namespace QuantConnect.IEX.Tests
namespace QuantConnect.Lean.DataSource.IEX.Tests
{
[TestFixture]
public class IEXDataHistoryTests
{
private static MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();

private IEXDataQueueHandler iexDataQueueHandler;
private IEXDataProvider iexDataProvider;

[SetUp]
public void SetUp()
{
iexDataQueueHandler = new IEXDataQueueHandler();
iexDataProvider = new IEXDataProvider();
}

[TearDown]
public void TearDown()
{
iexDataQueueHandler.Dispose();
iexDataProvider.Dispose();
}

/// <summary>
Expand Down Expand Up @@ -228,8 +228,8 @@ private Slice[] GetHistory(Symbol symbol, Resolution resolution, TickType tickTy
{
var requests = new[] { CreateHistoryRequest(symbol, resolution, tickType, period) };

var slices = iexDataQueueHandler.GetHistory(requests, TimeZones.Utc).ToArray();
Log.Trace("Data points retrieved: " + iexDataQueueHandler.DataPointCount);
var slices = iexDataProvider.GetHistory(requests, TimeZones.Utc).ToArray();
Log.Trace("Data points retrieved: " + iexDataProvider.DataPointCount);
Log.Trace("tick Type: " + tickType);
return slices;
}
Expand Down
115 changes: 87 additions & 28 deletions ...ect.IEX.Tests/IEXDataQueueHandlerTests.cs → ...Connect.IEX.Tests/IEXDataProviderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
using System.Threading;
using QuantConnect.Data;
using QuantConnect.Tests;
using QuantConnect.Packets;
using QuantConnect.Logging;
using Microsoft.CodeAnalysis;
using System.Threading.Tasks;
Expand All @@ -28,13 +29,16 @@
using System.Collections.Concurrent;
using QuantConnect.Data.UniverseSelection;

namespace QuantConnect.IEX.Tests
namespace QuantConnect.Lean.DataSource.IEX.Tests
{
[TestFixture, Explicit("This tests require a iexcloud.io api key")]
public class IEXDataQueueHandlerTests
public class IEXDataProviderTests
{
private readonly string _apiKey = Config.Get("iex-cloud-api-key");
private readonly string _pricePlan = Config.Get("iex-cloud-api-key");

private CancellationTokenSource _cancellationTokenSource;
private IEXDataQueueHandler iexDataQueueHandler;
private IEXDataProvider iexDataProvider;

private static readonly string[] HardCodedSymbolsSNP = {
"AAPL", "MSFT", "AMZN", "FB", "GOOGL", "GOOG", "BRK.B", "JNJ", "PG", "NVDA", "V", "JPM", "HD", "UNH", "MA", "VZ",
Expand Down Expand Up @@ -76,15 +80,15 @@ public class IEXDataQueueHandlerTests
public void SetUp()
{
_cancellationTokenSource = new();
iexDataQueueHandler = new IEXDataQueueHandler();
iexDataProvider = new IEXDataProvider();
}

[TearDown]
public void TearDown()
{
if (iexDataQueueHandler != null)
if (iexDataProvider != null)
{
iexDataQueueHandler.Dispose();
iexDataProvider.Dispose();
}

_cancellationTokenSource.Dispose();
Expand Down Expand Up @@ -119,7 +123,7 @@ public void IEXCouldSubscribeManyTimes(Queue<SubscriptionDataConfig> configs)
foreach (var config in configs)
{
ProcessFeed(
iexDataQueueHandler.Subscribe(config, (s, e) => { }),
iexDataProvider.Subscribe(config, (s, e) => { }),
tick =>
{
if (tick != null)
Expand Down Expand Up @@ -155,7 +159,7 @@ public void IEXCouldSubscribeManyTimes(Queue<SubscriptionDataConfig> configs)

foreach (var config in configs)
{
iexDataQueueHandler.Unsubscribe(config);
iexDataProvider.Unsubscribe(config);
}

_cancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(20));
Expand All @@ -177,12 +181,12 @@ public void IEXSubscribeToSeveralThenUnSubscribeExceptOne(Queue<SubscriptionData
foreach (var config in configs)
{
ProcessFeed(
iexDataQueueHandler.Subscribe(config, (s, e) => { }),
iexDataProvider.Subscribe(config, (s, e) => { }),
tick =>
{
if (tick != null)
{
Log.Debug($"{nameof(IEXDataQueueHandlerTests)}: tick: {tick}");
Log.Debug($"{nameof(IEXDataProviderTests)}: tick: {tick}");
tempDictionary.AddOrUpdate(tick.Symbol, 1, (id, count) => count + 1);
Expand All @@ -201,7 +205,7 @@ public void IEXSubscribeToSeveralThenUnSubscribeExceptOne(Queue<SubscriptionData
for (int i = configs.Count; i > 2; i--)
{
configs.TryDequeue(out var config);
iexDataQueueHandler.Unsubscribe(config);
iexDataProvider.Unsubscribe(config);
tempDictionary.TryRemove(config.Symbol, out _);
}

Expand All @@ -216,7 +220,7 @@ public void IEXSubscribeToSeveralThenUnSubscribeExceptOne(Queue<SubscriptionData
for (int i = configs.Count; i > 0; i--)
{
configs.TryDequeue(out var config);
iexDataQueueHandler.Unsubscribe(config);
iexDataProvider.Unsubscribe(config);
}

Thread.Sleep(TimeSpan.FromSeconds(1));
Expand Down Expand Up @@ -244,7 +248,7 @@ public void IEXCouldSubscribeAndUnsubscribe()
{
_cancellationTokenSource.Cancel();
};

var configs = new[] {
GetSubscriptionDataConfig<TradeBar>(Symbol.Create("MBLY", SecurityType.Equity, Market.USA), Resolution.Second),
GetSubscriptionDataConfig<TradeBar>(Symbol.Create("USO", SecurityType.Equity, Market.USA), Resolution.Second)
Expand All @@ -253,14 +257,14 @@ public void IEXCouldSubscribeAndUnsubscribe()
Array.ForEach(configs, (c) =>
{
ProcessFeed(
iexDataQueueHandler.Subscribe(c, (s, e) => { }),
iexDataProvider.Subscribe(c, (s, e) => { }),
callback,
throwExceptionCallback);
});

Assert.IsFalse(_cancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(20)), "The cancellation token was cancelled block thread.");

iexDataQueueHandler.Unsubscribe(Enumerable.First(configs, c => string.Equals(c.Symbol.Value, "MBLY")));
iexDataProvider.Unsubscribe(Enumerable.First(configs, c => string.Equals(c.Symbol.Value, "MBLY")));

Log.Trace("Unsubscribing");
_cancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(2));
Expand All @@ -279,12 +283,12 @@ public void IEXCouldSubscribeMoreThan100Symbols()
{
foreach (var config in GetSubscriptionDataConfigs(ticker, Resolution.Second))
{
ProcessFeed(iexDataQueueHandler.Subscribe(config, (s, e) => { }), throwExceptionCallback: () => _cancellationTokenSource.Cancel());
ProcessFeed(iexDataProvider.Subscribe(config, (s, e) => { }), throwExceptionCallback: () => _cancellationTokenSource.Cancel());
}
}

Assert.IsFalse(resetEvent.WaitOne(TimeSpan.FromMinutes(2), _cancellationTokenSource.Token), "The cancellation token was cancelled block thread.");
Assert.IsTrue(iexDataQueueHandler.IsConnected);
Assert.IsTrue(iexDataProvider.IsConnected);
}

[Test]
Expand All @@ -299,12 +303,12 @@ public void SubscribeOnALotOfSymbolsThrowArgumentExceptionExceedsAllowedLimit()
{
foreach (var config in GetSubscriptionDataConfigs(ticker, Resolution.Second))
{
iexDataQueueHandler.Subscribe(config, (s, e) => { });
iexDataProvider.Subscribe(config, (s, e) => { });
}
}
cancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(20));
});
Assert.Less(iexDataQueueHandler.maxAllowedSymbolLimit, symbolCounter);
Assert.Less(iexDataProvider.maxAllowedSymbolLimit, symbolCounter);
}

[Test]
Expand All @@ -315,7 +319,7 @@ public void NotSubscribeOnUniverseSymbol()

foreach (var config in GetSubscriptionDataConfigs(universeSymbol, Resolution.Second))
{
Assert.IsNull(iexDataQueueHandler.Subscribe(config, (s, e) => { }));
Assert.IsNull(iexDataProvider.Subscribe(config, (s, e) => { }));
}
}

Expand All @@ -326,7 +330,7 @@ public void NotSubscribeOnCanonicalSymbol()

foreach (var config in GetSubscriptionDataConfigs(spy, Resolution.Second))
{
Assert.IsNull(iexDataQueueHandler.Subscribe(config, (s, e) => { }));
Assert.IsNull(iexDataProvider.Subscribe(config, (s, e) => { }));
}
}

Expand All @@ -337,20 +341,21 @@ public void SubscribeWithWrongApiKeyThrowException()

var isSubscribeThrowException = false;

var iexDataQueueHandler = new IEXDataQueueHandler();
var iexDataProvider = new IEXDataProvider();

foreach (var config in GetSubscriptionDataConfigs(Symbols.SPY, Resolution.Second))
{
ProcessFeed(
iexDataQueueHandler.Subscribe(config, (s, e) => { }),
iexDataProvider.Subscribe(config, (s, e) => { }),
tick =>
{
if (tick != null)
if (tick != null)
{
Log.Debug($"{nameof(IEXDataQueueHandlerTests)}: tick: {tick}");
Log.Debug($"{nameof(IEXDataProviderTests)}: tick: {tick}");
}
},
() => {
},
() =>
{
isSubscribeThrowException = true;
_cancellationTokenSource.Cancel();
});
Expand All @@ -360,6 +365,60 @@ public void SubscribeWithWrongApiKeyThrowException()
Assert.IsTrue(isSubscribeThrowException);
}

[Test]
public void CanInitializeUsingJobPacket()
{
Config.Set("iex-cloud-api-key", "");

var job = new LiveNodePacket
{
BrokerageData = new Dictionary<string, string>() {
{ "iex-cloud-api-key", "InvalidApiKeyThatWontBeUsed" },
{ "iex-price-plan", "Launch" }
}
};

using var iexDataProvider = new IEXDataProvider();

Assert.Zero(iexDataProvider.maxAllowedSymbolLimit);

iexDataProvider.SetJob(job);

Assert.Greater(iexDataProvider.maxAllowedSymbolLimit, 0);

Config.Set("iex-cloud-api-key", _apiKey);
}

[Test]
public void JobPacketWontOverrideCredentials()
{
Config.Set("iex-cloud-api-key", "wrong_key");
Config.Set("iex-cloud-api-key", "Launch");

var job = new LiveNodePacket
{
BrokerageData = new Dictionary<string, string>() {
{ "iex-cloud-api-key", "InvalidApiKeyThatWontBeUsed" },
{ "iex-price-plan", "Enterprise" }
}
};

using var iexDataProvider = new IEXDataProvider();

var maxSymbolLimitBeforeSetJob = iexDataProvider.maxAllowedSymbolLimit;

// it has initialized already
Assert.Greater(maxSymbolLimitBeforeSetJob, 0);

iexDataProvider.SetJob(job);

// we use Enterprise plan in job variable => we must have unlimited maxAllowedSymbolLimit, but our config keep Launch
Assert.That(maxSymbolLimitBeforeSetJob, Is.EqualTo(iexDataProvider.maxAllowedSymbolLimit));

Config.Set("iex-cloud-api-key", _apiKey);
Config.Set("iex-cloud-api-key", _pricePlan);
}

private void ProcessFeed(IEnumerator<BaseData> enumerator, Action<BaseData> callback = null, Action throwExceptionCallback = null)
{
Task.Factory.StartNew(() =>
Expand All @@ -385,7 +444,7 @@ private void ProcessFeed(IEnumerator<BaseData> enumerator, Action<BaseData> call
}).ContinueWith(task =>
{
if (throwExceptionCallback != null)
{
{
throwExceptionCallback();
}
Log.Error("The throwExceptionCallback is null.");
Expand Down
10 changes: 5 additions & 5 deletions ...t.IEX.Tests/QuantConnect.IEX.Tests.csproj → .../QuantConnect.DataSource.IEX.Tests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
<IsPackable>false</IsPackable>
<TestProjectType>UnitTest</TestProjectType>
<OutputPath>bin\$(Configuration)\</OutputPath>
<Product>QuantConnect.IEX.Tests</Product>
<AssemblyName>QuantConnect.IEX.Tests</AssemblyName>
<RootNamespace>QuantConnect.IEX.Tests</RootNamespace>
<AssemblyTitle>QuantConnect.IEX.Tests</AssemblyTitle>
<Product>QuantConnect.Lean.DataSource.IEX.Tests</Product>
<AssemblyName>QuantConnect.Lean.DataSource.IEX.Tests</AssemblyName>
<RootNamespace>QuantConnect.Lean.DataSource.IEX.Tests</RootNamespace>
<AssemblyTitle>QuantConnect.Lean.DataSource.IEX.Tests</AssemblyTitle>
<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
</PropertyGroup>

Expand All @@ -26,7 +26,7 @@
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\QuantConnect.IEX\QuantConnect.IEX.csproj" />
<ProjectReference Include="..\QuantConnect.IEX\QuantConnect.DataSource.IEX.csproj" />
<ProjectReference Include="..\..\Lean\Tests\QuantConnect.Tests.csproj" />
</ItemGroup>

Expand Down
2 changes: 1 addition & 1 deletion QuantConnect.IEX.Tests/TestSetup.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
using QuantConnect.Logging;
using QuantConnect.Configuration;

namespace QuantConnect.IEX.Tests
namespace QuantConnect.Lean.DataSource.IEX.Tests
{
[SetUpFixture]
public class TestSetup
Expand Down
2 changes: 1 addition & 1 deletion QuantConnect.IEX/Constants/IEXDataStreamChannels.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
*
*/

namespace QuantConnect.IEX.Constants
namespace QuantConnect.Lean.DataSource.IEX.Constants
{
/// <summary>
/// Represents constants for accessing IEX Cloud data streams channels, specifically related to last sale and top-of-book information.
Expand Down
2 changes: 1 addition & 1 deletion QuantConnect.IEX/EnumeratorWrapper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
using QuantConnect.Data;
using System.Collections;

namespace QuantConnect.IEX
namespace QuantConnect.Lean.DataSource.IEX
{
/// <summary>
/// The new enumerator wrapper for this subscription request
Expand Down

0 comments on commit 0d365ad

Please sign in to comment.