Skip to content

Commit

Permalink
Migrate CoinApi Integration to Standalone Project (#1)
Browse files Browse the repository at this point in the history 
* feat: remove template files

* feat: prepare solution's structure
feat: github issue & PR templates
feat: prepare GH workflow file

* feat: paste coinApi files from ToolBox

* feat: paste coinApi.Converter files from ToolBox

* feat: paste coinApi.test files from ToolBox
feat: TestSetup initializer

* feat: divide HistoryProvider in file

* remove: does not support exchanges in SymbolMapper

* refactor: downgrade pckg Microsoft.NET.Test.Sdk

* feat: handle JsonSerializationException
feat: simplify code
feat: remove one warning msg

* feat: DQH tests with different param
feat: GetBrokerage CryptoFuture Symbol Test
feat: init with wrong api key test
feat: helper class for tests

* feat: forceTypeNameOnExisting set up to false for GetExportedValueByTypeName

* feat: add sync bash script in Converter project

* remove: unsupported markets from Converter

* feat: ValidateSubscription

* fix: productId in ValidateSubscription

* refactor: make static of TestHelper class
fix: deprecated GDAX to Coinbase Market in Symbol test

* feat: CoinAPIDataDownloader
fea: test of CoinAPIDataDownloader
fix: reset config in wrong api key test

* fix: handle exception when parsing response in History

* feat: update Readme

* Create LICENSE

* fix: reset config in test where we change config

* refactor: create RestClient at once time

* refactor: create RestRequest only once

* rename: Converter to DataProcessing

* refactor: test OneTimeSetUp to testing class

* refactor: increase delay in DQH tests

* fix: change delay and init DQH class tests

* refactor: GlobalSetup make static

* refactor: ProcessFeed in DQH tests

* feat: add some explicit and log trace

* refactor: validation on null tick
remove: thread sleep

* feat: add delay in ProcessFeed by cancellationToken
refactor: future test

* fix: tick symbol in CryptoFuture test

* remove: Explicit attribute in tests
  • Loading branch information
@Romazes
Romazes committed Feb 14, 2024
1 parent ad7e7f9 commit 89db062
Show file tree
Hide file tree
Showing 56 changed files with 3,236 additions and 1,373 deletions.
33 changes: 33 additions & 0 deletions .github/ISSUE_TEMPLATE.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
<!--- Provide a general summary of the issue in the Title above -->

## Expected Behavior
<!--- If you're describing a bug, tell us what should happen -->
<!--- If you're suggesting a change/improvement, tell us how it should work -->

## Current Behavior
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
<!--- If suggesting a change/improvement, explain the difference from current behavior -->

## Possible Solution
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
<!--- or ideas how to implement the addition or change -->

## Steps to Reproduce (for bugs)
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
<!--- reproduce this bug. Include code to reproduce, if relevant -->
1.
2.
3.
4.

## Context
<!--- How has this issue affected you? What are you trying to accomplish? -->
<!--- Providing context helps us come up with a solution that is most useful in the real world -->

## Your Environment
<!--- Include as many relevant details about the environment you experienced the bug in -->
* Version used:
* Environment name and version (e.g. PHP 5.4 on nginx 1.9.1):
* Server type and version:
* Operating System and version:
* Link to your project:
31 changes: 31 additions & 0 deletions .github/PULL_REQUEST_TEMPLATE.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
<!--- Provide a general summary of your changes in the Title above -->

## Description
<!--- Describe your changes in detail -->

## Motivation and Context
<!--- Why is this change required? What problem does it solve? -->
<!--- If it fixes an open issue, please link to the issue here. -->

## How Has This Been Tested?
<!--- Please describe in detail how you tested your changes. -->
<!--- Include details of your testing environment, and the tests you ran to -->
<!--- see how your change affects other areas of the code, etc. -->

## Screenshots (if appropriate):

## Types of changes
<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to change)

## Checklist:
<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
- [ ] My code follows the code style of this project.
- [ ] My change requires a change to the documentation.
- [ ] I have updated the documentation accordingly.
- [ ] I have read the **CONTRIBUTING** document.
- [ ] I have added tests to cover my changes.
- [ ] All new and existing tests passed.
35 changes: 22 additions & 13 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,18 @@ on:
jobs:
build:
runs-on: ubuntu-20.04

env:
QC_JOB_USER_ID: ${{ secrets.QC_JOB_USER_ID }}
QC_API_ACCESS_TOKEN: ${{ secrets.QC_API_ACCESS_TOKEN }}
QC_JOB_ORGANIZATION_ID: ${{ secrets.QC_JOB_ORGANIZATION_ID }}
QC_COINAPI_API_KEY: ${{ secrets.QC_COINAPI_API_KEY }}
steps:
- uses: actions/checkout@v2
- name: Checkout
uses: actions/checkout@v2

- name: Free space
run: df -h && rm -rf /opt/hostedtoolcache* && df -h

- name: Pull Foundation Image
uses: addnab/docker-run-action@v3
with:
image: quantconnect/lean:foundation

- name: Checkout Lean Same Branch
id: lean-same-branch
uses: actions/checkout@v2
Expand All @@ -40,11 +40,20 @@ jobs:
- name: Move Lean
run: mv Lean ../Lean

- name: BuildDataSource
run: dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
- name: Pull Foundation Image
uses: addnab/docker-run-action@v3
with:
image: quantconnect/lean:foundation
options: -v /home/runner/work:/__w --workdir /__w/Lean.DataSource.CoinAPI/Lean.DataSource.CoinAPI -e QC_JOB_USER_ID=${{ secrets.QC_JOB_USER_ID }} -e QC_API_ACCESS_TOKEN=${{ secrets.QC_API_ACCESS_TOKEN }} -e QC_JOB_ORGANIZATION_ID=${{ secrets.QC_JOB_ORGANIZATION_ID }} -e QC_COINAPI_API_KEY=${{ secrets.QC_COINAPI_API_KEY }}

- name: Build QuantConnect.CoinAPI
run: dotnet build ./QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1

- name: Build DataProcessing
run: dotnet build ./DataProcessing/DataProcessing.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1

- name: BuildTests
run: dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
- name: Build QuantConnect.CoinAPI.Tests
run: dotnet build ./QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1

- name: Run Tests
run: dotnet test ./tests/bin/Release/net6.0/Tests.dll
- name: Run QuantConnect.CoinAPI.Tests
run: dotnet test ./QuantConnect.CoinAPI.Tests/bin/Release/QuantConnect.CoinAPI.Tests.dll
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
*.hex

# QC Cloud Setup Bash Files
!DataProcessing/*.sh
*.sh
# Include docker launch scripts for Mac/Linux
!run_docker.sh
Expand Down
12 changes: 0 additions & 12 deletions DataProcessing/CLRImports.py
View file

This file was deleted.

255 changes: 255 additions & 0 deletions DataProcessing/CoinApiDataConverter.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,255 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
*
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using QuantConnect.Data;
using QuantConnect.Util;
using System.Diagnostics;
using QuantConnect.Logging;
using QuantConnect.ToolBox;
using QuantConnect.CoinAPI;

namespace QuantConnect.DataProcessing
{
/// <summary>
/// Console application for converting CoinApi raw data into Lean data format for high resolutions (tick, second and minute)
/// </summary>
public class CoinApiDataConverter
{
/// <summary>
/// List of supported exchanges
/// </summary>
private static readonly HashSet<string> SupportedMarkets = new[]
{
Market.Coinbase,
Market.Bitfinex,
Market.Binance,
Market.Kraken,
Market.BinanceUS
}.ToHashSet();

private readonly DirectoryInfo _rawDataFolder;
private readonly DirectoryInfo _destinationFolder;
private readonly SecurityType _securityType;
private readonly DateTime _processingDate;
private readonly string _market;

/// <summary>
/// CoinAPI data converter.
/// </summary>
/// <param name="date">the processing date.</param>
/// <param name="rawDataFolder">path to the raw data folder.</param>
/// <param name="destinationFolder">destination of the newly generated files.</param>
/// <param name="securityType">The security type to process</param>
/// <param name="market">The market to process (optional). Defaults to processing all markets in parallel.</param>
public CoinApiDataConverter(DateTime date, string rawDataFolder, string destinationFolder, string market = null, SecurityType securityType = SecurityType.Crypto)
{
_market = string.IsNullOrWhiteSpace(market)
? null
: market.ToLowerInvariant();

_processingDate = date;
_securityType = securityType;
_rawDataFolder = new DirectoryInfo(Path.Combine(rawDataFolder, "crypto", "coinapi"));
if (!_rawDataFolder.Exists)
{
throw new ArgumentException($"CoinApiDataConverter(): Source folder not found: {_rawDataFolder.FullName}");
}

_destinationFolder = new DirectoryInfo(destinationFolder);
_destinationFolder.Create();
}

/// <summary>
/// Runs this instance.
/// </summary>
/// <returns></returns>
public bool Run()
{
var stopwatch = Stopwatch.StartNew();

var symbolMapper = new CoinApiSymbolMapper();
var success = true;

// There were cases of files with with an extra suffix, following pattern:
// <TickType>-<ID>-<Exchange>_SPOT_<BaseCurrency>_<QuoteCurrency>_<ExtraSuffix>.csv.gz
// Those cases should be ignored for SPOT prices.
var tradesFolder = new DirectoryInfo(
Path.Combine(
_rawDataFolder.FullName,
"trades",
_processingDate.ToStringInvariant(DateFormat.EightCharacter)));

var quotesFolder = new DirectoryInfo(
Path.Combine(
_rawDataFolder.FullName,
"quotes",
_processingDate.ToStringInvariant(DateFormat.EightCharacter)));

var rawMarket = _market != null &&
CoinApiSymbolMapper.MapMarketsToExchangeIds.TryGetValue(_market, out var rawMarketValue)
? rawMarketValue
: null;

var securityTypeFilter = (string name) => name.Contains("_SPOT_");
if (_securityType == SecurityType.CryptoFuture)
{
securityTypeFilter = (string name) => name.Contains("_FTS_") || name.Contains("_PERP_");
}

// Distinct by tick type and first two parts of the raw file name, separated by '-'.
// This prevents us from double processing the same ticker twice, in case we're given
// two raw data files for the same symbol. Related: https://github.com/QuantConnect/Lean/pull/3262
var apiDataReader = new CoinApiDataReader(symbolMapper);
var filesToProcessCandidates = tradesFolder.EnumerateFiles("*.gz")
.Concat(quotesFolder.EnumerateFiles("*.gz"))
.Where(f => securityTypeFilter(f.Name) && (rawMarket == null || f.Name.Contains(rawMarket)))
.Where(f => f.Name.Split('_').Length == 4)
.ToList();

var filesToProcessKeys = new HashSet<string>();
var filesToProcess = new List<FileInfo>();

foreach (var candidate in filesToProcessCandidates)
{
try
{
var entryData = apiDataReader.GetCoinApiEntryData(candidate, _processingDate, _securityType);
CurrencyPairUtil.DecomposeCurrencyPair(entryData.Symbol, out var baseCurrency, out var quoteCurrency);

if (!candidate.FullName.Contains(baseCurrency) && !candidate.FullName.Contains(quoteCurrency))
{
throw new Exception($"Skipping {candidate.FullName} we have the wrong symbol {entryData.Symbol}!");
}

var key = candidate.Directory.Parent.Name + entryData.Symbol.ID;
if (filesToProcessKeys.Add(key))
{
// Separate list from HashSet to preserve ordering of viable candidates
filesToProcess.Add(candidate);
}
}
catch (Exception err)
{
// Most likely the exchange isn't supported. Log exception message to avoid excessive stack trace spamming in console output
Log.Error(err.Message);
}
}

Parallel.ForEach(filesToProcess, (file, loopState) =>
{
Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}...");
try
{
ProcessEntry(apiDataReader, file);
}
catch (Exception e)
{
Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}");
success = false;
loopState.Break();
}
}
);

Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}");
return success;
}

/// <summary>
/// Processes the entry.
/// </summary>
/// <param name="coinapiDataReader">The coinapi data reader.</param>
/// <param name="file">The file.</param>
private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file)
{
var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate, _securityType);

if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market))
{
// only convert data for supported exchanges
return;
}

var tickData = coinapiDataReader.ProcessCoinApiEntry(entryData, file);

// in some cases the first data points from '_processingDate' get's included in the previous date file
// so we will ready previous date data and drop most of it just to save these midnight ticks
var yesterdayDate = _processingDate.AddDays(-1);
var yesterdaysFile = new FileInfo(file.FullName.Replace(
_processingDate.ToStringInvariant(DateFormat.EightCharacter),
yesterdayDate.ToStringInvariant(DateFormat.EightCharacter)));
if (yesterdaysFile.Exists)
{
var yesterdaysEntryData = coinapiDataReader.GetCoinApiEntryData(yesterdaysFile, yesterdayDate, _securityType);
tickData = tickData.Concat(coinapiDataReader.ProcessCoinApiEntry(yesterdaysEntryData, yesterdaysFile));
}
else
{
Log.Error($"CoinApiDataConverter(): yesterdays data file not found '{yesterdaysFile.FullName}'");
}

// materialize the enumerable into a list, since we need to enumerate over it twice
var ticks = tickData.Where(tick => tick.Time.Date == _processingDate)
.OrderBy(t => t.Time)
.ToList();

var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType);
writer.Write(ticks);

Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}");
var consolidators = new List<TickAggregator>();

if (entryData.TickType == TickType.Trade)
{
consolidators.AddRange(new[]
{
new TradeTickAggregator(Resolution.Second),
new TradeTickAggregator(Resolution.Minute)
});
}
else
{
consolidators.AddRange(new[]
{
new QuoteTickAggregator(Resolution.Second),
new QuoteTickAggregator(Resolution.Minute)
});
}

foreach (var tick in ticks)
{
if (tick.Suspicious)
{
// When CoinAPI loses connectivity to the exchange, they indicate
// it in the data by providing a value of `-1` for bid/ask price.
// We will keep it in tick data, but will remove it from consolidated data.
continue;
}

foreach (var consolidator in consolidators)
{
consolidator.Update(tick);
}
}

foreach (var consolidator in consolidators)
{
writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType);
writer.Write(consolidator.Flush());
}
}
}
}

0 comments on commit 89db062

Please sign in to comment.