Minor fix for QC data download (#454)

* Minor fix for QC data download

* More fixes and improvements

README.md

@@ -880,16 +880,16 @@ Options:
   --dataset TEXT                  The name of the dataset to download non-interactively
   --overwrite                     Overwrite existing local data
   -y, --yes                       Automatically confirm payment confirmation prompts
-  --data-type [Trade|Quote|OpenInterest]
+  --data-type [Trade|Quote|Bulk|OpenInterest]
                                   Specify the type of historical data
   --resolution [Tick|Second|Minute|Hour|Daily]
                                   Specify the resolution of the historical data
   --security-type [Equity|Index|Forex|Cfd|Future|Crypto|CryptoFuture|Option|IndexOption|Commodity|FutureOption]
                                   Specify the security type of the historical data
   --market TEXT                   Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance')
-  --tickers TEXT                  Specify comma separated list of tickers to use for historical data request.
-  --start-date TEXT               Specify the start date for the historical data request in the format yyyyMMdd.
-  --end-date TEXT                 Specify the end date for the historical data request in the format yyyyMMdd. (defaults
+  --ticker TEXT                   Specify comma separated list of tickers to use for historical data request.
+  --start TEXT                    Specify the start date for the historical data request in the format yyyyMMdd.
+  --end TEXT                      Specify the end date for the historical data request in the format yyyyMMdd. (defaults
                                   to today)
   --image TEXT                    The LEAN engine image to use (defaults to quantconnect/lean:latest)
   --update                        Pull the LEAN engine image before running the Downloader Data Provider

lean/commands/data/download.py

@@ -352,7 +352,8 @@ def _select_products_non_interactive(organization: QCFullOrganization,
         if option.condition is not None and not option.condition.check(option_results):
             continue
 
-        user_input = ctx.params.get(option.id, None)
+        # if the option id has a '-' in its name, and it's a click option, in the click context it's available with '_'
+        user_input = ctx.params.get(option.id.replace('-', '_'), ctx.params.get(option.id, None))
 
         if user_input is None:
             missing_options.append(f"--{option.id} <{option.get_placeholder()}>: {option.description}")
@@ -512,13 +513,13 @@ def _configure_date_option(date_value: str, option_id: str, option_label: str) -
     help="Specify the security type of the historical data")
 @option("--market", type=str,
         help="Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance')")
-@option("--tickers",
+@option("--ticker",
         type=str,
         help="Specify comma separated list of tickers to use for historical data request.")
-@option("--start-date",
+@option("--start",
         type=str,
         help="Specify the start date for the historical data request in the format yyyyMMdd.")
-@option("--end-date",
+@option("--end",
         type=str,
         help="Specify the end date for the historical data request in the format yyyyMMdd. (defaults to today)")
 @option("--image",
@@ -543,9 +544,9 @@ def download(ctx: Context,
              resolution: Optional[str],
              security_type: Optional[str],
              market: Optional[str],
-             tickers: Optional[str],
-             start_date: Optional[str],
-             end_date: Optional[str],
+             ticker: Optional[str],
+             start: Optional[str],
+             end: Optional[str],
              image: Optional[str],
              update: bool,
              no_update: bool,
@@ -575,6 +576,9 @@ def download(ctx: Context,
     """
     organization = _get_organization()
 
+    if dataset:
+        data_provider_historical = 'QuantConnect'
+
     if data_provider_historical is None:
         data_provider_historical = _get_historical_data_provider()
 
@@ -626,17 +630,21 @@ def download(ctx: Context,
         market = _get_user_input_or_prompt(market, data_provider_support_markets,
                                            data_provider_historical, "Select a Market")
 
-        if not tickers:
-            tickers = ','.join(DatasetTextOption(id="id",
-                               label="Enter comma separated list of tickers to use for historical data request.",
-                               description="description",
-                               transform=DatasetTextOptionTransform.Lowercase,
-                               multiple=True).configure_interactive().value)
+        if not ticker:
+            ticker = ','.join(DatasetTextOption(id="id",
+                                                label="Enter comma separated list of tickers to use for historical data request.",
+                                                description="description",
+                                                transform=DatasetTextOptionTransform.Uppercase,
+                                                multiple=True).configure_interactive().value)
+        else:
+            split_tickers = ticker.split(',')
+            # don't trust user provider tickers without spaces in between
+            ticker = ','.join([a_ticker.strip().upper() for a_ticker in split_tickers])
 
-        start_date = _configure_date_option(start_date, "start", "Please enter a Start Date in the format")
-        end_date = _configure_date_option(end_date, "end", "Please enter a End Date in the format")
+        start = _configure_date_option(start, "start", "Please enter a Start Date in the format")
+        end = _configure_date_option(end, "end", "Please enter a End Date in the format")
 
-        if start_date.value >= end_date.value:
+        if start.value >= end.value:
             raise ValueError("Historical start date cannot be greater than or equal to historical end date.")
 
         logger = container.logger
@@ -674,12 +682,12 @@ def download(ctx: Context,
 
         dll_arguments = ["dotnet", "QuantConnect.DownloaderDataProvider.Launcher.dll",
                          "--data-type", data_type,
-                         "--start-date", start_date.value.strftime("%Y%m%d"),
-                         "--end-date", end_date.value.strftime("%Y%m%d"),
+                         "--start-date", start.value.strftime("%Y%m%d"),
+                         "--end-date", end.value.strftime("%Y%m%d"),
                          "--security-type", security_type,
                          "--market", market,
                          "--resolution", resolution,
-                         "--tickers", tickers]
+                         "--tickers", ticker]
 
         run_options["commands"].append(' '.join(dll_arguments))
 

lean/models/api.py

@@ -410,6 +410,7 @@ class QCMinimalOrganization(WrappedBaseModel):
 class QCDataType(str, Enum):
     Trade = "Trade"
     Quote = "Quote"
+    Bulk = "Bulk"
     OpenInterest = "OpenInterest"
 
     @classmethod

tests/commands/data/test_download.py

@@ -123,9 +123,9 @@ def _create_lean_data_download(data_provider_name: str,
                     "--data-type", data_type,
                     "--resolution", resolution,
                     "--security-type", security_type,
-                    "--tickers", ','.join(tickers),
-                    "--start-date", start_date,
-                    "--end-date", end_date,
+                    "--ticker", ','.join(tickers),
+                    "--start", start_date,
+                    "--end", end_date,
                 ]
                 if market:
                     run_parameters.extend(["--market", market])
@@ -135,7 +135,7 @@ def _create_lean_data_download(data_provider_name: str,
                 return CliRunner().invoke(lean, run_parameters)
 
 
-@pytest.mark.parametrize("data_provider,market,is_crypto,security_type,tickers,data_provider_parameters",
+@pytest.mark.parametrize("data_provider,market,is_crypto,security_type,ticker,data_provider_parameters",
                          [("Polygon", "NYSE", False, "Equity", ["AAPL"], ["--polygon-api-key", "123"]),
                           ("Binance", "Binance", True, "CryptoFuture", ["BTCUSDT"],
                            ["--binance-exchange-name", "BinanceUS", "--binanceus-api-key", "123",
@@ -145,9 +145,9 @@ def _create_lean_data_download(data_provider_name: str,
                           ("Interactive Brokers", "USA", False, "Index", ["INTL", "NVDA"],
                            ["--ib-user-name", "123", "--ib-account", "Individual", "--ib-password", "123"])])
 def test_download_data_non_interactive(data_provider: str, market: str, is_crypto: bool, security_type: str,
-                                       tickers: List[str], data_provider_parameters: List[str]):
+                                       ticker: List[str], data_provider_parameters: List[str]):
     run_data_download = _create_lean_data_download(
-        data_provider, "Trade", "Minute", security_type, tickers, "20240101", "20240202",
+        data_provider, "Trade", "Minute", security_type, ticker, "20240101", "20240202",
         _get_data_provider_config(is_crypto), market, data_provider_parameters)
     assert run_data_download.exit_code == 0
 
@@ -174,11 +174,11 @@ def test_download_data_non_interactive_wrong_security_type(data_provider: str, w
     assert wrong_security_type in error_msg
 
 
-@pytest.mark.parametrize("data_provider,start_date,end_date",
+@pytest.mark.parametrize("data_provider,start,end",
                          [("Polygon", "20240101", "20230202"), ("Polygon", "2024-01-01", "2023-02-02")])
-def test_download_data_non_interactive_wrong_start_end_date(data_provider: str, start_date: str, end_date: str):
-    run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", "Equity", ["AAPL"], start_date,
-                                                   end_date, _get_data_provider_config(), "USA",
+def test_download_data_non_interactive_wrong_start_end_date(data_provider: str, start: str, end: str):
+    run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", "Equity", ["AAPL"], start,
+                                                   end, _get_data_provider_config(), "USA",
                                                    extra_run_command=["--polygon-api-key", "123"])
     assert run_data_download.exit_code == 1