Merge pull request #442 from jhonabreul/feature-factset-support

Json modules path configs support

README.md

@@ -143,7 +143,7 @@ Options:
   -d, --detach                    Run the backtest in a detached Docker container and return immediately
   --debug [pycharm|ptvsd|vsdbg|rider|local-platform]
                                   Enable a certain debugging method (see --help for more information)
-  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link]
+  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link]
                                   Update the Lean configuration file to retrieve data from the given historical provider
   --binance-exchange-name [Binance|BinanceUS|Binance-USDM-Futures|Binance-COIN-Futures]
                                   Binance exchange name [Binance, BinanceUS, Binance-USDM-Futures, Binance-COIN-Futures]
@@ -161,6 +161,8 @@ Options:
   --iqfeed-version TEXT           The product version of your IQFeed developer account
   --iqfeed-host TEXT              The IQFeed host address
   --polygon-api-key TEXT          Your Polygon.io API Key
+  --factset-auth-config-file FILE
+                                  The path to the FactSet authentication configuration file
   --iex-cloud-api-key TEXT        Your iexcloud.io API token publishable key
   --iex-price-plan [Launch|Grow|Enterprise]
                                   Your IEX Cloud Price plan
@@ -1081,7 +1083,7 @@ Options:
                                   The brokerage to use
   --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|IQFeed|Polygon|IEX|CoinApi|Custom data only|Bybit]
                                   The live data provider to use
-  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local]
+  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|QuantConnect|Local]
                                   Update the Lean configuration file to retrieve data from the given historical provider
   --ib-user-name TEXT             Your Interactive Brokers username
   --ib-account TEXT               Your Interactive Brokers account id
@@ -1194,6 +1196,8 @@ Options:
   --coinapi-api-key TEXT          Your coinapi.io Api Key
   --coinapi-product [Free|Startup|Streamer|Professional|Enterprise]
                                   CoinApi pricing plan (https://www.coinapi.io/market-data-api/pricing)
+  --factset-auth-config-file FILE
+                                  The path to the FactSet authentication configuration file
   --alpha-vantage-api-key TEXT    Your Alpha Vantage Api Key
   --alpha-vantage-price-plan [Free|Plan30|Plan75|Plan150|Plan300|Plan600|Plan1200]
                                   Your Alpha Vantage Premium API Key plan
@@ -1486,7 +1490,7 @@ Options:
   --parameter <TEXT FLOAT FLOAT FLOAT>...
                                   The 'parameter min max step' pairs configuring the parameters to optimize
   --constraint TEXT               The 'statistic operator value' pairs configuring the constraints of the optimization
-  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link]
+  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link]
                                   Update the Lean configuration file to retrieve data from the given historical provider
   --download-data                 Update the Lean configuration file to download data from the QuantConnect API, alias
                                   for --data-provider-historical QuantConnect
@@ -1517,6 +1521,8 @@ Options:
   --iqfeed-version TEXT           The product version of your IQFeed developer account
   --iqfeed-host TEXT              The IQFeed host address
   --polygon-api-key TEXT          Your Polygon.io API Key
+  --factset-auth-config-file FILE
+                                  The path to the FactSet authentication configuration file
   --iex-cloud-api-key TEXT        Your iexcloud.io API token publishable key
   --iex-price-plan [Launch|Grow|Enterprise]
                                   Your IEX Cloud Price plan
@@ -1641,7 +1647,7 @@ Usage: lean research [OPTIONS] PROJECT
 
 Options:
   --port INTEGER                  The port to run Jupyter Lab on (defaults to 8888)
-  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link]
+  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link]
                                   Update the Lean configuration file to retrieve data from the given historical provider
   --binance-exchange-name [Binance|BinanceUS|Binance-USDM-Futures|Binance-COIN-Futures]
                                   Binance exchange name [Binance, BinanceUS, Binance-USDM-Futures, Binance-COIN-Futures]
@@ -1659,6 +1665,8 @@ Options:
   --iqfeed-version TEXT           The product version of your IQFeed developer account
   --iqfeed-host TEXT              The IQFeed host address
   --polygon-api-key TEXT          Your Polygon.io API Key
+  --factset-auth-config-file FILE
+                                  The path to the FactSet authentication configuration file
   --iex-cloud-api-key TEXT        Your iexcloud.io API token publishable key
   --iex-price-plan [Launch|Grow|Enterprise]
                                   Your IEX Cloud Price plan

lean/commands/backtest.py

@@ -352,12 +352,14 @@ def backtest(project: Path,
         data_provider_historical = "QuantConnect"
 
     organization_id = container.organization_manager.try_get_working_organization_id()
+    paths_to_mount = None
 
     if data_provider_historical is not None:
         data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
                                                               cli_data_downloaders, kwargs, logger, environment_name)
         data_provider.ensure_module_installed(organization_id)
         container.lean_config_manager.set_properties(data_provider.get_settings())
+        paths_to_mount = data_provider.get_paths_to_mount()
 
     lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit)
 
@@ -406,4 +408,5 @@ def backtest(project: Path,
                          debugging_method,
                          release,
                          detach,
-                         loads(extra_docker_config))
+                         loads(extra_docker_config),
+                         paths_to_mount)

lean/commands/live/deploy.py

@@ -282,9 +282,11 @@ def deploy(project: Path,
                                                                  environment_name=environment_name))
 
     organization_id = container.organization_manager.try_get_working_organization_id()
+    paths_to_mount = {}
     for module in (data_provider_live_instances + [data_downloader_instances, brokerage_instance]
                    + history_providers_instances):
         module.ensure_module_installed(organization_id)
+        paths_to_mount.update(module.get_paths_to_mount())
 
     if not lean_config["environments"][environment_name]["live-mode"]:
         raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)",
@@ -362,4 +364,13 @@ def deploy(project: Path,
             raise RuntimeError(f"InteractiveBrokers is currently not supported for ARM hosts")
 
     lean_runner = container.lean_runner
-    lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config))
+    lean_runner.run_lean(lean_config,
+                         environment_name,
+                         algorithm_file,
+                         output,
+                         engine_image,
+                         None,
+                         release,
+                         detach,
+                         loads(extra_docker_config),
+                         paths_to_mount)

lean/commands/optimize.py

@@ -305,11 +305,14 @@ def optimize(project: Path,
     if download_data:
         data_provider_historical = "QuantConnect"
 
+    paths_to_mount = None
+
     if data_provider_historical is not None:
         data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
                                                               cli_data_downloaders, kwargs, logger, environment_name)
         data_provider.ensure_module_installed(organization_id)
         container.lean_config_manager.set_properties(data_provider.get_settings())
+        paths_to_mount = data_provider.get_paths_to_mount()
 
     lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit)
 
@@ -339,7 +342,7 @@ def optimize(project: Path,
     container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)
 
     run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach,
-                                                      engine_image)
+                                                      engine_image, paths_to_mount)
 
     run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
     run_options["commands"].append(f"dotnet QuantConnect.Optimizer.Launcher.dll{' --estimate' if estimate else ''}")

lean/commands/research.py

@@ -113,12 +113,15 @@ def research(project: Path,
     if download_data:
         data_provider_historical = "QuantConnect"
 
+    paths_to_mount = None
+
     if data_provider_historical is not None:
         organization_id = container.organization_manager.try_get_working_organization_id()
         data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
                                                               cli_data_downloaders, kwargs, logger, environment_name)
         data_provider.ensure_module_installed(organization_id)
         container.lean_config_manager.set_properties(data_provider.get_settings())
+        paths_to_mount = data_provider.get_paths_to_mount()
     lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit)
 
     lean_runner = container.lean_runner
@@ -145,7 +148,8 @@ def research(project: Path,
                                                       None,
                                                       False,
                                                       detach,
-                                                      research_image)
+                                                      research_image,
+                                                      paths_to_mount)
 
     # Mount the config in the notebooks directory as well
     local_config_path = next(m["Source"] for m in run_options["mounts"] if m["Target"].endswith("config.json"))

lean/components/docker/lean_runner.py

@@ -73,7 +73,8 @@ def run_lean(self,
                  debugging_method: Optional[DebuggingMethod],
                  release: bool,
                  detach: bool,
-                 extra_docker_config: Optional[Dict[str, Any]] = None) -> None:
+                 extra_docker_config: Optional[Dict[str, Any]] = None,
+                 paths_to_mount: Optional[Dict[str, str]] = None) -> None:
         """Runs the LEAN engine locally in Docker.
 
         Raises an error if something goes wrong.
@@ -87,6 +88,7 @@ def run_lean(self,
         :param release: whether C# projects should be compiled in release configuration instead of debug
         :param detach: whether LEAN should run in a detached container
         :param extra_docker_config: additional docker configurations
+        :param paths_to_mount: additional paths to mount to the container
         """
         self._logger.debug(f'LeanRunner().run_lean: lean_config: {lean_config}')
         project_dir = algorithm_file.parent
@@ -99,7 +101,8 @@ def run_lean(self,
                                                    debugging_method,
                                                    release,
                                                    detach,
-                                                   image)
+                                                   image,
+                                                   paths_to_mount)
 
         # Add known additional run options from the extra docker config
         self.parse_extra_docker_config(run_options, extra_docker_config)
@@ -178,7 +181,8 @@ def get_basic_docker_config(self,
                                 debugging_method: Optional[DebuggingMethod],
                                 release: bool,
                                 detach: bool,
-                                image: DockerImage) -> Dict[str, Any]:
+                                image: DockerImage,
+                                paths_to_mount: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
         """Creates a basic Docker config to run the engine with.
 
         This method constructs the parts of the Docker config that is the same for both the engine and the optimizer.
@@ -191,6 +195,7 @@ def get_basic_docker_config(self,
         :param detach: whether LEAN should run in a detached container
         :param image: The docker image that will be used
         :return: the Docker configuration containing basic configuration to run Lean
+        :param paths_to_mount: additional paths to mount to the container
         """
         from docker.types import Mount
         from uuid import uuid4
@@ -240,6 +245,9 @@ def get_basic_docker_config(self,
             "ports": docker_project_config.get("ports", {})
         }
 
+        # mount the paths passed in
+        self.mount_paths(paths_to_mount, lean_config, run_options)
+
         # mount the project and library directories
         self.mount_project_and_library_directories(project_dir, run_options)
 
@@ -782,6 +790,30 @@ def mount_project_and_library_directories(self, project_dir: Path, run_options:
                 "mode": "rw"
             }
 
+    def mount_paths(self, paths_to_mount, lean_config, run_options):
+        if not paths_to_mount:
+            return
+
+        from docker.types import Mount
+
+        environment = {}
+        if "environment" in lean_config and "environments" in lean_config:
+            environment = lean_config["environments"][lean_config["environment"]]
+
+        mounts = run_options["mounts"]
+
+        for key, pathStr in paths_to_mount.items():
+            path = Path(pathStr).resolve()
+            target = f"/Files/{Path(path).name}"
+
+            self._logger.info(f"Mounting {path} to {target}")
+
+            mounts.append(Mount(target=target,
+                                source=str(path),
+                                type="bind",
+                                read_only=True))
+            environment[key] = target
+
     @staticmethod
     def parse_extra_docker_config(run_options: Dict[str, Any], extra_docker_config: Optional[Dict[str, Any]]) -> None:
         from docker.types import DeviceRequest

lean/models/json_module.py

@@ -20,7 +20,8 @@
 from lean.components.util.logger import Logger
 from lean.constants import MODULE_TYPE, MODULE_PLATFORM, MODULE_CLI_PLATFORM
 from lean.container import container
-from lean.models.configuration import BrokerageEnvConfiguration, Configuration, InternalInputUserInput
+from lean.models.configuration import BrokerageEnvConfiguration, Configuration, InternalInputUserInput, \
+    PathParameterUserInput
 from copy import copy
 from abc import ABC
 
@@ -228,6 +229,12 @@ def config_build(self,
                                .join(missing_options)}""".strip())
         return self
 
+    def get_paths_to_mount(self) -> Dict[str, str]:
+        return {config._id: config._value
+                for config in self._lean_configs
+                if (isinstance(config, PathParameterUserInput)
+                    and self._check_if_config_passes_filters(config, all_for_platform_type=False))}
+
     def ensure_module_installed(self, organization_id: str) -> None:
         if not self._is_module_installed and self._installs:
             container.logger.debug(f"JsonModule.ensure_module_installed(): installing module {self}: {self._product_id}")