[DRAFT] POC to add WebAssembly Based Plugin System to OpenBB Terminal

openbb_terminal/stocks/backtesting/bt_controller.py

@@ -43,6 +43,7 @@ def no_data_message():
 class BacktestingController(StockBaseController):
     """Backtesting Controller class"""
 
+    CHOICES_MENUS = ["extism"]
     CHOICES_COMMANDS = ["load", "ema", "emacross", "rsi", "whatif"]
     PATH = "/stocks/bt/"
     CHOICES_GENERATION = True
@@ -73,6 +74,8 @@ def print_help(self):
         mt.add_cmd("ema", self.ticker)
         mt.add_cmd("emacross", self.ticker)
         mt.add_cmd("rsi", self.ticker)
+        mt.add_raw("\n")
+        mt.add_menu("extism")
         console.print(text=mt.menu_text, menu="Stocks - Backtesting")
 
     def custom_reset(self):
@@ -81,6 +84,15 @@ def custom_reset(self):
             return ["stocks", f"load {self.ticker}", "bt"]
         return []
 
+    @log_start_end(log=logger)
+    def call_extism(self, _):
+        """Process bt command."""
+        from openbb_terminal.stocks.backtesting.extism_plugins import extism_controller
+
+        self.queue = self.load_class(
+            extism_controller.ExtismController, self.ticker, self.stock, self.queue
+        )
+
     @log_start_end(log=logger)
     def call_whatif(self, other_args: List[str]):
         """Call whatif"""

openbb_terminal/stocks/backtesting/extism_plugins/extism_controller.py

@@ -0,0 +1,201 @@
+"""Extism Controller Module"""
+__docformat__ = "numpy"
+
+import argparse
+import logging
+from typing import List, Optional, Annotated
+
+import matplotlib as mpl
+import pandas as pd
+import pandas_ta as ta
+
+import extism
+from extism import Plugin, host_fn, Json
+
+import json
+
+from openbb_terminal.core.session.current_user import get_current_user
+from openbb_terminal.custom_prompt_toolkit import NestedCompleter
+from openbb_terminal.decorators import log_start_end
+from openbb_terminal.helper_funcs import (
+    EXPORT_ONLY_RAW_DATA_ALLOWED,
+    check_non_negative_float,
+    check_positive,
+    valid_date,
+)
+from openbb_terminal.menu import session
+from openbb_terminal.parent_classes import StockBaseController
+from openbb_terminal.rich_config import MenuText, console
+
+# This code below aims to fix an issue with the fnn module, used by bt module
+# which forces matplotlib backend to be 'agg' which doesn't allow to plot
+# Save current matplotlib backend
+default_backend = mpl.get_backend()
+# Restore backend matplotlib used
+
+# pylint: disable=wrong-import-position
+from openbb_terminal.stocks.backtesting.extism_plugins import extism_view  # noqa: E402
+
+logger = logging.getLogger(__name__)
+
+mpl.use(default_backend)
+
+def ema(data, params):
+    length = params['periods']
+    ema = ta.ema(data, length)
+    return ema
+
+def get_frame(req):
+    df = pd.DataFrame({'prices': req['prices']})
+    df.index = pd.to_datetime(pd.to_numeric(df.index), unit='ms')
+    df['prices'] = pd.to_numeric(df['prices'], errors='coerce')
+    return df
+
+def make_response(ema_result):
+    # format for returning response to the plugin
+    ema_result.name = 'data'
+    ema_frame = ema_result.to_frame()
+    ema_frame.fillna(0, inplace=True)
+    ema_frame.index = ema_frame.index.date
+    json_response = ema_frame.to_json()
+    return json_response
+
+def handle_request(req):
+    df = get_frame(req)
+    params = json.loads(req['params'])
+
+    if req['name'] == 'ema':
+        ema_result = ema(df['prices'], params)
+        json_response = make_response(ema_result)
+        return json_response
+
+    if req['name'] == 'rsi':
+        rsi_result = rsi(df['prices'], params)
+        json_response = make_response(rsi_result)
+        return json_response
+
+    return nil
+
+# setup our Host Function for the plugin to request various technical indicators
+@host_fn()
+def get_ta(input: Annotated[dict, Json]) -> str:
+    req = input
+    rep = handle_request(input)
+    return rep
+
+# configure strategy backed by an Extism plugin loaded from a URL
+plugin_manifests = {
+    "ema": { "wasm": [{"url": "https://cdn.modsurfer.dylibso.com/api/v1/module/e5bad87f199010c70e7644654c4bb6ce0fa37b66f03343b1bf0d1fddbb05e12e.wasm"}]},
+}
+
+# a little plugin registry
+plugins = {}
+
+# instantiate plugins and load them in the registry
+for name,manifest in plugin_manifests.items():
+    plugins[name] = Plugin(manifest)
+
+
+def no_data_message():
+    """Print message when no ticker is loaded"""
+    console.print("[red]No data loaded. Use 'load' command to load a symbol[/red]")
+
+# helper function for setting up a plugin command alias
+def create_alias(cls, original_name, alias_name, value_add):
+    original_method = getattr(cls, original_name)
+    def alias_method(self):
+        return original_method(self, plugin_name=value_add)
+
+    setattr(cls, alias_name, alias_method)
+
+
+class ExtismController(StockBaseController):
+    """Extism Controller class"""
+
+    CHOICES_COMMANDS = ["load"]
+    PATH = "/stocks/bt/extism/"
+    CHOICES_GENERATION = True
+
+    def __init__(
+        self, ticker: str, stock: pd.DataFrame, queue: Optional[List[str]] = None
+    ):
+        """Constructor"""
+        for name,plugin in plugins.items():
+            create_alias(self, 'call_run', 'call_'+name, name)
+            self.CHOICES_COMMANDS.append(name)
+
+        super().__init__(queue)
+
+        self.ticker = ticker
+        self.stock = stock
+        if session and get_current_user().preferences.USE_PROMPT_TOOLKIT:
+            choices: dict = self.choices_default
+
+            self.completer = NestedCompleter.from_nested_dict(choices)
+
+    def print_help(self):
+        """Print help"""
+        mt = MenuText("stocks/bt/extism/")
+        mt.add_raw("")
+        mt.add_param("_ticker", self.ticker.upper() or "No Ticker Loaded")
+        mt.add_raw("\n")
+        mt.add_cmd("load")
+        mt.add_raw("\n")
+
+        # add any installed plugins to the menu
+        for name in plugins.keys():
+            mt.add_cmd(name, self.ticker)
+
+        console.print(text=mt.menu_text, menu="Stocks - Backtesting - Extism")
+
+    def custom_reset(self):
+        """Class specific component of reset command"""
+        if self.ticker:
+            return ["stocks", f"load {self.ticker}", "bt", "extism"]
+        return []
+
+    def call_run(self, other_args: List[str], plugin_name=""):
+        """Call extism"""
+
+        if plugin_name:
+            # get the plugin
+            plugin = plugins[plugin_name]
+            metadata = plugin.call("get_metadata", "")
+            metadata = json.loads(metadata)
+            description = metadata['description']
+        else:
+            description = "The base help message for the call_run method"
+
+        parser = argparse.ArgumentParser(
+            add_help=False,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            prog="extism",
+            description=description,
+        )
+
+        # iterate through all of the params and add arg options for each
+        params = json.loads(metadata['params'])
+        #console.print(params)
+        for param in params:
+            parser.add_argument(
+                param['flag'],
+                default=param['default'],
+                dest=param["param"],
+                help=param['desc'],
+            )
+
+        # if other_args and "-" not in other_args[0][0]:
+        #     other_args.insert(0, "-n")
+
+        ns_parser = self.parse_known_args_and_warn(parser, other_args)
+        if ns_parser and plugin_name:
+            if self.stock.empty:
+                no_data_message()
+                return
+
+            args_dict = vars(ns_parser)
+            args_dict.pop('help')
+            extism_view.display_strategy(plugin=plugin, name=plugin_name, symbol=self.ticker, data=self.stock, **args_dict)
+
+
+

openbb_terminal/stocks/backtesting/extism_plugins/extism_model.py

@@ -0,0 +1,159 @@
+"""Backtesting Model"""
+__docformat__ = "numpy"
+
+import logging
+import warnings
+
+import bt
+import pandas as pd
+import pandas_ta as ta
+import yfinance as yf
+
+import extism
+from extism import Plugin, host_fn, set_log_file
+
+import datetime
+
+import json
+
+# TODO: Remove this later
+from openbb_terminal.rich_config import console
+
+from openbb_terminal.common.technical_analysis import ta_helpers
+from openbb_terminal.decorators import log_start_end
+from openbb_terminal.helper_funcs import is_intraday
+
+logger = logging.getLogger(__name__)
+
+set_log_file('extism.out', level='debug')
+
+@log_start_end(log=logger)
+def get_data(symbol: str, start_date: str = "2019-01-01") -> pd.DataFrame:
+    """Function to replace bt.get, gets Adjusted close of symbol using yfinance.
+
+    Parameters
+    ----------
+    symbol: str
+        Ticker to get data for
+    start_date: str
+        Start date in YYYY-MM-DD format
+
+    Returns
+    -------
+    prices: pd.DataFrame
+        Dataframe of Adj Close with columns = [ticker]
+    """
+    data = yf.download(symbol, start=start_date, progress=False, ignore_tz=True)
+    close_col = ta_helpers.check_columns(data, high=False, low=False)
+    if close_col is None:
+        return pd.DataFrame()
+    df = pd.DataFrame(data[close_col])
+    df.columns = [symbol]
+
+    return df
+
+@log_start_end(log=logger)
+def buy_and_hold(symbol: str, start_date: str, name: str = "") -> bt.Backtest:
+    """Generates a buy and hold backtest object for the given ticker.
+
+    Parameters
+    ----------
+    symbol: str
+        Stock to test
+    start_date: str
+        Backtest start date, in YYYY-MM-DD format. Can be either string or datetime
+    name: str
+        Name of the backtest (for labeling purposes)
+
+    Returns
+    -------
+    bt.Backtest
+        Backtest object for buy and hold strategy
+    """
+    prices = get_data(symbol, start_date)
+    bt_strategy = bt.Strategy(
+        name,
+        [
+            bt.algos.RunOnce(),
+            bt.algos.SelectAll(),
+            bt.algos.WeighEqually(),
+            bt.algos.Rebalance(),
+        ],
+    )
+    return bt.Backtest(bt_strategy, prices)
+
+
+#(plugin, name: str, symbol: str, data: pd.DataFrame, **kwargs)
+@log_start_end(log=logger)
+def run_strategy(plugin, name: str, symbol: str, data: pd.DataFrame, **kwargs) -> bt.backtest.Result:
+    """Perform backtest for strategies backed by Extism Plugins.
+
+    Parameters
+    ----------
+    symbol: str
+        Stock ticker
+    data: pd.DataFrame
+        Dataframe of prices
+
+    Returns
+    -------
+    bt.backtest.Result
+        Backtest results
+    """
+
+    # console.print(plugin)
+    # console.print(kwargs)
+    # console.print(data)
+
+    # TODO: Help Wanted!
+    # Implement support for backtesting on intraday data
+    if is_intraday(data):
+        return None
+
+    data.index = pd.to_datetime(data.index.date)
+    symbol = symbol.lower()
+
+    start_date = data.index[0]
+    close_col = ta_helpers.check_columns(data, high=False, low=False)
+    if close_col is None:
+        return bt.backtest.Result()
+
+    prices = pd.DataFrame(data[close_col])
+    prices.columns = [symbol]
+
+     # build up the request to the plugin
+    json_string = prices.to_json()
+    req = json.loads(json_string)
+    req['prices'] = req.pop(symbol)
+    req.update({"params": kwargs})
+    req = json.dumps(req) 
+
+    # call the plugin
+    rep = plugin.call("call", req)
+
+    # parse the response from the plugin
+    signal = json.loads(rep)
+    signal = pd.DataFrame(signal)
+    signal.index = pd.to_datetime(pd.to_numeric(signal.index), unit='ms').date
+    signal.rename(columns={'signal': symbol}, inplace=True)
+    merged_data = bt.merge(signal, prices)
+    merged_data.columns = ["signal", "price"]
+
+    # run the response from the plugin through the backtester
+    warnings.simplefilter(action="ignore", category=FutureWarning)
+    bt_strategy = bt.Strategy(
+        "Strategy", [bt.algos.WeighTarget(signal), bt.algos.Rebalance()]
+    )
+    bt_backtest = bt.Backtest(bt_strategy, prices)
+    bt_backtest = bt.Backtest(bt_strategy, prices)
+    backtests = [bt_backtest]
+
+    with warnings.catch_warnings():
+        stock_bt = buy_and_hold(symbol, start_date, symbol.upper() + " Hold")
+        backtests.append(stock_bt)
+
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore")
+        res = bt.run(*backtests)
+
+    return res