diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 46548b382..ca3e6343c 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -16,6 +16,7 @@ Infrastructure / Support ---------------------- * Account-based authorization, incl. new decorators for endpoints [see `PR #210 `_] * Improve data specification for forecasting models using timely-beliefs data [see `PR #154 `_] +* Allow plugins to register their custom config settings, so that FlexMeasures can check whether they are set up correctly [see `PR #230 `_] v0.7.0 | October 26, 2021 diff --git a/documentation/dev/plugins.rst b/documentation/dev/plugins.rst index 7ef3c875e..0770f3604 100644 --- a/documentation/dev/plugins.rst +++ b/documentation/dev/plugins.rst @@ -150,12 +150,35 @@ This will find `css/styles.css` if you add that folder and file to your Blueprin .. note:: This styling will only apply to the pages defined in your plugin (to pages based on your own base template). To apply a styling to all other pages which are served by FlexMeasures, consider using the config setting :ref:`extra-css-config`. -Adding config options +Adding config settings ^^^^^^^^^^^^^^^^^^^^^^^^ -You might want to override some FlexMEasures configuration settings from within your plugin. Some examples for possible settings are named on this page, e.g. the custom style (see above) or custom logo (see below). There is a `record_once` function on Blueprints which can help with this. Example: +FlexMeasures can automatically check for you if any custom config settings, which your plugin is using, are present. +This can be very useful in maintaining installations of FlexMeasures with plugins. +Config settings can be registered by setting the (optional) ``__settings__`` attribute on your plugin module: -.. code-block: python +.. code-block:: python + + __settings__ = [ + "MY_PLUGIN_URL": { + "description": "URL used by my plugin for x.", + "level": "error", + }, + "MY_PLUGIN_TOKEN": { + "description": "Token used by my plugin for y.", + "level": "warning", + "message_if_missing": "Without this token, my plugin will not do y.", + "parse_as": str, + }, + "MY_PLUGIN_COLOR": { + "description": "Color used to override the default plugin color.", + "level": "info", + }, + ] + +You might want to override some FlexMeasures configuration settings from within your plugin. Some examples for possible settings are named on this page, e.g. the custom style (see above) or custom logo (see below). There is a `record_once` function on Blueprints which can help with this. Example: + +.. code-block:: python @our_client_bp.record_once def record_logo_path(setup_state): diff --git a/flexmeasures/app.py b/flexmeasures/app.py index 2beff0092..7f0a450c7 100644 --- a/flexmeasures/app.py +++ b/flexmeasures/app.py @@ -124,7 +124,8 @@ def create( # If plugins register routes, they'll have precedence over standard UI # routes (first registration wins). However, we want to control "/" separately. - from flexmeasures.utils.app_utils import root_dispatcher, register_plugins + from flexmeasures.utils.app_utils import root_dispatcher + from flexmeasures.utils.plugin_utils import register_plugins app.add_url_rule("/", view_func=root_dispatcher) register_plugins(app) diff --git a/flexmeasures/utils/app_utils.py b/flexmeasures/utils/app_utils.py index 907b7b757..cd1bfe531 100644 --- a/flexmeasures/utils/app_utils.py +++ b/flexmeasures/utils/app_utils.py @@ -1,11 +1,9 @@ from typing import Union, Tuple, List, Optional import os import sys -import importlib.util -from importlib.abc import Loader import click -from flask import Blueprint, Flask, current_app, redirect +from flask import Flask, current_app, redirect from flask.cli import FlaskGroup, with_appcontext from flask_security import current_user import sentry_sdk @@ -171,97 +169,7 @@ def parse_config_entry_by_account_roles( ]: return entry else: - app.logger.warn( + app.logger.warning( f"Setting '{config}' in {setting_name} is neither a string nor two-part tuple. Ignoring ..." ) return None - - -def register_plugins(app: Flask): - """ - Register FlexMeasures plugins as Blueprints. - This is configured by the config setting FLEXMEASURES_PLUGINS. - - Assumptions: - - a setting EITHER points to a plugin folder containing an __init__.py file - OR it is the name of an installed module, which can be imported. - - each plugin defines at least one Blueprint object. These will be registered with the Flask app, - so their functionality (e.g. routes) becomes available. - - If you load a plugin via a file path, we'll refer to the plugin with the name of your plugin folder - (last part of the path). - """ - plugins = app.config.get("FLEXMEASURES_PLUGINS", []) - if not plugins: - # this is deprecated behaviour which we should remove in version 1.0 - app.logger.debug( - "No plugins configured. Attempting deprecated setting FLEXMEASURES_PLUGIN_PATHS ..." - ) - plugins = app.config.get("FLEXMEASURES_PLUGIN_PATHS", []) - if not isinstance(plugins, list): - app.logger.error( - f"The value of FLEXMEASURES_PLUGINS is not a list: {plugins}. Cannot install plugins ..." - ) - return - app.config["LOADED_PLUGINS"] = {} - for plugin in plugins: - plugin_name = plugin.split("/")[-1] - app.logger.info(f"Importing plugin {plugin_name} ...") - module = None - if not os.path.exists(plugin): # assume plugin is a package - pkg_name = os.path.split(plugin)[ - -1 - ] # rule out attempts for relative package imports - app.logger.debug( - f"Attempting to import {pkg_name} as an installed package ..." - ) - try: - module = importlib.import_module(pkg_name) - except ModuleNotFoundError: - app.logger.error( - f"Attempted to import module {pkg_name} (as it is not a valid file path), but it is not installed." - ) - continue - else: # assume plugin is a file path - if not os.path.exists(os.path.join(plugin, "__init__.py")): - app.logger.error( - f"Plugin {plugin_name} is a valid file path, but does not contain an '__init__.py' file. Cannot load plugin {plugin_name}." - ) - continue - spec = importlib.util.spec_from_file_location( - plugin_name, os.path.join(plugin, "__init__.py") - ) - if spec is None: - app.logger.error( - f"Could not load specs for plugin {plugin_name} at {plugin}." - ) - continue - module = importlib.util.module_from_spec(spec) - sys.modules[plugin_name] = module - assert isinstance(spec.loader, Loader) - spec.loader.exec_module(module) - - if module is None: - app.logger.error(f"Plugin {plugin} could not be loaded.") - continue - - plugin_version = getattr(module, "__version__", "0.1") - - # Look for blueprints in the plugin's main __init__ module and register them - plugin_blueprints = [ - getattr(module, a) - for a in dir(module) - if isinstance(getattr(module, a), Blueprint) - ] - if not plugin_blueprints: - app.logger.warning( - f"No blueprints found for plugin {plugin_name} at {plugin}." - ) - continue - for plugin_blueprint in plugin_blueprints: - app.logger.debug(f"Registering {plugin_blueprint} ...") - app.register_blueprint(plugin_blueprint) - - app.config["LOADED_PLUGINS"][plugin_name] = plugin_version - app.logger.info(f"Loaded plugins: {app.config['LOADED_PLUGINS']}") - sentry_sdk.set_context("plugins", app.config.get("LOADED_PLUGINS", {})) diff --git a/flexmeasures/utils/plugin_utils.py b/flexmeasures/utils/plugin_utils.py new file mode 100644 index 000000000..ff387b813 --- /dev/null +++ b/flexmeasures/utils/plugin_utils.py @@ -0,0 +1,179 @@ +import importlib.util +import os +import sys +from importlib.abc import Loader +from typing import Dict + +import sentry_sdk +from flask import Flask, Blueprint + + +def register_plugins(app: Flask): + """ + Register FlexMeasures plugins as Blueprints. + This is configured by the config setting FLEXMEASURES_PLUGINS. + + Assumptions: + - a setting EITHER points to a plugin folder containing an __init__.py file + OR it is the name of an installed module, which can be imported. + - each plugin defines at least one Blueprint object. These will be registered with the Flask app, + so their functionality (e.g. routes) becomes available. + + If you load a plugin via a file path, we'll refer to the plugin with the name of your plugin folder + (last part of the path). + """ + plugins = app.config.get("FLEXMEASURES_PLUGINS", []) + if not plugins: + # this is deprecated behaviour which we should remove in version 1.0 + app.logger.debug( + "No plugins configured. Attempting deprecated setting FLEXMEASURES_PLUGIN_PATHS ..." + ) + plugins = app.config.get("FLEXMEASURES_PLUGIN_PATHS", []) + if not isinstance(plugins, list): + app.logger.error( + f"The value of FLEXMEASURES_PLUGINS is not a list: {plugins}. Cannot install plugins ..." + ) + return + app.config["LOADED_PLUGINS"] = {} + for plugin in plugins: + plugin_name = plugin.split("/")[-1] + app.logger.info(f"Importing plugin {plugin_name} ...") + module = None + if not os.path.exists(plugin): # assume plugin is a package + pkg_name = os.path.split(plugin)[ + -1 + ] # rule out attempts for relative package imports + app.logger.debug( + f"Attempting to import {pkg_name} as an installed package ..." + ) + try: + module = importlib.import_module(pkg_name) + except ModuleNotFoundError: + app.logger.error( + f"Attempted to import module {pkg_name} (as it is not a valid file path), but it is not installed." + ) + continue + else: # assume plugin is a file path + if not os.path.exists(os.path.join(plugin, "__init__.py")): + app.logger.error( + f"Plugin {plugin_name} is a valid file path, but does not contain an '__init__.py' file. Cannot load plugin {plugin_name}." + ) + continue + spec = importlib.util.spec_from_file_location( + plugin_name, os.path.join(plugin, "__init__.py") + ) + if spec is None: + app.logger.error( + f"Could not load specs for plugin {plugin_name} at {plugin}." + ) + continue + module = importlib.util.module_from_spec(spec) + sys.modules[plugin_name] = module + assert isinstance(spec.loader, Loader) + spec.loader.exec_module(module) + + if module is None: + app.logger.error(f"Plugin {plugin} could not be loaded.") + continue + + plugin_version = getattr(module, "__version__", "0.1") + plugin_settings = getattr(module, "__settings__", {}) + check_config_settings(app, plugin_settings) + + # Look for blueprints in the plugin's main __init__ module and register them + plugin_blueprints = [ + getattr(module, a) + for a in dir(module) + if isinstance(getattr(module, a), Blueprint) + ] + if not plugin_blueprints: + app.logger.warning( + f"No blueprints found for plugin {plugin_name} at {plugin}." + ) + continue + for plugin_blueprint in plugin_blueprints: + app.logger.debug(f"Registering {plugin_blueprint} ...") + app.register_blueprint(plugin_blueprint) + + app.config["LOADED_PLUGINS"][plugin_name] = plugin_version + app.logger.info(f"Loaded plugins: {app.config['LOADED_PLUGINS']}") + sentry_sdk.set_context("plugins", app.config.get("LOADED_PLUGINS", {})) + + +def check_config_settings(app, settings: Dict[str, dict]): + """Make sure expected config settings exist. + + For example: + + settings = { + "MY_PLUGIN_URL": { + "description": "URL used by my plugin for x.", + "level": "error", + }, + "MY_PLUGIN_TOKEN": { + "description": "Token used by my plugin for y.", + "level": "warning", + "message": "Without this token, my plugin will not do y.", + "parse_as": str, + }, + "MY_PLUGIN_COLOR": { + "description": "Color used to override the default plugin color.", + "level": "info", + }, + } + + """ + assert isinstance(settings, dict), f"{type(settings)} should be a dict" + for setting_name, setting_fields in settings.items(): + assert isinstance(setting_fields, dict), f"{setting_name} should be a dict" + + missing_config_settings = [] + config_settings_with_wrong_type = [] + for setting_name, setting_fields in settings.items(): + setting = app.config.get(setting_name) + if setting is None: + missing_config_settings.append(setting_name) + elif "parse_as" in setting_fields and not isinstance( + setting, setting_fields["parse_as"] + ): + config_settings_with_wrong_type.append((setting_name, setting)) + for setting_name, setting in config_settings_with_wrong_type: + log_wrong_type_for_config_setting( + app, setting_name, settings[setting_name], type(setting) + ) + for setting_name in missing_config_settings: + log_missing_config_setting(app, setting_name, settings[setting_name]) + + +def log_wrong_type_for_config_setting( + app, setting_name: str, setting_fields: dict, setting_type: type +): + """Log a message for this config setting that has the wrong type.""" + app.logger.warning( + f"Config setting '{setting_name}' is a {setting_type} whereas a {setting_fields['parse_as']} was expected." + ) + + +def log_missing_config_setting(app, setting_name: str, setting_fields: dict): + """Log a message for this missing config setting. + + The logging level is taken from the 'level' key. If missing, we default to error. + If present, we also log the 'description' and the 'message_if_missing' keys. + """ + message_if_missing = ( + f" {setting_fields['message_if_missing']}" + if "message_if_missing" in setting_fields + else "" + ) + description = ( + f" ({setting_fields['description']})" if "description" in setting_fields else "" + ) + level = setting_fields["level"] if "level" in setting_fields else "error" + if not hasattr(app.logger, level): + app.logger.warning( + f"Unrecognized logger level '{level}' for config setting '{setting_name}'." + ) + level = "error" + getattr(app.logger, level)( + f"Missing config setting '{setting_name}'{description}.{message_if_missing}", + )