From 6c6c2328c430e627cc6263ce91151a08169e3d8d Mon Sep 17 00:00:00 2001 From: "create-issue-branch[bot]" <53036503+create-issue-branch[bot]@users.noreply.github.com> Date: Fri, 9 Apr 2021 13:43:12 +0200 Subject: [PATCH 01/24] PostMeterData endpoint broken in API v2.0 (#95) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix v2_0.postMeterData endpoint Co-authored-by: nhoening Co-authored-by: Nicolas Höning --- documentation/changelog.rst | 8 ++++++++ flexmeasures/api/v2_0/routes.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 6fba835e8..742051b87 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -16,6 +16,14 @@ Infrastructure / Support * Integration with `timely beliefs `_ lib: Sensor data as TimedBeliefs [see `PR #79 `_] +v0.3.1 | April 9, 2021 +=========================== + +Bugfixes +-------- +* PostMeterData endpoint was broken in API v2.0 [see `PR #95 `_] + + v0.3.0 | April 2, 2021 =========================== diff --git a/flexmeasures/api/v2_0/routes.py b/flexmeasures/api/v2_0/routes.py index f3e4c3bc1..60cc27f07 100644 --- a/flexmeasures/api/v2_0/routes.py +++ b/flexmeasures/api/v2_0/routes.py @@ -761,7 +761,7 @@ def post_meter_data(): :status 403: INVALID_SENDER :status 405: INVALID_METHOD """ - return v2_0_implementations.post_meter_data_response() + return v2_0_implementations.sensors.post_meter_data_response() @flexmeasures_api_v2_0.route("/postPrognosis", methods=["POST"]) From ad6fa3f6210ef7ef4e2b1c01c79f9759f4197b3d Mon Sep 17 00:00:00 2001 From: "create-issue-branch[bot]" <53036503+create-issue-branch[bot]@users.noreply.github.com> Date: Sun, 11 Apr 2021 13:24:25 +0200 Subject: [PATCH 02/24] Using wrong format in asset editing leads to unhelpful error message (#93) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit asset edit form behaves better on form validation errors Co-authored-by: nhoening Co-authored-by: Nicolas Höning --- documentation/changelog.rst | 6 +++++- flexmeasures/ui/crud/assets.py | 15 +++++++++++++-- flexmeasures/ui/templates/crud/asset.html | 12 ++++++------ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 742051b87..f1dcdefd6 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -3,13 +3,17 @@ FlexMeasures Changelog ********************** -v0.2.5 | April XX, 2021 +v0.4.0 | April XX, 2021 =========================== New features ----------- * Add sensors with CLI command [see `PR #83 `_] +Bugfixes +----------- +* Asset edit form displayed wrong error message. Also enabled the asset edit form to display the invalid user input back to the user [see `PR #93 `_] + Infrastructure / Support ---------------------- * Updated dependencies, including Flask-Security-Too [see `PR #82 `_] diff --git a/flexmeasures/ui/crud/assets.py b/flexmeasures/ui/crud/assets.py index 609c95321..b779a8965 100644 --- a/flexmeasures/ui/crud/assets.py +++ b/flexmeasures/ui/crud/assets.py @@ -278,11 +278,22 @@ def post(self, id: str): else: asset_form = with_options(AssetForm()) if not asset_form.validate_on_submit(): + asset = Asset.query.get(id) + latest_measurement_time_str, asset_plot_html = get_latest_power_as_plot( + asset + ) + # Display the form data, but set some extra data which the page wants to show. + asset_info = asset_form.data.copy() + asset_info["id"] = id + asset_info["owner_id"] = asset.owner_id + asset_info["entity_address"] = asset.entity_address return render_flexmeasures_template( - "crud/asset_new.html", + "crud/asset.html", asset_form=asset_form, + asset=asset_info, msg="Cannot edit asset.", - map_center=get_center_location(db, user=current_user), + latest_measurement_time_str=latest_measurement_time_str, + asset_plot_html=asset_plot_html, mapboxAccessToken=current_app.config.get("MAPBOX_ACCESS_TOKEN", ""), ) patch_asset_response = InternalApi().patch( diff --git a/flexmeasures/ui/templates/crud/asset.html b/flexmeasures/ui/templates/crud/asset.html index 3e461fd16..0762524eb 100644 --- a/flexmeasures/ui/templates/crud/asset.html +++ b/flexmeasures/ui/templates/crud/asset.html @@ -49,7 +49,7 @@

Edit asset {{ asset.display_name }}

(Owned by {{ asset.owner_id | username }}) - +
{{ asset_form.display_name.label(class="col-sm-6 control-label") }}
@@ -198,7 +198,7 @@

Location

// create map var assetMap = L - .map('mapid', { center: [{{ asset.latitude }}, {{ asset.longitude }}], zoom: 10}) + .map('mapid', { center: [{{ asset.latitude | replace("None", 10) }}, {{ asset.longitude | replace("None", 10) }}], zoom: 10}) .on('popupopen', function () { $(function () { $('[data-toggle="tooltip"]').tooltip(); @@ -207,17 +207,17 @@

Location

addTileLayer(assetMap, '{{ mapboxAccessToken }}'); // create marker - var {{ asset.asset_type_name | parameterize }}_icon = new L.DivIcon({ + var asset_icon = new L.DivIcon({ className: 'map-icon', - html: '', + html: '', iconSize: [100, 100], // size of the icon iconAnchor: [50, 50], // point of the icon which will correspond to marker's location popupAnchor: [0, -50] // point from which the popup should open relative to the iconAnchor }); var marker = L .marker( - [{{ asset.latitude }}, {{ asset.longitude }}], - { icon: {{ asset.asset_type_name | parameterize }}_icon } + [{{ asset.latitude | replace("None", 10)}}, {{ asset.longitude | replace("None", 10) }}], + { icon: asset_icon } ).addTo(assetMap); assetMap.on('click', function (e) { From 134fb696587af2e38dd915f9aa18a0a934729cb7 Mon Sep 17 00:00:00 2001 From: "create-issue-branch[bot]" <53036503+create-issue-branch[bot]@users.noreply.github.com> Date: Mon, 12 Apr 2021 13:25:34 +0200 Subject: [PATCH 03/24] Plugin-ability for views and CLI (#91) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * no need to speak of Jeju island anymore in the base template * clean up CLI Readme * introduce FLEXMEASURES_LISTED_VIEWS, retire FLEXMEASURES_SHOW_CONTROL_UI * Analytics page: fix metrics table header for weather sensor * import plugins as Blueprints * enable listing configured plugin view names in menu * add general documentation for creating a plugin Co-authored-by: Nicolas Höning --- documentation/changelog.rst | 2 + documentation/configuration.rst | 43 +++++++--- documentation/dev/plugins.rst | 79 +++++++++++++++++++ documentation/index.rst | 2 +- flexmeasures/app.py | 4 + flexmeasures/data/scripts/cli_tasks/Readme.md | 17 +--- flexmeasures/ui/__init__.py | 2 +- flexmeasures/ui/templates/base.html | 5 +- flexmeasures/ui/templates/defaults.jinja | 41 ++++++---- .../ui/templates/views/analytics.html | 2 +- flexmeasures/utils/app_utils.py | 40 +++++++++- flexmeasures/utils/config_defaults.py | 9 ++- 12 files changed, 200 insertions(+), 46 deletions(-) create mode 100644 documentation/dev/plugins.rst diff --git a/documentation/changelog.rst b/documentation/changelog.rst index f1dcdefd6..e02b4c351 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -9,6 +9,8 @@ v0.4.0 | April XX, 2021 New features ----------- * Add sensors with CLI command [see `PR #83 `_] +* Configure views with ``FLEXMEASURES_LISTED_VIEWS`` [see `PR #91 `_] +* Allow for views and CLI functions to come from plugins [see also `PR #91 `_] Bugfixes ----------- diff --git a/documentation/configuration.rst b/documentation/configuration.rst index e737e0780..2262aaacc 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -6,7 +6,7 @@ Configuration The following configurations are used by FlexMeasures. Required settings (e.g. postgres db) are marked with a double star (**). -To enable easier quickstart tutorials, these settings can be set by env vars. +To enable easier quickstart tutorials, these settings can be set by environment variables. Recommended settings (e.g. mail, redis) are marked by one star (*). .. note:: FlexMeasures is best configured via a config file. The config file for FlexMeasures can be placed in one of two locations: @@ -15,6 +15,7 @@ Recommended settings (e.g. mail, redis) are marked by one star (*). * in the user's home directory (e.g. ``~/.flexmeasures.cfg`` on Unix). In this case, note the dot at the beginning of the filename! * in the app's instance directory (e.g. ``/path/to/your/flexmeasures/code/instance/flexmeasures.cfg``\ ). The path to that instance directory is shown to you by running flexmeasures (e.g. ``flexmeasures run``\ ) with required settings missing or otherwise by running ``flexmeasures shell``. + Basic functionality ------------------- @@ -51,6 +52,19 @@ and the first month when the domain was under the current owner's administration Default: ``{"flexmeasures.io": "2021-01"}`` + +.. _plugin-config: + +FLEXMEASURES_PLUGIN_PATHS +^^^^^^^^^^^^^^^^^^^^^^^^^ + +A list of absolute paths to Blueprint-based plugins for FlexMeasures (e.g. for custom views or CLI functions). +Each plugin path points to a folder, which should contain an ``__init__.py`` file where the Blueprint is defined. +See :ref:`plugins` on what is expected for content. + +Default: ``[]`` + + FLEXMEASURES_DB_BACKUP_PATH ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -65,6 +79,7 @@ Whether to turn on a feature which times requests made through FlexMeasures. Int Default: ``False`` + UI -- @@ -89,6 +104,7 @@ Interval in which viewing the queues dashboard refreshes itself, in milliseconds Default: ``3000`` (3 seconds) + Timing ------ @@ -113,6 +129,7 @@ The horizon to use when making schedules. Default: ``timedelta(hours=2 * 24)`` + Tokens ------ @@ -133,7 +150,7 @@ Default: ``None`` MAPBOX_ACCESS_TOKEN ^^^^^^^^^^^^^^^^^^^ -Token for accessing the mapbox API (for displaying maps on the dashboard and asset pages). You can learn how to obtain one `here `_ +Token for accessing the MapBox API (for displaying maps on the dashboard and asset pages). You can learn how to obtain one `here `_ Default: ``None`` @@ -144,6 +161,7 @@ Token which external services can use to check on the status of recurring tasks Default: ``None`` + SQLAlchemy ---------- @@ -172,6 +190,7 @@ Default: "connect_args": {"options": "-c timezone=utc"}, } + Security -------- @@ -215,7 +234,7 @@ Default: ``60 * 60 * 6`` (six hours) SECURITY_TRACKABLE ^^^^^^^^^^^^^^^^^^ -Wether to track user statistics. Turning this on requires certain user fields. +Whether to track user statistics. Turning this on requires certain user fields. We do not use this feature, but we do track number of logins. Default: ``False`` @@ -223,14 +242,14 @@ Default: ``False`` CORS_ORIGINS ^^^^^^^^^^^^ -Allowed cross-origins. Set to "*" to allow all. For development (e.g. javascript on localhost) you might use "null" in this list. +Allowed cross-origins. Set to "*" to allow all. For development (e.g. JavaScript on localhost) you might use "null" in this list. Default: ``[]`` CORS_RESOURCES: ^^^^^^^^^^^^^^^ -FlexMeasures resources which get cors protection. This can be a regex, a list of them or dict with all possible options. +FlexMeasures resources which get cors protection. This can be a regex, a list of them or a dictionary with all possible options. Default: ``[r"/api/*"]`` @@ -244,6 +263,7 @@ Allows users to make authenticated requests. If true, injects the Access-Control Default: ``True`` + .. _mail-config: Mail @@ -335,7 +355,7 @@ Default: ``6379`` FLEXMEASURES_REDIS_DB_NR (*) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Number of the redis database to use (Redis per default has 16 databases, nubered 0-15) +Number of the redis database to use (Redis per default has 16 databases, numbered 0-15) Default: ``0`` @@ -364,9 +384,14 @@ so that old imported data can be demoed as if it were current Default: ``None`` -FLEXMEASURES_SHOW_CONTROL_UI + +.. _menu-config: + +FLEXMEASURES_LISTED_VIEWS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The control page is still mocked, so this setting controls if it is to be shown. +A list of the views which are listed in the menu. -Default: ``False`` +.. note:: This setting is likely to be deprecated soon, as we might want to control it per account (once we implemented a multi-tenant data model per FlexMeasures server). + +Default: ``["dashboard", "analytics", "portfolio", "assets", "users"]`` diff --git a/documentation/dev/plugins.rst b/documentation/dev/plugins.rst new file mode 100644 index 000000000..c422b37a0 --- /dev/null +++ b/documentation/dev/plugins.rst @@ -0,0 +1,79 @@ +.. _plugins: + +Writing Plugins +==================== + +You can extend FlexMeasures with functionality like UI pages or CLI functions. + +A FlexMeasures plugin works as a `Flask Blueprint `_. + +.. todo:: We'll use this to allow for custom forecasting and scheduling algorithms, as well. + + +How it works +^^^^^^^^^^^^^^ + +Use the config setting :ref:`plugin-config` to point to your plugin(s). + +Here are the assumptions FlexMeasures makes to be able to import your Blueprint: + +- The plugin folder contains an __init__.py file. +- In this init, you define a Blueprint object called ``_bp``. + +We'll refer to the plugin with the name of your plugin folder. + + +Showcase +^^^^^^^^^ + +Here is a showcase file which constitutes a FlexMeasures plugin. We imagine that we made a plugin to implement some custom logic for a client. + +We created the file ``/our_client/__init__.py``. So, ``our_client`` is the plugin folder and becomes the plugin name. +All else that is needed for this showcase (not shown here) is ``/our_client/templates/metrics.html``, which works just as other FlexMeasures templates (they are Jinja2 templates and you can start them with ``{% extends "base.html" %}`` for integration into the FlexMeasures structure). + + +* We demonstrate adding a view which can be rendered via the FlexMeasures base templates. +* We also showcase a CLI function which has access to the FlexMeasures `app` object. It can be called via ``flexmeasures our_client test``. + +.. code-block:: python + + from flask import Blueprint, render_template, abort + + from flask_security import login_required + from flexmeasures.ui.utils.view_utils import render_flexmeasures_template + + + our_client_bp = Blueprint('our_client', 'our_client', + template_folder='templates') + + + # Showcase: Adding a view + + @our_client_bp.route('/metrics') + @login_required + def metrics(): + msg = "I am part of FM !" + # Note that we render via the in-built FlexMeasures way + return render_flexmeasures_template( + "metrics.html", + message=msg, + ) + + + # Showcase: Adding a CLI command + + import click + from flask import current_app + from flask.cli import with_appcontext + + + our_client_bp.cli.help = "Our client commands" + + @our_client_bp.cli.command("test") + @with_appcontext + def oc_test(): + print(f"I am a CLI command, part of FlexMeasures: {current_app}") + + + +.. note:: Plugin views can also be added to the FlexMeasures UI menu ― just name them in the config setting :ref:`menu-config`. \ No newline at end of file diff --git a/documentation/index.rst b/documentation/index.rst index 13c671cd4..d5af9d4f2 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -84,7 +84,7 @@ The platform operator of FlexMeasures can be an Aggregator. dev/data dev/api dev/ci - + dev/plugins .. toctree:: :caption: Integrations diff --git a/flexmeasures/app.py b/flexmeasures/app.py index e152b0517..c2ef442f2 100644 --- a/flexmeasures/app.py +++ b/flexmeasures/app.py @@ -104,6 +104,10 @@ def create(env: Optional[str] = None, path_to_config: Optional[str] = None) -> F register_ui_at(app) + from flexmeasures.utils.app_utils import register_plugins + + register_plugins(app) + # Profile endpoints (if needed, e.g. during development) @app.before_request def before_request(): diff --git a/flexmeasures/data/scripts/cli_tasks/Readme.md b/flexmeasures/data/scripts/cli_tasks/Readme.md index ee4a56beb..afa76806b 100644 --- a/flexmeasures/data/scripts/cli_tasks/Readme.md +++ b/flexmeasures/data/scripts/cli_tasks/Readme.md @@ -5,20 +5,9 @@ These scripts are made available as cli tasks. To view the available commands, run: - flask --help + flexmeasures --help -For help on individual commands, for example on the saving and loading functionality, type `flask db-save --help` or `flask db-load --help`. -These help messages are generated from the code (see the file db_pop.py in the cli_tasks directory). -Structural data refers to database tables with relatively little entries (they describe things like assets, markets and weather sensors). -Time series data refers to database tables with many entries (like power, price and temperature values). -The default location for storing database backups is within the top-level `migrations` directory. -The contents of this folder are not part of the code repository, and database backups will be lost when deleted. - -The load functionality is also made available as an API endpoint called _restoreData_, and described as such in the user documentation for the play server. -The relevant API endpoint is set up in the `flexmeasures/api/play` directory. -The file `routes.py` contains its registration and documentation, while the file `implementations.py` contains the functional logic that connects the API endpoint to the same scripts that are accessible through the command line interface. - -The save functionality is currently not available as an API endpoint. -This script cannot be executed within the lifetime of an https request, and would require processing within a separate thread, similar to how forecasting jobs are handled by FlexMeasures. +For help on individual commands, type `flexmesaures --help`. +Structural data refers to database tables which do not contain time series data. To create new commands, be sure to register any new file (containing the corresponding script) with the flask cli in `flexmeasures/data/__init__.py`. diff --git a/flexmeasures/ui/__init__.py b/flexmeasures/ui/__init__.py index c3df28dff..7e4d2681c 100644 --- a/flexmeasures/ui/__init__.py +++ b/flexmeasures/ui/__init__.py @@ -138,7 +138,7 @@ def add_jinja_variables(app): for v in ( "FLEXMEASURES_MODE", "FLEXMEASURES_PLATFORM_NAME", - "FLEXMEASURES_SHOW_CONTROL_UI", + "FLEXMEASURES_LISTED_VIEWS", "FLEXMEASURES_PUBLIC_DEMO_CREDENTIALS", ): app.jinja_env.globals[v] = app.config.get(v, "") diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html index 666643572..2bfd0be51 100644 --- a/flexmeasures/ui/templates/base.html +++ b/flexmeasures/ui/templates/base.html @@ -64,8 +64,7 @@ {{ FLEXMEASURES_PLATFORM_NAME }} - {{ self.title() }} {% if user_is_logged_in and not "Error" in self.title() %} for {{ user_name }} on - Jeju island {% endif %} + {{ self.title() }} {% if user_is_logged_in and not "Error" in self.title() %} for {{ user_name }} {% endif %}
@@ -301,4 +300,4 @@

Icons from Flaticon -{% endblock base %} \ No newline at end of file +{% endblock base %} diff --git a/flexmeasures/ui/templates/defaults.jinja b/flexmeasures/ui/templates/defaults.jinja index a361acae9..2b436f031 100644 --- a/flexmeasures/ui/templates/defaults.jinja +++ b/flexmeasures/ui/templates/defaults.jinja @@ -3,26 +3,37 @@ {# Front-end app naming #} -{% set show_queues = True if current_user.is_authenticated and (current_user.has_role('admin') or FLEXMEASURES_MODE == "demo") else False %} - {# Front-end menu, as columns with href, id, caption, and (fa fa-)icon #} -{% set navigation_bar = [ - ('dashboard', 'dashboard', 'Dashboard', 'dashboard'), - ('assets', 'assets', 'Assets', 'list-ul'), -] if current_user.is_authenticated else [] %} -{% do navigation_bar.append(('users', 'users', 'Users', 'users')) if current_user.has_role('admin') %} -{% do navigation_bar.extend([ - ('portfolio', 'portfolio', 'Portfolio overview', 'briefcase'), - ('analytics', 'analytics', 'Analytics', 'bar-chart'), - ('upload', 'upload', 'Upload data', 'cloud-upload'), -]) if current_user.is_authenticated %} -{% do navigation_bar.extend([ - ('control', 'control', 'Flexibility actions', 'wrench'), -]) if FLEXMEASURES_SHOW_CONTROL_UI and current_user.is_authenticated %} +{% set navigation_bar = [] %} + +{% set nav_bar_specs = { + "dashboard": dict(title="Dashboard", icon="dashboard"), + "assets": dict(title="Assets", icon="list-ul"), + "users": dict(title="Users", icon="users"), + "portfolio": dict(title="Portfolio overview", icon="briefcase"), + "analytics": dict(title="Analytics", icon="bar-chart"), + "upload": dict(title="Upload data", icon="cloud-upload"), + "control": dict(title="Flexibility actions", icon="wrench") +} +%} + +{% for view_name in FLEXMEASURES_LISTED_VIEWS %} + {# add specs for views we don't know (plugin views) #} + {% do nav_bar_specs.update({view_name: dict(title=view_name.capitalize(), icon="info")}) if view_name not in nav_bar_specs %} + {# add view to menu if user is authenticated #} + {% do navigation_bar.append( + (view_name, view_name, nav_bar_specs[view_name]["title"], nav_bar_specs[view_name]["icon"]) + ) if current_user.is_authenticated %} +{% endfor %} + + +{% set show_queues = True if current_user.is_authenticated and (current_user.has_role('admin') or FLEXMEASURES_MODE == "demo") else False %} {% do navigation_bar.append(('tasks', 'tasks', 'Tasks', 'tasks')) if show_queues %} + {% do navigation_bar.append(('account', 'account', '', 'user')) if current_user.is_authenticated %} + {% do navigation_bar.append(('ui/static/documentation/html/index.html', 'docs', '', 'question')) if documentation_exists and current_user.is_authenticated %} {% set active_page = active_page|default('dashboard') -%} diff --git a/flexmeasures/ui/templates/views/analytics.html b/flexmeasures/ui/templates/views/analytics.html index 7c145c128..10591e128 100644 --- a/flexmeasures/ui/templates/views/analytics.html +++ b/flexmeasures/ui/templates/views/analytics.html @@ -157,7 +157,7 @@

Metrics

Rev./Costs {% endif %} - {% if selected_weather_sensor %} + {% if selected_sensor %} {{ selected_sensor_type.display_name | capitalize }} diff --git a/flexmeasures/utils/app_utils.py b/flexmeasures/utils/app_utils.py index 7d1845db6..a075d14b7 100644 --- a/flexmeasures/utils/app_utils.py +++ b/flexmeasures/utils/app_utils.py @@ -1,7 +1,9 @@ import os import sys +import importlib.util import click +from flask import Flask from flask.cli import FlaskGroup from flexmeasures.app import create as create_app @@ -36,7 +38,7 @@ def set_secret_key(app, filename="secret_key"): try: app.config["SECRET_KEY"] = open(filename, "rb").read() except IOError: - print( + app.logger.error( """ Error: No secret key set. @@ -63,3 +65,39 @@ def set_secret_key(app, filename="secret_key"): ) sys.exit(2) + + +def register_plugins(app: Flask): + """ + Register FlexMeasures plugins as Blueprints. + This is configured by the config setting FLEXMEASURES_PLUGIN_PATHS. + + Assumptions: + - Your plugin folders contains an __init__.py file. + - In this init, you define a Blueprint object called _bp + + We'll refer to the plugins with the name of your plugin folders (last part of tthe path). + """ + plugin_paths = app.config.get("FLEXMEASURES_PLUGIN_PATHS", "") + if not isinstance(plugin_paths, list): + app.logger.warning( + f"The value of FLEXMEASURES_PLUGIN_PATHS is not a list: {plugin_paths}. Cannot install plugins ..." + ) + return + for plugin_path in plugin_paths: + plugin_name = plugin_path.split("/")[-1] + if not os.path.exists(os.path.join(plugin_path, "__init__.py")): + app.logger.warning( + f"Plugin {plugin_name} does not contain an '__init__.py' file. Cannot load plugin {plugin_name}." + ) + return + app.logger.debug(f"Importing plugin {plugin_name} ...") + spec = importlib.util.spec_from_file_location( + plugin_name, os.path.join(plugin_path, "__init__.py") + ) + app.logger.debug(spec) + module = importlib.util.module_from_spec(spec) + app.logger.debug(module) + sys.modules[plugin_name] = module + spec.loader.exec_module(module) + app.register_blueprint(getattr(module, f"{plugin_name}_bp")) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index c84ebe111..995bf9d7f 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -78,7 +78,6 @@ class Config(object): FLEXMEASURES_PLATFORM_NAME: str = "FlexMeasures" FLEXMEASURES_MODE: str = "" FLEXMEASURES_TIMEZONE: str = "Asia/Seoul" - FLEXMEASURES_SHOW_CONTROL_UI: bool = False FLEXMEASURES_HIDE_NAN_IN_UI: bool = False FLEXMEASURES_PUBLIC_DEMO_CREDENTIALS: Optional[Tuple] = None FLEXMEASURES_DEMO_YEAR: Optional[int] = None @@ -86,8 +85,16 @@ class Config(object): # This setting contains the domain on which FlexMeasures runs # and the first month when the domain was under the current owner's administration FLEXMEASURES_HOSTS_AND_AUTH_START: dict = {"flexmeasures.io": "2021-01"} + FLEXMEASURES_PLUGIN_PATHS: List[str] = [] FLEXMEASURES_PROFILE_REQUESTS: bool = False FLEXMEASURES_DB_BACKUP_PATH: str = "migrations/dumps" + FLEXMEASURES_LISTED_VIEWS: List[str] = [ + "dashboard", + "analytics", + "portfolio", + "assets", + "users", + ] FLEXMEASURES_LP_SOLVER: str = "cbc" FLEXMEASURES_PLANNING_HORIZON: timedelta = timedelta(hours=2 * 24) FLEXMEASURES_PLANNING_TTL: timedelta = timedelta( From b46c78b7210bde8d5d8b295e0466d049ffc9c53d Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Fri, 16 Apr 2021 16:36:09 +0200 Subject: [PATCH 04/24] Cli csv upload (#85) Add CLI command to add beliefs from csv, optionally allowing to overwrite existing data. * Add CLI command to add beliefs from csv * Refactor if-else block * Add docstring note about belief timing * Overwrite beliefs (#98) * Try tb upsert functionality * Try to bulk insert without overwrite (faster) before attempting to merge with overwrite (slower) * Make overwriting data a CLI option * Copy docstring from tb * Simplification * UX improvement * Add bulk saving as class method option, but not as CLI option; change class method default to not commit transaction * Update dependency Co-authored-by: F.N. Claessen * Print useful error message if sensor is not found * Changelog entry * Move CLI commands to dev-add group * Update changelog entry Co-authored-by: F.N. Claessen --- documentation/changelog.rst | 3 +- flexmeasures/data/models/data_sources.py | 2 +- flexmeasures/data/models/time_series.py | 28 ++++- .../data/scripts/cli_tasks/data_add.py | 114 +++++++++++++++++- requirements/app.in | 2 +- requirements/app.txt | 2 +- 6 files changed, 140 insertions(+), 11 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index e02b4c351..eeec74783 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -8,7 +8,6 @@ v0.4.0 | April XX, 2021 New features ----------- -* Add sensors with CLI command [see `PR #83 `_] * Configure views with ``FLEXMEASURES_LISTED_VIEWS`` [see `PR #91 `_] * Allow for views and CLI functions to come from plugins [see also `PR #91 `_] @@ -20,6 +19,8 @@ Infrastructure / Support ---------------------- * Updated dependencies, including Flask-Security-Too [see `PR #82 `_] * Integration with `timely beliefs `_ lib: Sensor data as TimedBeliefs [see `PR #79 `_] +* Add sensors with CLI command currently meant for developers only [see `PR #83 `_] +* Add data (beliefs about sensor events) with CLI command currently meant for developers only [see `PR #85 `_] v0.3.1 | April 9, 2021 diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 430dc4de9..0ebed3eb5 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -46,7 +46,7 @@ def label(self): return f"schedule by {self.name}" elif self.type == "crawling script": return f"data retrieved from {self.name}" - elif self.type == "demo script": + elif self.type in ("demo script", "CLI script"): return f"demo data entered by {self.name}" else: return f"data from {self.name}" diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index f77d3285f..873e7088b 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -110,17 +110,37 @@ def search( ) @classmethod - def add(cls, bdf: tb.BeliefsDataFrame, commit_transaction: bool = True): + def add( + cls, + bdf: tb.BeliefsDataFrame, + expunge_session: bool = False, + allow_overwrite: bool = False, + bulk_save_objects: bool = False, + commit_transaction: bool = False, + ): """Add a BeliefsDataFrame as timed beliefs in the database. :param bdf: the BeliefsDataFrame to be persisted - :param commit_transaction: if True, the session is committed - if False, you can still add other data to the session - and commit it all within an atomic transaction + :param expunge_session: if True, all non-flushed instances are removed from the session before adding beliefs. + Expunging can resolve problems you might encounter with states of objects in your session. + When using this option, you might want to flush newly-created objects which are not beliefs + (e.g. a sensor or data source object). + :param allow_overwrite: if True, new objects are merged + if False, objects are added to the session or bulk saved + :param bulk_save_objects: if True, objects are bulk saved with session.bulk_save_objects(), + which is quite fast but has several caveats, see: + https://docs.sqlalchemy.org/orm/persistence_techniques.html#bulk-operations-caveats + if False, objects are added to the session with session.add_all() + :param commit_transaction: if True, the session is committed + if False, you can still add other data to the session + and commit it all within an atomic transaction """ return cls.add_to_session( session=db.session, beliefs_data_frame=bdf, + expunge_session=expunge_session, + allow_overwrite=allow_overwrite, + bulk_save_objects=bulk_save_objects, commit_transaction=commit_transaction, ) diff --git a/flexmeasures/data/scripts/cli_tasks/data_add.py b/flexmeasures/data/scripts/cli_tasks/data_add.py index ff8b7e4ed..ab8f0a9f7 100644 --- a/flexmeasures/data/scripts/cli_tasks/data_add.py +++ b/flexmeasures/data/scripts/cli_tasks/data_add.py @@ -1,7 +1,7 @@ """CLI Tasks for (de)populating the database - most useful in development""" from datetime import timedelta -from typing import List +from typing import List, Optional import pandas as pd import pytz @@ -10,13 +10,18 @@ from flask_security.utils import hash_password import click import getpass +from sqlalchemy.exc import IntegrityError +import timely_beliefs as tb +from flexmeasures.data import db from flexmeasures.data.services.forecasting import create_forecasting_jobs from flexmeasures.data.services.users import create_user -from flexmeasures.data.models.time_series import Sensor, SensorSchema +from flexmeasures.data.models.time_series import Sensor, SensorSchema, TimedBelief from flexmeasures.data.models.assets import Asset, AssetSchema from flexmeasures.data.models.markets import Market from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorSchema +from flexmeasures.data.models.data_sources import DataSource +from flexmeasures.utils.time_utils import server_now @click.group("add") @@ -24,6 +29,11 @@ def fm_add_data(): """FlexMeasures: Add data.""" +@click.group("dev-add") +def fm_dev_add_data(): + """Developer CLI commands not yet meant for users: Add data.""" + + @fm_add_data.command("user") @with_appcontext @click.option("--username", required=True) @@ -63,7 +73,7 @@ def new_user(username: str, email: str, roles: List[str], timezone: str): print(f"Successfully created user {created_user}") -@fm_add_data.command("sensor") +@fm_dev_add_data.command("sensor") @with_appcontext @click.option("--name", required=True) @click.option("--unit", required=True, help="e.g. °C, m/s, kW/m²") @@ -201,6 +211,103 @@ def add_initial_structure(): populate_structure(app.db) +@fm_dev_add_data.command("beliefs") +@with_appcontext +@click.argument("file", type=click.Path(exists=True)) +@click.option( + "--sensor-id", + required=True, + type=click.IntRange(min=1), + help="Sensor to which the beliefs pertain.", +) +@click.option( + "--horizon", + required=False, + type=int, + help="Belief horizon in minutes (use positive horizon for ex-ante beliefs or negative horizon for ex-post beliefs).", +) +@click.option( + "--cp", + required=False, + type=click.FloatRange(0, 1), + help="Cumulative probability in the range [0, 1].", +) +@click.option( + "--allow-overwrite/--do-not-allow-overwrite", + default=False, + help="Allow overwriting possibly already existing data.\n" + "Not allowing overwriting can be much more efficient", +) +def add_beliefs( + file: str, + sensor_id: int, + horizon: Optional[int] = None, + cp: Optional[float] = None, + allow_overwrite: bool = False, +): + """Add sensor data from a csv file. + + Structure your csv file as follows: + + - One header line (will be ignored!) + - UTC datetimes in 1st column + - values in 2nd column + + For example: + + Date,Inflow (cubic meter) + 2020-12-03 14:00,212 + 2020-12-03 14:10,215.6 + 2020-12-03 14:20,203.8 + + In case no --horizon is specified, the moment of executing this CLI command is taken + as the time at which the beliefs were recorded. + """ + sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none() + if sensor is None: + print(f"Failed to create beliefs: no sensor found with id {sensor_id}.") + return + source = ( + DataSource.query.filter(DataSource.name == "Seita") + .filter(DataSource.type == "CLI script") + .one_or_none() + ) + if not source: + print("SETTING UP CLI SCRIPT AS NEW DATA SOURCE...") + source = DataSource(name="Seita", type="CLI script") + db.session.add(source) + db.session.flush() # assigns id + bdf = tb.read_csv( + file, + sensor, + source=source, + cumulative_probability=cp, + parse_dates=True, + infer_datetime_format=True, + **( + dict(belief_horizon=timedelta(minutes=horizon)) + if horizon is not None + else dict( + belief_time=server_now().astimezone(pytz.timezone(sensor.timezone)) + ) + ), + ) + try: + TimedBelief.add( + bdf, + expunge_session=True, + allow_overwrite=allow_overwrite, + bulk_save_objects=True, + commit_transaction=True, + ) + print(f"Successfully created beliefs\n{bdf}") + except IntegrityError as e: + db.session.rollback() + print(f"Failed to create beliefs due to the following error: {e.orig}") + if not allow_overwrite: + print("As a possible workaround, use the --allow-overwrite flag.") + + @fm_add_data.command("forecasts") @with_appcontext @click.option( @@ -338,6 +445,7 @@ def collect_weather_data(region, location, num_cells, method, store_in_db): app.cli.add_command(fm_add_data) +app.cli.add_command(fm_dev_add_data) def check_timezone(timezone): diff --git a/requirements/app.in b/requirements/app.in index 09d7da32f..37e6ee956 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -32,7 +32,7 @@ netCDF4 siphon tables timetomodel>=0.6.8 -timely-beliefs>=1.3.0 +timely-beliefs>=1.4.0 python-dotenv # a backport, not needed in Python3.8 importlib_metadata diff --git a/requirements/app.txt b/requirements/app.txt index 35570ffd7..8827feb91 100644 --- a/requirements/app.txt +++ b/requirements/app.txt @@ -317,7 +317,7 @@ tables==3.6.1 # via -r requirements/app.in threadpoolctl==2.1.0 # via scikit-learn -timely-beliefs==1.3.0 +timely-beliefs==1.4.0 # via -r requirements/app.in timetomodel==0.6.9 # via -r requirements/app.in From 8ed869d8d1fa84800a826c975ab9c51bcbfe2bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 21 Apr 2021 11:45:22 +0200 Subject: [PATCH 05/24] small doc improvements from user testing (#97) * small improvements to getting-started guide from user testing * more clarity about resolutions * add a note how to use other Python files in plugins * reporting API changes under the more fitting versions * horizon is optional again, as the documentation says; move play mode specific handling of horizon and prior fields into the validator decorator (and ensure postPrognosis requires a horizon for API v1.x) * document demo and play mode; small fixes/improvements in demo mode & move away from mode-conditional public documentation * Explicit decorator parameter for inference of horizon and prior field on servers in play mode Co-authored-by: F.N. Claessen --- documentation/api/change_log.rst | 27 +++++++---- documentation/api/introduction.rst | 9 ++-- documentation/changelog.rst | 3 +- documentation/configuration.rst | 9 +++- documentation/dev/data.rst | 27 ++++++----- documentation/dev/modes.rst | 41 +++++++++++++++++ documentation/dev/plugins.rst | 19 +++++++- documentation/getting-started.rst | 3 +- documentation/index.rst | 1 + flexmeasures/api/common/responses.py | 4 +- flexmeasures/api/common/utils/validators.py | 46 +++++++++++-------- flexmeasures/api/v1/implementations.py | 4 +- flexmeasures/api/v1_1/implementations.py | 14 ++++-- .../api/v2_0/implementations/sensors.py | 16 +++---- .../data/scripts/cli_tasks/data_add.py | 4 +- flexmeasures/data/services/time_series.py | 3 ++ flexmeasures/ui/utils/plotting_utils.py | 6 ++- flexmeasures/ui/utils/view_utils.py | 2 +- flexmeasures/utils/app_utils.py | 2 +- to_pypi.sh | 12 ++--- 20 files changed, 179 insertions(+), 73 deletions(-) create mode 100644 documentation/dev/modes.rst diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index db5141fec..672284232 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -3,12 +3,11 @@ API change log =============== -v2.0 | 2021-04-02 +v2.0-2 | 2021-04-02 """"""""""""""""" - [**Breaking change**] Switched the interpretation of horizons to rolling horizons. - [**Breaking change**] Deprecated the use of ISO 8601 repeating time intervals to denote rolling horizons. -- [**Breaking change**] Deprecated the automatic inference of horizons for *postMeterData*, *postPrognosis*, *postPriceData* and *postWeatherData* endpoints for API version below v2.0. - Introduced the "prior" field for *postMeterData*, *postPrognosis*, *postPriceData* and *postWeatherData* endpoints. - Changed the Introduction section: @@ -18,16 +17,31 @@ v2.0 | 2021-04-02 - Rewrote relevant examples using horizon and prior fields. -v2.0 | 2021-02-19 +v2.0-1 | 2021-02-19 """"""""""""""""""" - REST endpoints for managing users: `/users/` (GET), `/user/` (GET, PATCH) and `/user//password-reset` (PATCH). -v2.0 | 2020-11-14 +v2.0-0 | 2020-11-14 """"""""""""""""""" - REST endpoints for managing assets: `/assets/` (GET, POST) and `/asset/` (GET, PATCH, DELETE). + +v1.3.9 | 2021-04-XX +""""""""""""""""" + +*Affects all versions since v1.0*. + +- Fixed regression by partially reverting the breaking change of v1.3-8: Re-instantiated automatic inference of horizons for Post requests for API versions below v2.0, but changed to inference policy: now inferring the data was recorded **right after each event** took place (leading to a zero horizon for each data point) rather than **after the last event** took place (which led to a different horizon for each data point); the latter had been the inference policy before v1.3-8. + +v1.3-8 | 2020-04-02 +""""""""""""""""""" + +*Affects all versions since v1.0*. + +- [**Breaking change**, partially reverted in v1.3-9] Deprecated the automatic inference of horizons for *postMeterData*, *postPrognosis*, *postPriceData* and *postWeatherData* endpoints for API version below v2.0. + v1.3-7 | 2020-12-16 """"""""""""""""""" @@ -155,10 +169,7 @@ v1.1-2 | 2018-08-15 - Added the *postPriceData* endpoint - Added a description of the *postPrognosis* endpoint in the Aggregator section - Added a description of the *postPriceData* endpoint in the Aggregator and Supplier sections - -.. ifconfig:: FLEXMEASURES_MODE == "play" - - - Added the *restoreData* endpoint +- Added the *restoreData* endpoint for servers in play mode v1.1-1 | 2018-08-06 """"""""""""""""""" diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 117113d51..6c58cfc0a 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -257,7 +257,9 @@ In case of a single group of connections, the message may be flattened to: Timeseries ^^^^^^^^^^ -Timestamps and durations are consistent with the ISO 8601 standard. All timestamps in requests to the API must be timezone-aware. The timezone indication "Z" indicates a zero offset from UTC. Additionally, we use the following shorthand for sequential values within a time interval: +Timestamps and durations are consistent with the ISO 8601 standard. The resolution of the data is implicit, see :ref:`resolutions`. + +All timestamps in requests to the API must be timezone-aware. The timezone indication "Z" indicates a zero offset from UTC. Additionally, we use the following shorthand for sequential values within a time interval: .. code-block:: json @@ -405,8 +407,9 @@ This denotes that the prognosed interval has 5 minutes left to be concluded. Resolutions ^^^^^^^^^^^ -Specifying a resolution is redundant for POST requests that contain both "values" and a "duration". -Also, posted data is checked against the required resolution of the assets which are posted to. +Specifying a resolution is redundant for POST requests that contain both "values" and a "duration" ― FlexMeasures computes the resolution by dividing the duration by the number of values. + +When POSTing data, FlexMeasures checks this computed resolution against the required resolution of the assets which are posted to. If these can't be matched (through upsampling), an error will occur. GET requests (such as *getMeterData*) return data in the resolution which the sensor is configured for. A "resolution" may be specified explicitly to obtain the data in downsampled form, diff --git a/documentation/changelog.rst b/documentation/changelog.rst index eeec74783..ff83f33a6 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -18,7 +18,8 @@ Bugfixes Infrastructure / Support ---------------------- * Updated dependencies, including Flask-Security-Too [see `PR #82 `_] -* Integration with `timely beliefs `_ lib: Sensor data as TimedBeliefs [see `PR #79 `_] +* Improved documentation after user feedback [see `PR #97 `_] +* Begin experimental integration with `timely beliefs `_ lib: Sensor data as TimedBeliefs [see `PR #79 `_] * Add sensors with CLI command currently meant for developers only [see `PR #83 `_] * Add data (beliefs about sensor events) with CLI command currently meant for developers only [see `PR #85 `_] diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 2262aaacc..954c09e4a 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -26,11 +26,14 @@ Level above which log messages are added to the log file. See the ``logging`` pa Default: ``logging.WARNING`` + +.. _modes-config: + FLEXMEASURES_MODE ^^^^^^^^^^^^^^^^^ The mode in which FlexMeasures is being run, e.g. "demo" or "play". -This is used to turn on certain extra behaviours. +This is used to turn on certain extra behaviours, see :ref:`modes-dev` for details. Default: ``""`` @@ -369,6 +372,8 @@ Default: ``None`` Demonstrations -------------- +.. _demo-credentials-config: + FLEXMEASURES_PUBLIC_DEMO_CREDENTIALS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -376,6 +381,8 @@ When ``FLEXMEASURES_MODE=demo``\ , this can hold login credentials (demo user em Default: ``None`` +.. _demo-year-config: + FLEXMEASURES_DEMO_YEAR ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/documentation/dev/data.rst b/documentation/dev/data.rst index 39a5ca81f..4db5635db 100644 --- a/documentation/dev/data.rst +++ b/documentation/dev/data.rst @@ -100,36 +100,35 @@ Or, from within Postgres console: CREATE DATABASE flexmeasures_test WITH OWNER = flexmeasures_test; -Log in as the postgres superuser and connect to your newly-created database: +Finally, test if you can log in as the flexmeasures user: .. code-block:: bash - sudo -u postgres psql + psql -U flexmeasures --password -h 127.0.0.1 -d flexmeasures .. code-block:: sql - \connect flexmeasures + \q + +Add Postgres Extensions to your database(s) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +To find the nearest sensors, FlexMeasures needs some extra POstgres support. Add the following extensions while logged in as the postgres superuser: +.. code-block:: bash + + sudo -u postgres psql + .. code-block:: sql + \connect flexmeasures CREATE EXTENSION cube; CREATE EXTENSION earthdistance; -Connect to the ``flexmeasures_test`` database and repeat creating these extensions there. Then ``exit``. - -Finally, try logging in as the flexmeasures user once: - -.. code-block:: bash - - psql -U flexmeasures --password -h 127.0.0.1 -d flexmeasures - -.. code-block:: sql - - \q +If you have it, connect to the ``flexmeasures_test`` database and repeat creating these extensions there. Then ``exit``. Configure FlexMeasures app for that database diff --git a/documentation/dev/modes.rst b/documentation/dev/modes.rst new file mode 100644 index 000000000..238082d0f --- /dev/null +++ b/documentation/dev/modes.rst @@ -0,0 +1,41 @@ +.. _modes-dev: + +Modes +============ + +FlexMeasures can be run in specific modes (see the :ref:`modes-config` config setting). +This is useful for certain special situations. Two are supported out of the box and we document here +how FlexMeasures behaves differently in these modes. + +Demo +------- + +In this mode, the server is assumed to be used as a demonstration tool. Most of the following adaptations therefore happen in the UI. + +- [Data] Demo data is often from an older source, and it's a hassle to change the year to the current year. FlexMeasures allows to set :ref:`demo-year-config` and when in ``demo`` mode, the current year will be translated to that year in the background. +- [UI] Logged-in users can view queues on the demo server (usually only admins can do that) +- [UI] Demo servers often display login credentials, so visitors can try out functionality. Use the :ref:`demo-credentials-config` config setting to do this. +- [UI] The dashboard shows all non-empty asset groups, instead of only the ones for the current user. +- [UI] The analytics page mocks confidence intervals around power, price and weather data, so that the demo data doesn't need to have them. +- [UI] The portfolio page mocks flexibility numbers and a mocked control action. + +Play +------ + +In this mode, the server is assumed to be used to run simulations. + +Big features +^^^^^^^^^^^^^ + +- [Data] Allows overwriting existing data when saving data to the database. +- [API] The inferred recording time of incoming data is immediately after the event took place, rather than the actual time at which the server received the data. +- [API] Posting price or weather data does not trigger forecasting jobs. +- [API] The ``restoreData`` endpoint is registered, enabling database resets through the API. +- [API] When posting weather data for a new location, a new weather sensor is automatically created, instead of returning the nearest available weather sensor to post data to. + +Small features +^^^^^^^^^^^^^^^ + +- [API] Posted UDI events are not enforced to be consecutive. +- [API] Names in ``GetConnectionResponse`` are the connections' unique database names rather than their display names (this feature is planned to be deprecated). +- [UI] The dashboard plot showing the latest power value is not enforced to lie in the past (in case of simulating future values). diff --git a/documentation/dev/plugins.rst b/documentation/dev/plugins.rst index c422b37a0..8fbf02385 100644 --- a/documentation/dev/plugins.rst +++ b/documentation/dev/plugins.rst @@ -76,4 +76,21 @@ All else that is needed for this showcase (not shown here) is ``/ou -.. note:: Plugin views can also be added to the FlexMeasures UI menu ― just name them in the config setting :ref:`menu-config`. \ No newline at end of file +.. note:: Plugin views can also be added to the FlexMeasures UI menu ― just name them in the config setting :ref:`menu-config`. + + +Using other files in your plugin +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Say you want to include other Python files in your plugin, importing them in your ``__init__.py`` file. +This can be done if you put the plugin path on the import path. Do it like this in your ``__init__.py``: + +.. code-block:: python + + import os + import sys + + HERE = os.path.dirname(os.path.abspath(__file__)) + sys.path.insert(0, HERE) + + from my_other_file import my_function \ No newline at end of file diff --git a/documentation/getting-started.rst b/documentation/getting-started.rst index ca95bc513..4bbf9d6d7 100644 --- a/documentation/getting-started.rst +++ b/documentation/getting-started.rst @@ -17,6 +17,7 @@ Install dependencies and the ``flexmeasures`` platform itself: pip install flexmeasures +.. note:: With newer Python versions and Windows, some smaller dependencies (e.g. ``tables`` or ``rq-win``) might cause issues as support is often slower. You might overcome this with a little research, by `installing from wheels `_ or `from the repo `_, respectively. Make a secret key for sessions and password salts @@ -217,7 +218,7 @@ To collect weather measurements and forecasts from the DarkSky API, there is a t flexmeasures add external-weather-forecasts --location 33.4366,126.5269 --store-in-db -.. note:: DarkSky is not handing out tokens anymore, as they have been bought by Apple (see `issue 3 `_). +.. note:: DarkSky is not handing out tokens any more, as they have been bought by Apple (see `issue 3 `_). Preparing the job queue database and start workers diff --git a/documentation/index.rst b/documentation/index.rst index d5af9d4f2..b05c75b0c 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -85,6 +85,7 @@ The platform operator of FlexMeasures can be an Aggregator. dev/api dev/ci dev/plugins + dev/modes .. toctree:: :caption: Integrations diff --git a/flexmeasures/api/common/responses.py b/flexmeasures/api/common/responses.py index d381fa8d5..6b2133d06 100644 --- a/flexmeasures/api/common/responses.py +++ b/flexmeasures/api/common/responses.py @@ -64,7 +64,7 @@ def invalid_domain(message: str) -> ResponseTuple: return dict(result="Rejected", status="INVALID_DOMAIN", message=message), 400 -@BaseMessage("The prognosis horizon in your request could not be parsed.") +@BaseMessage("The horizon field in your request could not be parsed.") def invalid_horizon(message: str) -> ResponseTuple: return dict(result="Rejected", status="INVALID_HORIZON", message=message), 400 @@ -86,7 +86,7 @@ def invalid_ptu_duration(message: str) -> ResponseTuple: ) -@BaseMessage("Only the following resolutions are supported:") +@BaseMessage("Only the following resolutions in the data are supported:") def unapplicable_resolution(message: str) -> ResponseTuple: return dict(result="Rejected", status="INVALID_RESOLUTION", message=message), 400 diff --git a/flexmeasures/api/common/utils/validators.py b/flexmeasures/api/common/utils/validators.py index 6fe7b318c..e41c4d80f 100644 --- a/flexmeasures/api/common/utils/validators.py +++ b/flexmeasures/api/common/utils/validators.py @@ -296,7 +296,9 @@ def decorated_service(*args, **kwargs): return wrapper -def optional_prior_accepted(ex_post: bool = False, infer_missing: bool = True): +def optional_prior_accepted( + ex_post: bool = False, infer_missing: bool = True, infer_missing_play: bool = False +): """Decorator which specifies that a GET or POST request accepts an optional prior. It parses relevant form data and sets the "prior" keyword param. @@ -304,9 +306,15 @@ def optional_prior_accepted(ex_post: bool = False, infer_missing: bool = True): - Denotes "at least before " - This results in the filter belief_time_window = (None, prior) - Optionally, an ex_post flag can be passed to the decorator to indicate that only ex-post datetimes are allowed. - As a useful setting (at least for POST requests), set infer_missing to True to have servers - (that are not in play mode) derive a prior from the server time. + Interpretation for POST requests: + - Denotes "recorded to some datetime, + - this results in the assignment belief_time = prior + + :param ex_post: if True, only ex-post datetimes are allowed. + :param infer_missing: if True, servers assume that the belief_time of posted + values is server time. This setting is meant to be used for POST requests. + :param infer_missing_play: if True, servers in play mode assume that the belief_time of posted + values is server time. This setting is meant to be used for POST requests. """ def wrapper(fn): @@ -332,11 +340,11 @@ def decorated_service(*args, **kwargs): if prior < knowledge_time: extra_info = "Meter data can only be observed after the fact." return invalid_horizon(extra_info) - elif ( - infer_missing is True - and current_app.config.get("FLEXMEASURES_MODE", "") != "play" + elif infer_missing is True or ( + infer_missing_play is True + and current_app.config.get("FLEXMEASURES_MODE", "") == "play" ): - # A missing prior is inferred by the server (if not in play mode) + # A missing prior is inferred by the server prior = server_now() else: # Otherwise, a missing prior is fine (a horizon may still be inferred by the server) @@ -353,6 +361,7 @@ def decorated_service(*args, **kwargs): def optional_horizon_accepted( # noqa C901 ex_post: bool = False, infer_missing: bool = True, + infer_missing_play: bool = False, accept_repeating_interval: bool = False, ): """Decorator which specifies that a GET or POST request accepts an optional horizon. @@ -376,11 +385,13 @@ def optional_horizon_accepted( # noqa C901 def post_meter_data(horizon): return 'Meter data posted' - :param ex_post: if True, only non-positive horizons are allowed. - :param infer_missing: if True, servers that are in play mode assume that the belief_horizon of posted - values is 0 hours. This setting is meant to be used for POST requests. - :param accept_repeating_interval: if True, the "rolling" keyword param is also set - (this was used for POST requests before v2.0) + :param ex_post: if True, only non-positive horizons are allowed. + :param infer_missing: if True, servers assume that the belief_horizon of posted + values is 0 hours. This setting is meant to be used for POST requests. + :param infer_missing_play: if True, servers in play mode assume that the belief_horizon of posted + values is 0 hours. This setting is meant to be used for POST requests. + :param accept_repeating_interval: if True, the "rolling" keyword param is also set + (this was used for POST requests before v2.0) """ def wrapper(fn): @@ -410,15 +421,12 @@ def decorated_service(*args, **kwargs): "For example: R/P1D should be replaced by P1D." ) return invalid_horizon(extra_info) - elif ( - infer_missing is True + elif infer_missing is True or ( + infer_missing_play is True and current_app.config.get("FLEXMEASURES_MODE", "") == "play" ): - # A missing horizon is set to zero for servers in play mode + # A missing horizon is set to zero horizon = timedelta(hours=0) - elif infer_missing is True and accept_repeating_interval is True: - extra_info = "Missing horizons are no longer accepted for API versions below v2.0." - return invalid_horizon(extra_info) else: # Otherwise, a missing horizon is fine (a prior may still be inferred by the server) horizon = None diff --git a/flexmeasures/api/v1/implementations.py b/flexmeasures/api/v1/implementations.py index 373677ddb..893f4c68d 100644 --- a/flexmeasures/api/v1/implementations.py +++ b/flexmeasures/api/v1/implementations.py @@ -97,7 +97,9 @@ def get_meter_data_response( @units_accepted("power", "MW") @assets_required("connection") @values_required -@optional_horizon_accepted(ex_post=True, accept_repeating_interval=True) +@optional_horizon_accepted( + ex_post=True, infer_missing=True, accept_repeating_interval=True +) @period_required @post_data_checked_for_required_resolution("connection") @as_json diff --git a/flexmeasures/api/v1_1/implementations.py b/flexmeasures/api/v1_1/implementations.py index 3efdf2ffe..5de2bb3cd 100644 --- a/flexmeasures/api/v1_1/implementations.py +++ b/flexmeasures/api/v1_1/implementations.py @@ -13,6 +13,7 @@ invalid_domain, invalid_unit, unrecognized_market, + invalid_horizon, ) from flexmeasures.api.common.utils.api_utils import ( save_to_db, @@ -62,7 +63,7 @@ def get_connection_response(): @type_accepted("PostPriceDataRequest") @units_accepted("price", "EUR/MWh", "KRW/kWh") @assets_required("market") -@optional_horizon_accepted(accept_repeating_interval=True) +@optional_horizon_accepted(infer_missing=True, accept_repeating_interval=True) @values_required @period_required @post_data_checked_for_required_resolution("market") @@ -137,7 +138,7 @@ def post_price_data_response( @type_accepted("PostWeatherDataRequest") @unit_required @assets_required("sensor") -@optional_horizon_accepted(accept_repeating_interval=True) +@optional_horizon_accepted(infer_missing=True, accept_repeating_interval=True) @values_required @period_required @post_data_checked_for_required_resolution("sensor") @@ -263,7 +264,9 @@ def get_prognosis_response( @units_accepted("power", "MW") @assets_required("connection") @values_required -@optional_horizon_accepted(ex_post=False, accept_repeating_interval=True) +@optional_horizon_accepted( + ex_post=False, infer_missing=False, accept_repeating_interval=True +) @period_required @post_data_checked_for_required_resolution("connection") @as_json @@ -281,6 +284,11 @@ def post_prognosis_response( Store the new power values for each asset. """ + if horizon is None: + # API versions before v2.0 cannot handle a missing horizon, because there is no prior + extra_info = "Please specify the horizon field using an ISO 8601 duration (such as 'PT24H')." + return invalid_horizon(extra_info) + return create_connection_and_value_groups( unit, generic_asset_name_groups, value_groups, horizon, rolling, start, duration ) diff --git a/flexmeasures/api/v2_0/implementations/sensors.py b/flexmeasures/api/v2_0/implementations/sensors.py index a4c8d6382..093a7f8b2 100644 --- a/flexmeasures/api/v2_0/implementations/sensors.py +++ b/flexmeasures/api/v2_0/implementations/sensors.py @@ -46,8 +46,8 @@ @type_accepted("PostPriceDataRequest") @units_accepted("price", "EUR/MWh", "KRW/kWh") @assets_required("market") -@optional_horizon_accepted() -@optional_prior_accepted() +@optional_horizon_accepted(infer_missing=False, infer_missing_play=True) +@optional_prior_accepted(infer_missing=True, infer_missing_play=False) @values_required @period_required @post_data_checked_for_required_resolution("market") @@ -130,8 +130,8 @@ def post_price_data_response( # noqa C901 @type_accepted("PostWeatherDataRequest") @unit_required @assets_required("sensor") -@optional_horizon_accepted() -@optional_prior_accepted() +@optional_horizon_accepted(infer_missing=False, infer_missing_play=True) +@optional_prior_accepted(infer_missing=True, infer_missing_play=False) @values_required @period_required @post_data_checked_for_required_resolution("sensor") @@ -222,8 +222,8 @@ def post_weather_data_response( # noqa: C901 @units_accepted("power", "MW") @assets_required("connection") @values_required -@optional_horizon_accepted(ex_post=True) -@optional_prior_accepted(ex_post=True) +@optional_horizon_accepted(ex_post=True, infer_missing=False, infer_missing_play=True) +@optional_prior_accepted(ex_post=True, infer_missing=True, infer_missing_play=False) @period_required @post_data_checked_for_required_resolution("connection") @as_json @@ -254,8 +254,8 @@ def post_meter_data_response( @units_accepted("power", "MW") @assets_required("connection") @values_required -@optional_horizon_accepted(ex_post=False) -@optional_prior_accepted(ex_post=False) +@optional_horizon_accepted(ex_post=False, infer_missing=False, infer_missing_play=False) +@optional_prior_accepted(ex_post=False, infer_missing=True, infer_missing_play=False) @period_required @post_data_checked_for_required_resolution("connection") @as_json diff --git a/flexmeasures/data/scripts/cli_tasks/data_add.py b/flexmeasures/data/scripts/cli_tasks/data_add.py index ab8f0a9f7..f0937d247 100644 --- a/flexmeasures/data/scripts/cli_tasks/data_add.py +++ b/flexmeasures/data/scripts/cli_tasks/data_add.py @@ -337,14 +337,14 @@ def add_beliefs( multiple=True, type=click.Choice(["1", "6", "24", "48"]), default=["1", "6", "24", "48"], - help="Forecasting horizon in hours. This argument can be given multiple times.", + help="Forecasting horizon in hours. This argument can be given multiple times. Defaults to all possible horizons.", ) @click.option( "--as-job", is_flag=True, help="Whether to queue a forecasting job instead of computing directly." " Useful to run locally and create forecasts on a remote server. In that case, just point the redis db in your" - " config settings to that of the remote server. To process the job, run a worker to process the forecasting queue.", + " config settings to that of the remote server. To process the job, run a worker to process the forecasting queue. Defaults to False.", ) def create_forecasts( asset_type: str = None, diff --git a/flexmeasures/data/services/time_series.py b/flexmeasures/data/services/time_series.py index 44fa20757..1c0af44eb 100644 --- a/flexmeasures/data/services/time_series.py +++ b/flexmeasures/data/services/time_series.py @@ -257,6 +257,9 @@ def convert_query_window_for_demo( end = (query_window[-1] + timedelta(days=1)).replace(year=demo_year) else: end = query_window[-1] + + if start >= end: + start, end = (end, start) return start, end diff --git a/flexmeasures/ui/utils/plotting_utils.py b/flexmeasures/ui/utils/plotting_utils.py index 9c407e01b..ac69d44f4 100644 --- a/flexmeasures/ui/utils/plotting_utils.py +++ b/flexmeasures/ui/utils/plotting_utils.py @@ -564,7 +564,11 @@ def get_latest_power_as_plot(asset: Asset, small: bool = False) -> Tuple[str, st First returned string is the measurement time, second string is the html string.""" if current_app.config.get("FLEXMEASURES_MODE", "") == "demo": - before = server_now().replace(year=2015) + demo_year = current_app.config.get("FLEXMEASURES_DEMO_YEAR", None) + if demo_year is None: + before = server_now() + else: + before = server_now().replace(year=demo_year) elif current_app.config.get("FLEXMEASURES_MODE", "") == "play": before = None # type:ignore else: diff --git a/flexmeasures/ui/utils/view_utils.py b/flexmeasures/ui/utils/view_utils.py index 6ee924e06..1b50a9caa 100644 --- a/flexmeasures/ui/utils/view_utils.py +++ b/flexmeasures/ui/utils/view_utils.py @@ -131,7 +131,7 @@ def set_time_range_for_session(): .astimezone(time_utils.get_timezone()) ) - # Our demo server works only with the current year's data + # Our demo server's UI should only work with the current year's data if current_app.config.get("FLEXMEASURES_MODE", "") == "demo": session["start_time"] = session["start_time"].replace(year=datetime.now().year) session["end_time"] = session["end_time"].replace(year=datetime.now().year) diff --git a/flexmeasures/utils/app_utils.py b/flexmeasures/utils/app_utils.py index a075d14b7..4065d79bd 100644 --- a/flexmeasures/utils/app_utils.py +++ b/flexmeasures/utils/app_utils.py @@ -76,7 +76,7 @@ def register_plugins(app: Flask): - Your plugin folders contains an __init__.py file. - In this init, you define a Blueprint object called _bp - We'll refer to the plugins with the name of your plugin folders (last part of tthe path). + We'll refer to the plugins with the name of your plugin folders (last part of the path). """ plugin_paths = app.config.get("FLEXMEASURES_PLUGIN_PATHS", "") if not isinstance(plugin_paths, list): diff --git a/to_pypi.sh b/to_pypi.sh index a26723733..22342682c 100755 --- a/to_pypi.sh +++ b/to_pypi.sh @@ -8,24 +8,24 @@ # The version comes from setuptools_scm. See `python setup.py --version`. # setuptools_scm works via git tags that should implement a semantic versioning scheme, e.g. v0.2.3 # -# If there were zero commits since since tag, we have a real release and the version basicaly *is* what the tag says. -# Otherwise, the version also include a .devN identifier, where N is the number of commits since the last version tag. +# If there were zero commits since since tag, we have a real release and the version basically *is* what the tag says. +# Otherwise, the version also includes a .devN identifier, where N is the number of commits since the last version tag. # # More information on creating a dev release # ------------------------------------------- # Note that the only way to create a new dev release is to add another commit on your development branch. -# It might have been convenient to not have to commit to do that (for exoerimenting with very small changes), +# It might have been convenient to not have to commit to do that (for experimenting with very small changes), # but we decided against that. Let's explore why for a bit: # # First, setuptools_scm has the ability to add a local scheme (git commit and date/time) to the version, # but we've disabled that, as that extra part isn't formatted in a way that Pypi accepts it. -# Another way would have been to add a local version identifier ("+M", not the plus sign), +# Another way would have been to add a local version identifier ("+M", note the plus sign), # which is allowed in PEP 440 but explicitly disallowed by Pypi. # Finally, if we simply add a number to .devN (-> .devNM), the ordering of dev versions would be # disturbed after the next local commit (e.g. we add 1 to .dev4, making it .dev41, and then the next version, .dev5, # is not the highest version chosen by PyPi). # -# So we'll use these tools as the experts intend us to. +# So we'll use these tools as the experts intended. # If you want, you can read more about acceptable versions in PEP 440: https://www.python.org/dev/peps/pep-0440/ @@ -35,4 +35,4 @@ pip -q install twine python setup.py egg_info sdist python setup.py egg_info bdist_wheel -twine upload dist/* \ No newline at end of file +twine upload dist/* From 04e1076d9322353020135bff81df5f4bda564db4 Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Thu, 29 Apr 2021 13:03:24 +0200 Subject: [PATCH 06/24] Allow reading beliefs from xlsx files, and from data files with multiple headers (#103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Allow reading beliefs from xlsx files, and from data files with multiple headers * Allow creating new sources and looking up existing sources * Refactor get_or_create_source * Optional flush * Take into account that current_user can be of LocalProxy type, and use get_current_object to get the User object * Improve docstring Co-authored-by: Nicolas Höning Co-authored-by: F.N. Claessen Co-authored-by: Nicolas Höning --- flexmeasures/api/common/utils/api_utils.py | 12 --- flexmeasures/api/v1/implementations.py | 4 +- flexmeasures/api/v1_1/implementations.py | 6 +- .../api/v2_0/implementations/sensors.py | 8 +- ...rce_id_as_primary_key_for_timed_beliefs.py | 39 ++++++++ flexmeasures/data/models/data_sources.py | 35 ++++++- flexmeasures/data/models/user.py | 11 +++ .../data/scripts/cli_tasks/data_add.py | 98 ++++++++++++++----- requirements/app.in | 2 +- requirements/app.txt | 2 +- 10 files changed, 170 insertions(+), 47 deletions(-) create mode 100644 flexmeasures/data/migrations/versions/04f0e2d2924a_add_source_id_as_primary_key_for_timed_beliefs.py diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index e56c15952..dbbcca910 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -13,9 +13,7 @@ from flexmeasures.data import db from flexmeasures.data.models.assets import Asset, Power from flexmeasures.data.models.markets import Market, Price -from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.weather import WeatherSensor, Weather -from flexmeasures.data.models.user import User from flexmeasures.data.utils import save_to_session from flexmeasures.utils.entity_address_utils import parse_entity_address from flexmeasures.api.common.responses import ( @@ -284,16 +282,6 @@ def asset_replace_name_with_id(connections_as_name: List[str]) -> List[str]: return connections_as_ea -def get_or_create_user_data_source(user: User) -> DataSource: - data_source = DataSource.query.filter(DataSource.user == user).one_or_none() - if not data_source: - current_app.logger.info("SETTING UP USER AS NEW DATA SOURCE...") - data_source = DataSource(user=user) - db.session.add(data_source) - db.session.flush() # flush so that we can reference the new object in the current db session - return data_source - - def get_weather_sensor_by( weather_sensor_type_name: str, latitude: float = 0, longitude: float = 0 ) -> WeatherSensor: diff --git a/flexmeasures/api/v1/implementations.py b/flexmeasures/api/v1/implementations.py index 893f4c68d..8ddbc3b7a 100644 --- a/flexmeasures/api/v1/implementations.py +++ b/flexmeasures/api/v1/implementations.py @@ -12,6 +12,7 @@ EntityAddressException, ) from flexmeasures.data.models.assets import Asset, Power +from flexmeasures.data.models.data_sources import get_or_create_source from flexmeasures.data.services.resources import get_assets from flexmeasures.data.services.forecasting import create_forecasting_jobs from flexmeasures.api.common.responses import ( @@ -24,7 +25,6 @@ ) from flexmeasures.api.common.utils.api_utils import ( groups_to_dict, - get_or_create_user_data_source, save_to_db, ) from flexmeasures.api.common.utils.validators import ( @@ -244,7 +244,7 @@ def create_connection_and_value_groups( # noqa: C901 current_app.logger.info("POSTING POWER DATA") - data_source = get_or_create_user_data_source(current_user) + data_source = get_or_create_source(current_user) user_assets = get_assets() if not user_assets: current_app.logger.info("User doesn't seem to have any assets") diff --git a/flexmeasures/api/v1_1/implementations.py b/flexmeasures/api/v1_1/implementations.py index 5de2bb3cd..abf3c8b93 100644 --- a/flexmeasures/api/v1_1/implementations.py +++ b/flexmeasures/api/v1_1/implementations.py @@ -17,7 +17,6 @@ ) from flexmeasures.api.common.utils.api_utils import ( save_to_db, - get_or_create_user_data_source, ) from flexmeasures.api.common.utils.validators import ( type_accepted, @@ -38,6 +37,7 @@ create_connection_and_value_groups, ) from flexmeasures.api.common.utils.api_utils import get_weather_sensor_by +from flexmeasures.data.models.data_sources import get_or_create_source from flexmeasures.data.models.markets import Market, Price from flexmeasures.data.models.weather import Weather from flexmeasures.data.services.resources import get_assets @@ -80,7 +80,7 @@ def post_price_data_response( current_app.logger.info("POSTING PRICE DATA") - data_source = get_or_create_user_data_source(current_user) + data_source = get_or_create_source(current_user) prices = [] forecasting_jobs = [] for market_group, value_group in zip(generic_asset_name_groups, value_groups): @@ -155,7 +155,7 @@ def post_weather_data_response( # noqa: C901 current_app.logger.info("POSTING WEATHER DATA") - data_source = get_or_create_user_data_source(current_user) + data_source = get_or_create_source(current_user) weather_measurements = [] forecasting_jobs = [] for sensor_group, value_group in zip(generic_asset_name_groups, value_groups): diff --git a/flexmeasures/api/v2_0/implementations/sensors.py b/flexmeasures/api/v2_0/implementations/sensors.py index 093a7f8b2..b8a5cb687 100644 --- a/flexmeasures/api/v2_0/implementations/sensors.py +++ b/flexmeasures/api/v2_0/implementations/sensors.py @@ -15,7 +15,6 @@ ResponseTuple, ) from flexmeasures.api.common.utils.api_utils import ( - get_or_create_user_data_source, get_weather_sensor_by, save_to_db, determine_belief_timing, @@ -33,6 +32,7 @@ values_required, ) from flexmeasures.data.models.assets import Asset, Power +from flexmeasures.data.models.data_sources import get_or_create_source from flexmeasures.data.models.markets import Market, Price from flexmeasures.data.models.weather import Weather from flexmeasures.data.services.forecasting import create_forecasting_jobs @@ -69,7 +69,7 @@ def post_price_data_response( # noqa C901 current_app.logger.info("POSTING PRICE DATA") - data_source = get_or_create_user_data_source(current_user) + data_source = get_or_create_source(current_user) prices = [] forecasting_jobs = [] for market_group, event_values in zip(generic_asset_name_groups, value_groups): @@ -152,7 +152,7 @@ def post_weather_data_response( # noqa: C901 current_app.logger.info("POSTING WEATHER DATA") - data_source = get_or_create_user_data_source(current_user) + data_source = get_or_create_source(current_user) weather_measurements = [] forecasting_jobs = [] for sensor_group, event_values in zip(generic_asset_name_groups, value_groups): @@ -301,7 +301,7 @@ def post_power_data( current_app.logger.info("POSTING POWER DATA") - data_source = get_or_create_user_data_source(current_user) + data_source = get_or_create_source(current_user) user_assets = get_assets() if not user_assets: current_app.logger.info("User doesn't seem to have any assets") diff --git a/flexmeasures/data/migrations/versions/04f0e2d2924a_add_source_id_as_primary_key_for_timed_beliefs.py b/flexmeasures/data/migrations/versions/04f0e2d2924a_add_source_id_as_primary_key_for_timed_beliefs.py new file mode 100644 index 000000000..fe2d80d88 --- /dev/null +++ b/flexmeasures/data/migrations/versions/04f0e2d2924a_add_source_id_as_primary_key_for_timed_beliefs.py @@ -0,0 +1,39 @@ +"""add source id as primary key for timed beliefs + +Revision ID: 04f0e2d2924a +Revises: e62ac5f519d7 +Create Date: 2021-04-10 13:53:22.561718 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "04f0e2d2924a" +down_revision = "e62ac5f519d7" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_constraint("timed_belief_pkey", "timed_belief") + op.create_primary_key( + "timed_belief_pkey", + "timed_belief", + [ + "event_start", + "belief_horizon", + "cumulative_probability", + "sensor_id", + "source_id", + ], + ) + + +def downgrade(): + op.drop_constraint("timed_belief_pkey", "timed_belief") + op.create_primary_key( + "timed_belief_pkey", + "timed_belief", + ["event_start", "belief_horizon", "cumulative_probability", "sensor_id"], + ) diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 0ebed3eb5..3c502745b 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -1,9 +1,10 @@ -from typing import Optional +from typing import Optional, Union import timely_beliefs as tb +from flask import current_app from flexmeasures.data.config import db -from flexmeasures.data.models.user import User +from flexmeasures.data.models.user import User, is_user class DataSource(db.Model, tb.BeliefSourceDBMixin): @@ -53,3 +54,33 @@ def label(self): def __repr__(self): return "" % (self.id, self.label) + + +def get_or_create_source( + source: Union[User, str], source_type: str = "user", flush: bool = True +) -> DataSource: + query = DataSource.query.filter(DataSource.type == source_type) + if is_user(source): + query = query.filter(DataSource.user == source) + elif isinstance(source, str): + query = query.filter(DataSource.name == source) + else: + raise TypeError("source should be of type User or str") + _source = query.one_or_none() + if not _source: + current_app.logger.info(f"Setting up '{source}' as new data source...") + if is_user(source): + _source = DataSource(user=source) + else: + _source = DataSource(name=source, type=source_type) + db.session.add(_source) + if flush: + # assigns id so that we can reference the new object in the current db session + db.session.flush() + return _source + + +def get_source_or_none(source: int, source_type: str) -> Optional[DataSource]: + query = DataSource.query.filter(DataSource.type == source_type) + query = query.filter(DataSource.id == int(source)) + return query.one_or_none() diff --git a/flexmeasures/data/models/user.py b/flexmeasures/data/models/user.py index c2fa4ea9f..fca257a48 100644 --- a/flexmeasures/data/models/user.py +++ b/flexmeasures/data/models/user.py @@ -98,3 +98,14 @@ def remember_login(the_app, user): if user.login_count is None: user.login_count = 0 user.login_count = user.login_count + 1 + + +def is_user(o) -> bool: + """True if object is or proxies a User, False otherwise. + + Takes into account that object can be of LocalProxy type, and + uses get_current_object to get the underlying (User) object. + """ + return isinstance(o, User) or ( + hasattr(o, "_get_current_object") and isinstance(o._get_current_object(), User) + ) diff --git a/flexmeasures/data/scripts/cli_tasks/data_add.py b/flexmeasures/data/scripts/cli_tasks/data_add.py index f0937d247..40090296e 100644 --- a/flexmeasures/data/scripts/cli_tasks/data_add.py +++ b/flexmeasures/data/scripts/cli_tasks/data_add.py @@ -20,7 +20,10 @@ from flexmeasures.data.models.assets import Asset, AssetSchema from flexmeasures.data.models.markets import Market from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorSchema -from flexmeasures.data.models.data_sources import DataSource +from flexmeasures.data.models.data_sources import ( + get_or_create_source, + get_source_or_none, +) from flexmeasures.utils.time_utils import server_now @@ -220,6 +223,12 @@ def add_initial_structure(): type=click.IntRange(min=1), help="Sensor to which the beliefs pertain.", ) +@click.option( + "--source", + required=True, + type=str, + help="Source of the beliefs (an existing source id or name, or a new name).", +) @click.option( "--horizon", required=False, @@ -238,16 +247,55 @@ def add_initial_structure(): help="Allow overwriting possibly already existing data.\n" "Not allowing overwriting can be much more efficient", ) +@click.option( + "--skiprows", + required=False, + default=1, + type=int, + help="Number of rows to skip from the top. Set to >1 to skip additional headers.", +) +@click.option( + "--nrows", + required=False, + type=int, + help="Number of rows to read (from the top, after possibly skipping rows). Leave out to read all rows.", +) +@click.option( + "--datecol", + required=False, + default=0, + type=int, + help="Column number with datetimes (0 is 1st column, the default)", +) +@click.option( + "--valuecol", + required=False, + default=1, + type=int, + help="Column number with values (1 is 2nd column, the default)", +) +@click.option( + "--sheet_number", + required=False, + type=int, + help="[For xls or xlsx files] Sheet number with the data (0 is 1st sheet)", +) def add_beliefs( file: str, sensor_id: int, + source: str, horizon: Optional[int] = None, cp: Optional[float] = None, allow_overwrite: bool = False, + skiprows: int = 1, + nrows: Optional[int] = None, + datecol: int = 0, + valuecol: int = 1, + sheet_number: Optional[int] = None, ): - """Add sensor data from a csv file. + """Add sensor data from a csv file (also accepts xls or xlsx). - Structure your csv file as follows: + To use default settings, structure your csv file as follows: - One header line (will be ignored!) - UTC datetimes in 1st column @@ -267,30 +315,36 @@ def add_beliefs( if sensor is None: print(f"Failed to create beliefs: no sensor found with id {sensor_id}.") return - source = ( - DataSource.query.filter(DataSource.name == "Seita") - .filter(DataSource.type == "CLI script") - .one_or_none() - ) - if not source: - print("SETTING UP CLI SCRIPT AS NEW DATA SOURCE...") - source = DataSource(name="Seita", type="CLI script") - db.session.add(source) - db.session.flush() # assigns id + if source.isdigit(): + _source = get_source_or_none(int(source), source_type="CLI script") + if not _source: + print(f"Failed to find source {source}.") + return + else: + _source = get_or_create_source(source, source_type="CLI script") + + # Set up optional parameters for read_csv + kwargs = dict() + if file.split(".")[-1].lower() == "csv": + kwargs["infer_datetime_format"] = True + if sheet_number is not None: + kwargs["sheet_name"] = sheet_number + if horizon is not None: + kwargs["belief_horizon"] = timedelta(minutes=horizon) + else: + kwargs["belief_time"] = server_now().astimezone(pytz.timezone(sensor.timezone)) + bdf = tb.read_csv( file, sensor, - source=source, + source=_source, cumulative_probability=cp, + header=None, + skiprows=skiprows, + nrows=nrows, + usecols=[datecol, valuecol], parse_dates=True, - infer_datetime_format=True, - **( - dict(belief_horizon=timedelta(minutes=horizon)) - if horizon is not None - else dict( - belief_time=server_now().astimezone(pytz.timezone(sensor.timezone)) - ) - ), + **kwargs, ) try: TimedBelief.add( diff --git a/requirements/app.in b/requirements/app.in index 37e6ee956..bd0450c63 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -32,7 +32,7 @@ netCDF4 siphon tables timetomodel>=0.6.8 -timely-beliefs>=1.4.0 +timely-beliefs>=1.4.3 python-dotenv # a backport, not needed in Python3.8 importlib_metadata diff --git a/requirements/app.txt b/requirements/app.txt index 8827feb91..a941f07ef 100644 --- a/requirements/app.txt +++ b/requirements/app.txt @@ -317,7 +317,7 @@ tables==3.6.1 # via -r requirements/app.in threadpoolctl==2.1.0 # via scikit-learn -timely-beliefs==1.4.0 +timely-beliefs==1.4.3 # via -r requirements/app.in timetomodel==0.6.9 # via -r requirements/app.in From d23bf71931a52d3d76da44b4e87794db0383d0d0 Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Thu, 29 Apr 2021 13:16:04 +0200 Subject: [PATCH 07/24] Sensor view (#99) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new UI view for individual Sensors, initially showing Altair graphs of sensor data but meant to be expanded with crud capabilities. The ability to graph sensor data is implemented as a new Sensor method, which is accessible via the API. Chart specs are part of the data package (#104). API functions relating to Sensors are not part of the official documentation until the new data model is properly integrated (https://github.com/SeitaBV/flexmeasures/projects/3). * Add CLI command to add beliefs from csv * Refactor if-else block * Add API functions to chart sensor data * Move chart specs to dedicated module * Docstring correction * Fix data loading * Rename variables in sync with version 1.4.1 of timely-beliefs, and expand docstrings * Clarify what belief_charts_mapping does * Make mapping chart types to specs a function * Adjust query parameters to renamed tb variables * Separate endpoint for loading chart data and clean up boolean chart options * Upgrade tb dependency * Dedicated endpoint for get_chart * Split SensorView into UI and API parts * Remove the need for jquery in sensors.html * Add FlexMeasures menu (integrate page) and move js util functions * Better Altair js dependency management * Merge class methods (chart_data into search_beliefs) * Missing indentation * Move SensorAPI to API package under dev version * Update API path in frontend * flexmeasures.js should be module when exporting functions * Factor out as_html * Move chart specs from the UI package into the data package (#104) * Move code to data package Co-authored-by: F.N. Claessen * Fix regression Co-authored-by: Nicolas Höning Co-authored-by: F.N. Claessen Co-authored-by: Nicolas Höning --- flexmeasures/api/__init__.py | 2 + flexmeasures/api/common/schemas/times.py | 13 ++ flexmeasures/api/dev/__init__.py | 9 ++ flexmeasures/api/dev/sensors.py | 74 +++++++++ .../api/v2_0/implementations/assets.py | 2 +- flexmeasures/data/models/charts/__init__.py | 26 ++++ .../data/models/charts/belief_charts.py | 27 ++++ flexmeasures/data/models/charts/defaults.py | 42 +++++ flexmeasures/data/models/charts/readme.md | 7 + flexmeasures/data/models/time_series.py | 147 ++++++++++++++---- flexmeasures/ui/__init__.py | 2 + flexmeasures/ui/static/js/flexmeasures.js | 21 +++ flexmeasures/ui/templates/base.html | 2 +- flexmeasures/ui/templates/views/sensors.html | 137 ++++++++++++++++ flexmeasures/ui/utils/view_utils.py | 1 + flexmeasures/ui/views/sensors.py | 60 +++++++ flexmeasures/utils/config_defaults.py | 6 + requirements/app.in | 1 + 18 files changed, 549 insertions(+), 30 deletions(-) create mode 100644 flexmeasures/api/dev/__init__.py create mode 100644 flexmeasures/api/dev/sensors.py create mode 100644 flexmeasures/data/models/charts/__init__.py create mode 100644 flexmeasures/data/models/charts/belief_charts.py create mode 100644 flexmeasures/data/models/charts/defaults.py create mode 100644 flexmeasures/data/models/charts/readme.md create mode 100644 flexmeasures/ui/templates/views/sensors.html create mode 100644 flexmeasures/ui/views/sensors.py diff --git a/flexmeasures/api/__init__.py b/flexmeasures/api/__init__.py index ef838271c..32b008f51 100644 --- a/flexmeasures/api/__init__.py +++ b/flexmeasures/api/__init__.py @@ -105,9 +105,11 @@ def register_at(app: Flask): from flexmeasures.api.v1_2 import register_at as v1_2_register_at from flexmeasures.api.v1_3 import register_at as v1_3_register_at from flexmeasures.api.v2_0 import register_at as v2_0_register_at + from flexmeasures.api.dev import register_at as dev_register_at v1_register_at(app) v1_1_register_at(app) v1_2_register_at(app) v1_3_register_at(app) v2_0_register_at(app) + dev_register_at(app) diff --git a/flexmeasures/api/common/schemas/times.py b/flexmeasures/api/common/schemas/times.py index 8f2108ee8..a9bff0d07 100644 --- a/flexmeasures/api/common/schemas/times.py +++ b/flexmeasures/api/common/schemas/times.py @@ -62,3 +62,16 @@ def ground_from( ) return (pd.Timestamp(start) + offset).to_pydatetime() - start return duration + + +class AwareDateTimeField(fields.AwareDateTime): + """Field that deserializes to a timezone aware datetime + and serializes back to a string.""" + + def _deserialize(self, value: str, attr, obj, **kwargs) -> datetime: + """ + Work-around until this PR lands: + https://github.com/marshmallow-code/marshmallow/pull/1787 + """ + value = value.replace(" ", "+") + return fields.AwareDateTime._deserialize(self, value, attr, obj, **kwargs) diff --git a/flexmeasures/api/dev/__init__.py b/flexmeasures/api/dev/__init__.py new file mode 100644 index 000000000..c175be741 --- /dev/null +++ b/flexmeasures/api/dev/__init__.py @@ -0,0 +1,9 @@ +from flask import Flask + + +def register_at(app: Flask): + """This can be used to register FlaskViews.""" + + from flexmeasures.api.dev.sensors import SensorAPI + + SensorAPI.register(app, route_prefix="/api/dev") diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py new file mode 100644 index 000000000..ca6794159 --- /dev/null +++ b/flexmeasures/api/dev/sensors.py @@ -0,0 +1,74 @@ +import json + +from flask_classful import FlaskView, route +from flask_login import login_required +from flask_security import roles_required +from marshmallow import fields +from webargs.flaskparser import use_kwargs +from werkzeug.exceptions import abort + +from flexmeasures.api.common.schemas.times import AwareDateTimeField +from flexmeasures.data.models.time_series import Sensor + + +class SensorAPI(FlaskView): + """ + This view exposes sensor attributes through API endpoints under development. + These endpoints are not yet part of our official API, but support the FlexMeasures UI. + """ + + route_base = "/sensor" + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + @route("//chart/") + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + "include_data": fields.Boolean(required=False), + "dataset_name": fields.Str(required=False), + }, + location="query", + ) + def get_chart(self, id, **kwargs): + """GET from /sensor//chart""" + sensor = get_sensor_or_abort(id) + return json.dumps(sensor.chart(**kwargs)) + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + @route("//chart_data/") + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + }, + location="query", + ) + def get_chart_data(self, id, **kwargs): + """GET from /sensor//chart_data + + Data for use in charts (in case you have the chart specs already). + """ + sensor = get_sensor_or_abort(id) + return sensor.search_beliefs(as_json=True, **kwargs) + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + def get(self, id: int): + """GET from /sensor/""" + sensor = get_sensor_or_abort(id) + attributes = ["name", "timezone", "timerange"] + return {attr: getattr(sensor, attr) for attr in attributes} + + +def get_sensor_or_abort(id: int) -> Sensor: + sensor = Sensor.query.filter(Sensor.id == id).one_or_none() + if sensor is None: + raise abort(404, f"Sensor {id} not found") + return sensor diff --git a/flexmeasures/api/v2_0/implementations/assets.py b/flexmeasures/api/v2_0/implementations/assets.py index d4fb8f79e..0d05533f0 100644 --- a/flexmeasures/api/v2_0/implementations/assets.py +++ b/flexmeasures/api/v2_0/implementations/assets.py @@ -66,7 +66,7 @@ def load_asset(admins_only: bool = False): should be allowed. @app.route('/asset/') - @check_asset + @load_asset def get_asset(asset): return asset_schema.dump(asset), 200 diff --git a/flexmeasures/data/models/charts/__init__.py b/flexmeasures/data/models/charts/__init__.py new file mode 100644 index 000000000..a25bd3cf3 --- /dev/null +++ b/flexmeasures/data/models/charts/__init__.py @@ -0,0 +1,26 @@ +from inspect import getmembers, isfunction + +from . import belief_charts +from .defaults import apply_chart_defaults + + +def chart_type_to_chart_specs(chart_type: str, **kwargs) -> dict: + """Create chart specs of a given chart type, using FlexMeasures defaults for settings like width and height. + + :param chart_type: Name of a variable defining chart specs or a function returning chart specs. + The chart specs can be a dictionary or an Altair chart specification. + - In case of a dictionary, the creator needs to ensure that the dictionary contains valid specs + - In case of an Altair chart specification, Altair validates for you + :returns: A dictionary containing a vega-lite chart specification + """ + # Create a dictionary mapping chart types to chart specs, and apply defaults to the chart specs, too. + belief_charts_mapping = { + chart_type: apply_chart_defaults(chart_specs) + for chart_type, chart_specs in getmembers(belief_charts) + if isfunction(chart_specs) or isinstance(chart_specs, dict) + } + # Create chart specs + chart_specs_or_fnc = belief_charts_mapping[chart_type] + if isfunction(chart_specs_or_fnc): + return chart_specs_or_fnc(**kwargs) + return chart_specs_or_fnc diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py new file mode 100644 index 000000000..e9c810de0 --- /dev/null +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -0,0 +1,27 @@ +from flexmeasures.data.models.charts.defaults import TIME_TITLE, TIME_TOOLTIP_TITLE + + +def bar_chart(title: str, quantity: str = "unknown quantity", unit: str = "a.u."): + if not unit: + unit = "a.u." + return { + "description": "A simple bar chart.", + "title": title, + "mark": "bar", + "encoding": { + "x": {"field": "event_start", "type": "T", "title": TIME_TITLE}, + "y": { + "field": "event_value", + "type": "quantitative", + "title": quantity + " (" + unit + ")", + }, + "tooltip": [ + {"field": "full_date", "title": TIME_TOOLTIP_TITLE, "type": "nominal"}, + { + "field": "event_value", + "title": quantity + " (" + unit + ")", + "type": "quantitative", + }, + ], + }, + } diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py new file mode 100644 index 000000000..4d979c5a6 --- /dev/null +++ b/flexmeasures/data/models/charts/defaults.py @@ -0,0 +1,42 @@ +from functools import wraps +from typing import Callable, Union + +import altair as alt + + +HEIGHT = 300 +WIDTH = 600 +REDUCED_HEIGHT = REDUCED_WIDTH = 60 +SELECTOR_COLOR = "darkred" +TIME_FORMAT = "%I:%M %p on %A %b %e, %Y" +TIME_TOOLTIP_TITLE = "Time and date" +TIME_TITLE = None +TIME_SELECTION_TOOLTIP = "Click and drag to select a time window" + + +def apply_chart_defaults(fn): + @wraps(fn) + def decorated_chart_specs(*args, **kwargs): + dataset_name = kwargs.pop("dataset_name", None) + if isinstance(fn, Callable): + # function that returns a chart specification + chart_specs: Union[dict, alt.TopLevelMixin] = fn(*args, **kwargs) + else: + # not a function, but a direct chart specification + chart_specs: Union[dict, alt.TopLevelMixin] = fn + if isinstance(chart_specs, alt.TopLevelMixin): + chart_specs = chart_specs.to_dict() + chart_specs.pop("$schema") + if dataset_name: + chart_specs["data"] = {"name": dataset_name} + chart_specs["height"] = HEIGHT + chart_specs["width"] = WIDTH + chart_specs["transform"] = [ + { + "as": "full_date", + "calculate": f"timeFormat(datum.event_start, '{TIME_FORMAT}')", + } + ] + return chart_specs + + return decorated_chart_specs diff --git a/flexmeasures/data/models/charts/readme.md b/flexmeasures/data/models/charts/readme.md new file mode 100644 index 000000000..f918ff80f --- /dev/null +++ b/flexmeasures/data/models/charts/readme.md @@ -0,0 +1,7 @@ +# Developer docs for adding chart specs + +Chart specs can be specified as a dictionary with a vega-lite specification or as an altair chart. +Alternatively, they can be specified as a function that returns a dict (with vega-lite specs) or an altair chart. +This approach is useful if you need to parameterize the specification with kwargs. + +Todo: support a plug-in architecture, see https://packaging.python.org/guides/creating-and-discovering-plugins/ diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 873e7088b..aafdc5542 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -1,5 +1,6 @@ from typing import List, Dict, Optional, Union, Tuple from datetime import datetime as datetime_type, timedelta +import json from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import Query, Session @@ -17,6 +18,9 @@ exclude_source_type_filter, ) from flexmeasures.data.services.time_series import collect_time_series_data +from flexmeasures.data.models.charts import chart_type_to_chart_specs +from flexmeasures.utils.time_utils import server_now +from flexmeasures.utils.flexmeasures_inflection import capitalize class Sensor(db.Model, tb.SensorDBMixin): @@ -29,27 +33,116 @@ def __init__(self, name: str, **kwargs): def search_beliefs( self, - event_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]] = ( - None, - None, - ), - belief_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]] = ( - None, - None, - ), + event_starts_after: Optional[datetime_type] = None, + event_ends_before: Optional[datetime_type] = None, + beliefs_after: Optional[datetime_type] = None, + beliefs_before: Optional[datetime_type] = None, source: Optional[Union[int, List[int], str, List[str]]] = None, + as_json: bool = False, ): """Search all beliefs about events for this sensor. - :param event_time_window: search only events within this time window - :param belief_time_window: search only beliefs within this time window - :param source: search only beliefs by this source (pass its name or id) or list of sources""" - return TimedBelief.search( + :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) + :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) + :param beliefs_after: only return beliefs formed after this datetime (inclusive) + :param beliefs_before: only return beliefs formed before this datetime (inclusive) + :param source: search only beliefs by this source (pass its name or id) or list of sources + :param as_json: return beliefs in JSON format (e.g. for use in charts) rather than as BeliefsDataFrame + """ + bdf = TimedBelief.search( sensor=self, - event_time_window=event_time_window, - belief_time_window=belief_time_window, + event_starts_after=event_starts_after, + event_ends_before=event_ends_before, + beliefs_after=beliefs_after, + beliefs_before=beliefs_before, source=source, ) + if as_json: + df = bdf.reset_index() + df["source"] = df["source"].apply(lambda x: x.name) + return df.to_json(orient="records") + return bdf + + def chart( + self, + chart_type: str = "bar_chart", + event_starts_after: Optional[datetime_type] = None, + event_ends_before: Optional[datetime_type] = None, + beliefs_after: Optional[datetime_type] = None, + beliefs_before: Optional[datetime_type] = None, + source: Optional[Union[int, List[int], str, List[str]]] = None, + include_data: bool = False, + dataset_name: Optional[str] = None, + **kwargs, + ) -> dict: + """Create a chart showing sensor data. + + :param chart_type: currently only "bar_chart" # todo: where can we properly list the available chart types? + :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) + :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) + :param beliefs_after: only return beliefs formed after this datetime (inclusive) + :param beliefs_before: only return beliefs formed before this datetime (inclusive) + :param source: search only beliefs by this source (pass its name or id) or list of sources + :param include_data: if True, include data in the chart, or if False, exclude data + :param dataset_name: optionally name the dataset used in the chart (the default name is sensor_) + """ + + # Set up chart specification + if dataset_name is None: + dataset_name = "sensor_" + str(self.id) + self.sensor_type = ( + self.name + ) # todo remove this placeholder when sensor types are modelled + chart_specs = chart_type_to_chart_specs( + chart_type, + title=capitalize(self.name), + quantity=capitalize(self.sensor_type), + unit=self.unit, + dataset_name=dataset_name, + **kwargs, + ) + + if include_data: + # Set up data + data = self.search_beliefs( + as_json=True, + event_starts_after=event_starts_after, + event_ends_before=event_ends_before, + beliefs_after=beliefs_after, + beliefs_before=beliefs_before, + source=source, + ) + # Combine chart specs and data + chart_specs["datasets"] = {dataset_name: json.loads(data)} + return chart_specs + + @property + def timerange(self) -> Dict[str, datetime_type]: + """Timerange for which sensor data exists. + + :returns: dictionary with start and end, for example: + { + 'start': datetime.datetime(2020, 12, 3, 14, 0, tzinfo=pytz.utc), + 'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc) + } + """ + least_recent_query = ( + TimedBelief.query.filter(TimedBelief.sensor == self) + .order_by(TimedBelief.event_start.asc()) + .limit(1) + ) + most_recent_query = ( + TimedBelief.query.filter(TimedBelief.sensor == self) + .order_by(TimedBelief.event_start.desc()) + .limit(1) + ) + results = least_recent_query.union_all(most_recent_query).all() + if not results: + # return now in case there is no data for the sensor + now = server_now() + return dict(start=now, end=now) + least_recent, most_recent = results + return dict(start=least_recent.event_start, end=most_recent.event_end) def __repr__(self) -> str: return f"" @@ -82,30 +175,28 @@ def __init__( def search( cls, sensor: Sensor, - event_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]] = ( - None, - None, - ), - belief_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]] = ( - None, - None, - ), + event_starts_after: Optional[datetime_type] = None, + event_ends_before: Optional[datetime_type] = None, + beliefs_after: Optional[datetime_type] = None, + beliefs_before: Optional[datetime_type] = None, source: Optional[Union[int, List[int], str, List[str]]] = None, ) -> tb.BeliefsDataFrame: """Search all beliefs about events for a given sensor. :param sensor: search only this sensor - :param event_time_window: search only events within this time window - :param belief_time_window: search only beliefs within this time window + :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) + :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) + :param beliefs_after: only return beliefs formed after this datetime (inclusive) + :param beliefs_before: only return beliefs formed before this datetime (inclusive) :param source: search only beliefs by this source (pass its name or id) or list of sources """ return cls.search_session( session=db.session, sensor=sensor, - event_before=event_time_window[1], - event_not_before=event_time_window[0], - belief_before=belief_time_window[1], - belief_not_before=belief_time_window[0], + event_starts_after=event_starts_after, + event_ends_before=event_ends_before, + beliefs_after=beliefs_after, + beliefs_before=beliefs_before, source=source, ) diff --git a/flexmeasures/ui/__init__.py b/flexmeasures/ui/__init__.py index 7e4d2681c..6348aa578 100644 --- a/flexmeasures/ui/__init__.py +++ b/flexmeasures/ui/__init__.py @@ -33,9 +33,11 @@ def register_at(app: Flask): from flexmeasures.ui.crud.assets import AssetCrudUI from flexmeasures.ui.crud.users import UserCrudUI + from flexmeasures.ui.views.sensors import SensorUI AssetCrudUI.register(app) UserCrudUI.register(app) + SensorUI.register(app) import flexmeasures.ui.views # noqa: F401 this is necessary to load the views diff --git a/flexmeasures/ui/static/js/flexmeasures.js b/flexmeasures/ui/static/js/flexmeasures.js index 8dd0becba..66e00404c 100644 --- a/flexmeasures/ui/static/js/flexmeasures.js +++ b/flexmeasures/ui/static/js/flexmeasures.js @@ -310,3 +310,24 @@ function submit_market() { function submit_sensor_type() { $("#sensor_type-form").attr("action", empty_location).submit(); } + + +// Date range utils +export function subtract(oldDate, nDays) { + var newDate = new Date(oldDate) + newDate.setDate(newDate.getDate() - nDays); + return newDate; +} +export function thisMonth(oldDate) { + var d1 = new Date(oldDate) + d1.setDate(1); + var d2 = new Date(d1.getFullYear(), d1.getMonth() + 1, 0); + return [d1, d2]; +}; +export function lastNMonths(oldDate, nMonths) { + var d0 = new Date(oldDate) + var d1 = new Date(d0.getFullYear(), d0.getMonth() - nMonths + 2, 0); + d1.setDate(1); + var d2 = new Date(d0.getFullYear(), d0.getMonth() + 1, 0); + return [d1, d2]; +}; diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html index 2bfd0be51..328f35830 100644 --- a/flexmeasures/ui/templates/base.html +++ b/flexmeasures/ui/templates/base.html @@ -209,7 +209,7 @@ - + {% if show_datepicker %} {% endif %} diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html new file mode 100644 index 000000000..2800d5f8c --- /dev/null +++ b/flexmeasures/ui/templates/views/sensors.html @@ -0,0 +1,137 @@ +{% extends "base.html" %} + +{% set active_page = "assets" %} + +{% block title %} Assets {% endblock %} + +{% block divs %} + +
+
+
+
+
+ + + + + + + + + + + + + + +{% endblock %} \ No newline at end of file diff --git a/flexmeasures/ui/utils/view_utils.py b/flexmeasures/ui/utils/view_utils.py index 1b50a9caa..50f9674f2 100644 --- a/flexmeasures/ui/utils/view_utils.py +++ b/flexmeasures/ui/utils/view_utils.py @@ -84,6 +84,7 @@ def render_flexmeasures_template(html_filename: str, **variables): variables["user_name"] = ( current_user.is_authenticated and current_user.username or "" ) + variables["js_versions"] = current_app.config.get("FLEXMEASURES_JS_VERSIONS") return render_template(html_filename, **variables) diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py new file mode 100644 index 000000000..c9e4eafd0 --- /dev/null +++ b/flexmeasures/ui/views/sensors.py @@ -0,0 +1,60 @@ +from altair.utils.html import spec_to_html +from flask import current_app +from flask_classful import FlaskView, route +from flask_security import login_required, roles_required +from marshmallow import fields +from webargs.flaskparser import use_kwargs + +from flexmeasures.api.common.schemas.times import AwareDateTimeField +from flexmeasures.api.dev.sensors import SensorAPI +from flexmeasures.ui.utils.view_utils import render_flexmeasures_template + + +class SensorUI(FlaskView): + """ + This view creates several new UI endpoints for viewing sensors. + + todo: consider extending this view for crud purposes + """ + + route_base = "/sensors" + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + @route("//chart/") + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + "dataset_name": fields.Str(required=False), + }, + location="query", + ) + def get_chart(self, id, **kwargs): + """GET from /sensors//chart""" + chart_specs = SensorAPI().get_chart( + id, include_data=True, as_html=True, **kwargs + ) + return spec_to_html( + chart_specs, + "vega-lite", + vega_version=current_app.config.get("FLEXMEASURES_JS_VERSIONS").vega, + vegaembed_version=current_app.config.get( + "FLEXMEASURES_JS_VERSIONS" + ).vegaembed, + vegalite_version=current_app.config.get( + "FLEXMEASURES_JS_VERSIONS" + ).vegalite, + ) + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + def get(self, id: int): + """GET from /sensors/""" + return render_flexmeasures_template( + "views/sensors.html", + sensor_id=id, + msg="", + ) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 995bf9d7f..685736d0f 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -105,6 +105,12 @@ class Config(object): FLEXMEASURES_REDIS_PORT: int = 6379 FLEXMEASURES_REDIS_DB_NR: int = 0 # Redis per default has 16 databases, [0-15] FLEXMEASURES_REDIS_PASSWORD: Optional[str] = None + FLEXMEASURES_JS_VERSIONS: dict = dict( + vega="5", + vegaembed="6.17.0", + vegalite="5.0.0", + # todo: expand with other js versions used in FlexMeasures + ) # names of settings which cannot be None diff --git a/requirements/app.in b/requirements/app.in index bd0450c63..82dd94e72 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -1,4 +1,5 @@ # see ui/utils/plotting_utils: separate_legend() and create_hover_tool() +altair bokeh==1.0.4 colour pscript From 576ad256eeeab4a20d70b493e2a7af857ece28c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 29 Apr 2021 14:37:24 +0200 Subject: [PATCH 08/24] prepare 0.4.0 release --- documentation/api/change_log.rst | 3 +++ documentation/changelog.rst | 11 +++++++---- documentation/cli/change_log.rst | 5 +++++ documentation/concepts/services.rst | 2 ++ requirements/app.txt | 7 ++++--- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 672284232..4bc6712c4 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -3,6 +3,9 @@ API change log =============== +.. note:: The FlexMeasures API follows its own versioning scheme. This also reflects in the URL, allowing developers to upgrade at their own pace. + + v2.0-2 | 2021-04-02 """"""""""""""""" diff --git a/documentation/changelog.rst b/documentation/changelog.rst index ff83f33a6..557e6bb2b 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -3,14 +3,17 @@ FlexMeasures Changelog ********************** -v0.4.0 | April XX, 2021 +v0.4.0 | April 29, 2021 =========================== New features ----------- -* Configure views with ``FLEXMEASURES_LISTED_VIEWS`` [see `PR #91 `_] +* Configure the UI menu with ``FLEXMEASURES_LISTED_VIEWS`` [see `PR #91 `_] * Allow for views and CLI functions to come from plugins [see also `PR #91 `_] +.. note:: Read more on these features on `the FlexMeasures blog `__. + + Bugfixes ----------- * Asset edit form displayed wrong error message. Also enabled the asset edit form to display the invalid user input back to the user [see `PR #93 `_] @@ -19,9 +22,9 @@ Infrastructure / Support ---------------------- * Updated dependencies, including Flask-Security-Too [see `PR #82 `_] * Improved documentation after user feedback [see `PR #97 `_] -* Begin experimental integration with `timely beliefs `_ lib: Sensor data as TimedBeliefs [see `PR #79 `_] +* Begin experimental integration with `timely beliefs `_ lib: Sensor data as TimedBeliefs [see `PR #79 `_ and `PR #99 `_] * Add sensors with CLI command currently meant for developers only [see `PR #83 `_] -* Add data (beliefs about sensor events) with CLI command currently meant for developers only [see `PR #85 `_] +* Add data (beliefs about sensor events) with CLI command currently meant for developers only [see `PR #85 `_ and `PR #103 `_] v0.3.1 | April 9, 2021 diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index df31de54e..28131d801 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -4,6 +4,11 @@ FlexMeasures CLI Changelog ********************** +since v0.4.0 | April 2, 2021 +===================== + +* Add the ``dev-add`` command group for experimental features around the upcoming data model refactoring. + since v0.3.0 | April 2, 2021 ===================== diff --git a/documentation/concepts/services.rst b/documentation/concepts/services.rst index a3f958791..2c742ec4a 100644 --- a/documentation/concepts/services.rst +++ b/documentation/concepts/services.rst @@ -15,6 +15,8 @@ The FlexMeasures platform continuously reads in meter data from your assets. To * Data gaps & strange outliers (assure data quality) * Idle processes / leaks (minimise waste) +.. todo:: These features are work in progress. Most of our customers already do this by themselves in a straightforward manner. + Forecasting -------------- diff --git a/requirements/app.txt b/requirements/app.txt index a941f07ef..977070295 100644 --- a/requirements/app.txt +++ b/requirements/app.txt @@ -6,8 +6,10 @@ # alembic==1.5.8 # via flask-migrate -altair==3.0.0 - # via timely-beliefs +altair==4.1.0 + # via + # -r requirements/app.in + # timely-beliefs arrow==1.0.3 # via rq-dashboard attrs==20.3.0 @@ -285,7 +287,6 @@ siphon==0.9 # via -r requirements/app.in six==1.15.0 # via - # altair # bcrypt # bokeh # cycler From 6dbc2c83af02302981c7a47535fd56e02c47b23e Mon Sep 17 00:00:00 2001 From: "create-issue-branch[bot]" <53036503+create-issue-branch[bot]@users.noreply.github.com> Date: Sat, 1 May 2021 15:25:37 +0200 Subject: [PATCH 09/24] get_or_create_source might create data source with type=user but no user set (#111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * make sure we don't create data sources with type=user but no user; complain if no type was passed and no user either Co-authored-by: nhoening Co-authored-by: Nicolas Höning --- flexmeasures/data/models/data_sources.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 3c502745b..66696f37d 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -32,6 +32,8 @@ def __init__( name = user.username type = "user" self.user_id = user.id + elif user is None and type == "user": + raise TypeError("A data source cannot have type 'user' but no user set.") self.type = type tb.BeliefSourceDBMixin.__init__(self, name=name) db.Model.__init__(self, **kwargs) @@ -57,8 +59,10 @@ def __repr__(self): def get_or_create_source( - source: Union[User, str], source_type: str = "user", flush: bool = True + source: Union[User, str], source_type: Optional[str] = None, flush: bool = True ) -> DataSource: + if is_user(source): + source_type = "user" query = DataSource.query.filter(DataSource.type == source_type) if is_user(source): query = query.filter(DataSource.user == source) @@ -72,6 +76,8 @@ def get_or_create_source( if is_user(source): _source = DataSource(user=source) else: + if source_type is None: + raise TypeError("Please specify a source type") _source = DataSource(name=source, type=source_type) db.session.add(_source) if flush: From 6505852a7066020ddf3837c75a70c2fb3fd3ca05 Mon Sep 17 00:00:00 2001 From: "create-issue-branch[bot]" <53036503+create-issue-branch[bot]@users.noreply.github.com> Date: Sat, 1 May 2021 15:55:04 +0200 Subject: [PATCH 10/24] Enforce correct asset units (#108) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Create draft PR for #107 * enforce MW as asset unit for now, as well in CLI * make MW the default on Asset, fix test data * add change log * Typo Co-authored-by: nhoening Co-authored-by: Nicolas Höning Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> --- documentation/changelog.rst | 9 ++++++++- flexmeasures/api/v2_0/tests/utils.py | 2 +- flexmeasures/data/models/assets.py | 5 ++++- flexmeasures/data/scripts/cli_tasks/data_add.py | 14 ++++++++++++-- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 557e6bb2b..72380bb04 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -3,6 +3,14 @@ FlexMeasures Changelog ********************** +v0.5.0 | May XX, 2021 +=========================== + +Infrastructure / Support +---------------------- +* Make assets use MW as their default unit and enforce that in CLI, as well (API already did) [see `PR #108 `_] + + v0.4.0 | April 29, 2021 =========================== @@ -13,7 +21,6 @@ New features .. note:: Read more on these features on `the FlexMeasures blog `__. - Bugfixes ----------- * Asset edit form displayed wrong error message. Also enabled the asset edit form to display the invalid user input back to the user [see `PR #93 `_] diff --git a/flexmeasures/api/v2_0/tests/utils.py b/flexmeasures/api/v2_0/tests/utils.py index b07bd0700..a7d0e311f 100644 --- a/flexmeasures/api/v2_0/tests/utils.py +++ b/flexmeasures/api/v2_0/tests/utils.py @@ -18,7 +18,7 @@ def get_asset_post_data() -> dict: post_data = { "name": "Test battery 2", - "unit": "kW", + "unit": "MW", "capacity_in_mw": 3, "event_resolution": timedelta(minutes=10).seconds / 60, "latitude": 30.1, diff --git a/flexmeasures/data/models/assets.py b/flexmeasures/data/models/assets.py index 68a0a18f3..8fb5f1ac8 100644 --- a/flexmeasures/data/models/assets.py +++ b/flexmeasures/data/models/assets.py @@ -113,9 +113,12 @@ def __init__(self, **kwargs): else: # The UI may initialize Asset objects from API form data with a known id sensor_id = kwargs["id"] - + if "unit" not in kwargs: + kwargs["unit"] = "MW" # current default super(Asset, self).__init__(**kwargs) self.id = sensor_id + if self.unit != "MW": + raise Exception("FlexMeasures only supports MW as unit for now.") self.name = self.name.replace(" (MW)", "") if "display_name" not in kwargs: self.display_name = humanize(self.name) diff --git a/flexmeasures/data/scripts/cli_tasks/data_add.py b/flexmeasures/data/scripts/cli_tasks/data_add.py index 40090296e..aab374aca 100644 --- a/flexmeasures/data/scripts/cli_tasks/data_add.py +++ b/flexmeasures/data/scripts/cli_tasks/data_add.py @@ -108,8 +108,18 @@ def add_sensor(**args): @with_appcontext @click.option("--name", required=True) @click.option("--asset-type-name", required=True) -@click.option("--unit", required=True, help="e.g. MW, kW/h", default="MW") -@click.option("--capacity-in-MW", required=True, type=float) +@click.option( + "--unit", + help="unit of rate, just MW (default) for now", + type=click.Choice(["MW"]), + default="MW", +) # TODO: enable others +@click.option( + "--capacity-in-MW", + required=True, + type=float, + help="Maximum rate of this asset in MW", +) @click.option( "--event-resolution", required=True, From 3dc87dd272e48195a4668da7f684dae57ec9f418 Mon Sep 17 00:00:00 2001 From: "create-issue-branch[bot]" <53036503+create-issue-branch[bot]@users.noreply.github.com> Date: Sat, 1 May 2021 22:45:30 +0200 Subject: [PATCH 11/24] Plugins can override existing routes. (#106) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Create draft PR for #100 * register plugins before UI, so that plugin routes have precedence * allow to overwrite the teaser on the login page & document how that works * add changelog entry * update the plugin docs to showcase overwriting the '/' route Co-authored-by: nhoening Co-authored-by: Nicolas Höning Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Co-authored-by: Nicolas Höning --- documentation/changelog.rst | 4 +++ documentation/dev/plugins.rst | 28 ++++++++++++++++++- flexmeasures/api/common/utils/api_utils.py | 2 +- flexmeasures/app.py | 13 ++++++--- .../ui/templates/admin/login_user.html | 2 ++ 5 files changed, 43 insertions(+), 6 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 72380bb04..d5a24a9a5 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -6,6 +6,10 @@ FlexMeasures Changelog v0.5.0 | May XX, 2021 =========================== +New features +----------- +* Allow plugins to overwrite UI routes and customise the teaser on the login form [see `PR #106 `_] + Infrastructure / Support ---------------------- * Make assets use MW as their default unit and enforce that in CLI, as well (API already did) [see `PR #108 `_] diff --git a/documentation/dev/plugins.rst b/documentation/dev/plugins.rst index 8fbf02385..ba65e2a9c 100644 --- a/documentation/dev/plugins.rst +++ b/documentation/dev/plugins.rst @@ -49,6 +49,7 @@ All else that is needed for this showcase (not shown here) is ``/ou # Showcase: Adding a view + @our_client_bp.route('/') @our_client_bp.route('/metrics') @login_required def metrics(): @@ -75,6 +76,7 @@ All else that is needed for this showcase (not shown here) is ``/ou print(f"I am a CLI command, part of FlexMeasures: {current_app}") +.. note:: You can overwrite FlexMeasures routes here. In our example above, we set the root route ``/``. FlexMeasures registers plugin routes before its own, so in this case visiting the root URL of your app will display this plugged-in view (the same you'd see at `/metrics`). .. note:: Plugin views can also be added to the FlexMeasures UI menu ― just name them in the config setting :ref:`menu-config`. @@ -93,4 +95,28 @@ This can be done if you put the plugin path on the import path. Do it like this HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) - from my_other_file import my_function \ No newline at end of file + from my_other_file import my_function + + +Customising the login teaser +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +FlexMeasures shows an image carousel next to its login form (see ``ui/templates/admin/login_user.html``). + +You can overwrite this content by adding your own login template and defining the ``teaser`` block yourself, e.g.: + +.. code-block:: html + + {% extends "admin/login_user.html" %} + + {% block teaser %} + +

Welcome to my plugin!

+ + {% endblock %} + +Place this template file in the template folder of your plugin blueprint (see above). Your template must have a different filename than "login_user", so FlexMeasures will find it properly! + +Finally, add this config setting to your FlexMeasures config file (using the template filename you chose, obviously): + + SECURITY_LOGIN_USER_TEMPLATE = "my_user_login.html" diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index dbbcca910..ac729707d 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -360,7 +360,7 @@ def get_generic_asset( def save_to_db( timed_values: List[Union[Power, Price, Weather]], forecasting_jobs: List[Job] ) -> ResponseTuple: - """Put the timed values into the database and create forecasting jobs. + """Put the timed values into the database and enqueue forecasting jobs. Data can only be replaced on servers in play mode. diff --git a/flexmeasures/app.py b/flexmeasures/app.py index c2ef442f2..9918b6d9e 100644 --- a/flexmeasures/app.py +++ b/flexmeasures/app.py @@ -98,17 +98,22 @@ def create(env: Optional[str] = None, path_to_config: Optional[str] = None) -> F register_api_at(app) + # Register plugins + + from flexmeasures.utils.app_utils import register_plugins + + register_plugins(app) + # Register the UI + # If plugins registered routes already (e.g. "/"), + # they have precedence (first registration wins). from flexmeasures.ui import register_at as register_ui_at register_ui_at(app) - from flexmeasures.utils.app_utils import register_plugins - - register_plugins(app) - # Profile endpoints (if needed, e.g. during development) + @app.before_request def before_request(): if app.config.get("FLEXMEASURES_PROFILE_REQUESTS", False): diff --git a/flexmeasures/ui/templates/admin/login_user.html b/flexmeasures/ui/templates/admin/login_user.html index a8e2a74d1..557fc9028 100644 --- a/flexmeasures/ui/templates/admin/login_user.html +++ b/flexmeasures/ui/templates/admin/login_user.html @@ -40,6 +40,7 @@

Interested in a demo?

{% endif %}
+ {% block teaser %}

The FlexMeasures Platform

+ {% endblock teaser %}
From 52a8f417adc0dd25062d0c37fe5a6f1434ae2a7c Mon Sep 17 00:00:00 2001 From: "create-issue-branch[bot]" <53036503+create-issue-branch[bot]@users.noreply.github.com> Date: Sat, 1 May 2021 23:51:17 +0200 Subject: [PATCH 12/24] Bug: logging in after clearing session redirects to clearing session (#112) Prevent user from being logged out when clearing the session. * Create draft PR for #28 * More session keys to avoid clearing * Changelog entry Co-authored-by: Flix6x Co-authored-by: F.N. Claessen Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> --- documentation/changelog.rst | 4 ++++ flexmeasures/ui/utils/view_utils.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index d5a24a9a5..5fef02082 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -10,6 +10,10 @@ New features ----------- * Allow plugins to overwrite UI routes and customise the teaser on the login form [see `PR #106 `_] +Bugfixes +----------- +* Prevent logging out user when clearing the session [see `PR #112 `_] + Infrastructure / Support ---------------------- * Make assets use MW as their default unit and enforce that in CLI, as well (API already did) [see `PR #108 `_] diff --git a/flexmeasures/ui/utils/view_utils.py b/flexmeasures/ui/utils/view_utils.py index 50f9674f2..8e6c83cc3 100644 --- a/flexmeasures/ui/utils/view_utils.py +++ b/flexmeasures/ui/utils/view_utils.py @@ -91,7 +91,9 @@ def render_flexmeasures_template(html_filename: str, **variables): def clear_session(): for skey in [ - k for k in session.keys() if k not in ("_id", "user_id", "csrf_token") + k + for k in session.keys() + if k not in ("_fresh", "_id", "_user_id", "csrf_token", "fs_cc", "fs_paa") ]: current_app.logger.info( "Removing %s:%s from session ... " % (skey, session[skey]) From 760ea2ab7a6143612738c9abe6d961ab9f1bdfd4 Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Thu, 6 May 2021 12:07:23 +0200 Subject: [PATCH 13/24] Add footer blocks (copyright notice and credits) (#123) Allows plugins to customise some elements of the footer. * Add footer blocks (copyright notice and credits) * Changelog entry Co-authored-by: F.N. Claessen --- documentation/changelog.rst | 1 + flexmeasures/ui/templates/base.html | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 5fef02082..31b31b839 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -9,6 +9,7 @@ v0.5.0 | May XX, 2021 New features ----------- * Allow plugins to overwrite UI routes and customise the teaser on the login form [see `PR #106 `_] +* Allow plugins to customise the copyright notice and credits in the UI footer [see `PR #123 `_] Bugfixes ----------- diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html index 328f35830..b0fcc770f 100644 --- a/flexmeasures/ui/templates/base.html +++ b/flexmeasures/ui/templates/base.html @@ -230,11 +230,14 @@