diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 13c96d0d5..6832fc0e4 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -14,8 +14,10 @@ New features * Add CLI option to specify custom strings that should be interpreted as NaN values when reading in time series data from CSV [see `PR #357 `_] * Add CLI commands ``flexmeasures add sensor``, ``flexmeasures add asset-type``, ``flexmeasures add beliefs`` (which were experimental features before). [see `PR #337 `_] * Add CLI commands for showing data [see `PR #339 `_] + * Add CLI command for attaching annotations to assets: ``flexmeasures add holidays`` adds public holidays [see `PR #343 `_] * Add CLI command for resampling existing sensor data to new resolution [see `PR #360 `_] +* Add CLI command to add a toy account for tutorials and trying things [see `PR #368 `_]. * Support for percent (%) and permille (‰) sensor units [see `PR #359 `_] Bugfixes diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 058f0828e..2be1bd5ac 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -7,22 +7,24 @@ FlexMeasures CLI Changelog since v0.9.0 | January 26, 2022 ===================== -* add CLI comands for showing data ``flexmeasures show accounts``, ``flexmeasures show account``, ``flexmeasures show roles``, ``flexmeasures show asset-types``, ``flexmeasures show asset`` and ``flexmeasures show data-sources``. +* Add CLI commands for showing data ``flexmeasures show accounts``, ``flexmeasures show account``, ``flexmeasures show roles``, ``flexmeasures show asset-types``, ``flexmeasures show asset`` and ``flexmeasures show data-sources``. * Add ``flexmeasures db-ops resample-data`` CLI command to resample sensor data to a different resolution. +* Add ``flexmeasures add toy-account`` for tutorials and trying things. +* Rename ``flexmeasures add structure`` to ``flexmeasures add initial-structure``. -since v0.9.0 | January 26, 2022 +since v0.8.0 | January 26, 2022 ===================== -* add ``flexmeasures add sensor``, ''flexmeasures add asset-type``, ```flexmeasures add beliefs``. These were previously experimental features (under the `dev-add` command group). +* Add ``flexmeasures add sensor``, ''flexmeasures add asset-type``, ```flexmeasures add beliefs``. These were previously experimental features (under the `dev-add` command group). * ``flexmeasures add asset`` now directly creates an asset in the new data model. -* add ``flexmeasures delete sensor``, ``flexmeasures delete nan-beliefs`` and ``flexmeasures delete unchanged-beliefs``. +* Add ``flexmeasures delete sensor``, ``flexmeasures delete nan-beliefs`` and ``flexmeasures delete unchanged-beliefs``. since v0.6.0 | April 2, 2021 ===================== -* add ``flexmeasures add account``, ``flexmeasures delete account``, and the ``--account-id`` param to ``flexmeasures add user``. +* Add ``flexmeasures add account``, ``flexmeasures delete account``, and the ``--account-id`` param to ``flexmeasures add user``. since v0.4.0 | April 2, 2021 @@ -36,4 +38,4 @@ since v0.3.0 | April 2, 2021 * Refactor CLI into the main groups ``add``, ``delete``, ``jobs`` and ``db-ops`` * Add ``flexmeasures add asset``, ``flexmeasures add user`` and ``flexmeasures add weather-sensor`` -* split the ``populate-db`` command into ``flexmeasures add structure`` and ``flexmeasures add forecasts`` +* Split the ``populate-db`` command into ``flexmeasures add structure`` and ``flexmeasures add forecasts`` diff --git a/documentation/cli/commands.rst b/documentation/cli/commands.rst index 90ee5fb7c..3d02fb315 100644 --- a/documentation/cli/commands.rst +++ b/documentation/cli/commands.rst @@ -23,8 +23,7 @@ of which some are referred to in this documentation. -------------- ================================================= ======================================= -``flexmeasures add structure`` Initialize structural data like asset types, - market types and weather sensor types. +``flexmeasures add initial-structure`` Initialize structural data like users, roles and asset types. ``flexmeasures add account-role`` Create a FlexMeasures tenant account role. ``flexmeasures add account`` Create a FlexMeasures tenant account. ``flexmeasures add user`` Create a FlexMeasures user. @@ -35,6 +34,7 @@ of which some are referred to in this documentation. ``flexmeasures add external-weather-forecasts`` Collect weather forecasts from the DarkSky API. ``flexmeasures add beliefs`` Load beliefs from file. ``flexmeasures add forecasts`` Create forecasts. +``flexmeasures add toy-account`` Create a toy account, for tutorials and trying things. ================================================= ======================================= @@ -56,7 +56,7 @@ of which some are referred to in this documentation. ================================================= ======================================= ``flexmeasures delete structure`` Delete all structural (non time-series) data like assets (types), - markets (types) and weather sensors (types) and users. + roles and users. ``flexmeasures delete account-role`` Delete a tenant account role. ``flexmeasures delete account`` Delete a tenant account & also their users (with assets and power measurements). ``flexmeasures delete user`` Delete a user & also their assets and power measurements. diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index d89d4e818..db6454d4e 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -15,6 +15,11 @@ from workalendar.registry import registry as workalendar_registry from flexmeasures.data import db +from flexmeasures.data.scripts.data_gen import ( + add_transmission_zone_asset, + populate_initial_structure, + add_default_asset_types, +) from flexmeasures.data.services.forecasting import create_forecasting_jobs from flexmeasures.data.services.users import create_user from flexmeasures.data.models.user import Account, AccountRole, RolesAccounts @@ -297,13 +302,11 @@ def add_weather_sensor(**args): print(f" You can access it at its entity address {sensor.entity_address}") -@fm_add_data.command("structure") +@fm_add_data.command("initial-structure") @with_appcontext def add_initial_structure(): """Initialize useful structural data.""" - from flexmeasures.data.scripts.data_gen import populate_structure - - populate_structure(db) + populate_initial_structure(db) @fm_add_data.command("beliefs") @@ -798,6 +801,82 @@ def create_forecasts( ) +@fm_add_data.command("toy-account") +@with_appcontext +@click.option( + "--kind", + default="battery", + type=click.Choice(["battery"]), + help="What kind of toy account. Defaults to a battery.", +) +@click.option("--name", type=str, default="Toy Account", help="Name of the account") +def add_toy_account(kind: str, name: str): + """ + Create a toy account, for tutorials and trying things. + """ + asset_types = add_default_asset_types(db=db) + location = (52.374, 4.88969) # Amsterdam + if kind == "battery": + # make an account (if not exist) + account = Account.query.filter(Account.name == name).one_or_none() + if account: + click.echo(f"Account {name} already exists. Aborting ...") + raise click.Abort() + # make an account user (account-admin?) + user = create_user( + email="toy-user@flexmeasures.io", + check_email_deliverability=False, + password="toy-password", + user_roles=["account-admin"], + account_name=name, + ) + # make assets + for asset_type in ("solar", "building", "battery"): + asset = GenericAsset( + name=f"toy-{asset_type}", + generic_asset_type=asset_types[asset_type], + owner=user.account, + latitude=location[0], + longitude=location[1], + ) + db.session.add(asset) + if asset_type == "battery": + asset.attributes = dict( + capacity_in_mw=0.005, min_soc_in_mwh=0.0005, max_soc_in_mwh=0.0045 + ) + # add charging sensor to battery + charging_sensor = Sensor( + name="charging", + generic_asset=asset, + unit="kW", + timezone="Europe/Amsterdam", + event_resolution=timedelta(minutes=15), + ) + db.session.add(charging_sensor) + + # add public day-ahead market (as sensor of transmission zone asset) + nl_zone = add_transmission_zone_asset("NL", db=db) + day_ahead_sensor = Sensor.query.filter( + Sensor.generic_asset == nl_zone, Sensor.name == "Day ahead prices" + ).one_or_none() + if not day_ahead_sensor: + day_ahead_sensor = Sensor( + name="Day ahead prices", + generic_asset=nl_zone, + unit="EUR/MWh", + timezone="Europe/Amsterdam", + event_resolution=timedelta(minutes=60), + ) + db.session.add(day_ahead_sensor) + + db.session.commit() + + click.echo( + f"Toy account {name} with user {user.email} created successfully. You might want to run `flexmeasures show account --id {user.account.id}`" + ) + click.echo(f"The sensor for Day ahead prices is {day_ahead_sensor}.") + + @fm_add_data.command("external-weather-forecasts") @with_appcontext @click.option( diff --git a/flexmeasures/cli/data_delete.py b/flexmeasures/cli/data_delete.py index 759489007..b422b6fb3 100644 --- a/flexmeasures/cli/data_delete.py +++ b/flexmeasures/cli/data_delete.py @@ -144,7 +144,7 @@ def delete_structure(force): TODO: This could in our future data model (currently in development) be replaced by `flexmeasures delete generic-asset-type`, `flexmeasures delete generic-asset` - and `flexmeasures delete sensor`. + and so on. """ if not force: confirm_deletion(structure=True) @@ -298,7 +298,8 @@ def delete_nan_beliefs(sensor_id: Optional[int] = None): @fm_delete_data.command("sensor") @with_appcontext @click.option( - "--sensor-id", + "--id", + "sensor_id", type=int, required=True, help="Delete a single sensor and its (time series) data. Follow up with the sensor's ID.", diff --git a/flexmeasures/cli/data_show.py b/flexmeasures/cli/data_show.py index 6e39b8c95..5c9d98278 100644 --- a/flexmeasures/cli/data_show.py +++ b/flexmeasures/cli/data_show.py @@ -4,6 +4,7 @@ from flask import current_app as app from flask.cli import with_appcontext from tabulate import tabulate +from humanize import naturaldelta, naturaltime from flexmeasures.data.models.user import Account, AccountRole, User, Role from flexmeasures.data.models.data_sources import DataSource @@ -71,7 +72,7 @@ def list_roles(): @fm_show_data.command("account") @with_appcontext -@click.option("--account-id", type=int, required=True) +@click.option("--id", "account_id", type=int, required=True) def show_account(account_id): """ Show information about an account, including users and assets. @@ -103,7 +104,7 @@ def show_account(account_id): user.id, user.username, user.email, - user.last_login_at, + naturaltime(user.last_login_at), "".join([role.name for role in user.roles]), ) for user in users @@ -149,7 +150,7 @@ def list_asset_types(): @fm_show_data.command("asset") @with_appcontext -@click.option("--asset-id", type=int, required=True) +@click.option("--id", "asset_id", type=int, required=True) def show_generic_asset(asset_id): """ Show asset info and list sensors @@ -185,7 +186,7 @@ def show_generic_asset(asset_id): sensor.id, sensor.name, sensor.unit, - sensor.event_resolution, + naturaldelta(sensor.event_resolution), sensor.timezone, "".join([f"{k}:{v}\n" for k, v in sensor.attributes.items()]), ) diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 777e5a3ef..2186a2933 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -41,7 +41,7 @@ def __init__( if user is not None: name = user.username type = "user" - self.user_id = user.id + self.user = user elif user is None and type == "user": raise TypeError("A data source cannot have type 'user' but no user set.") self.type = type diff --git a/flexmeasures/data/scripts/data_gen.py b/flexmeasures/data/scripts/data_gen.py index 72a0281d1..3bf49fe80 100644 --- a/flexmeasures/data/scripts/data_gen.py +++ b/flexmeasures/data/scripts/data_gen.py @@ -1,7 +1,7 @@ """ Populate the database with data we know or read in. """ -from typing import List, Optional +from typing import List, Optional, Dict from pathlib import Path from shutil import rmtree from datetime import datetime, timedelta @@ -10,7 +10,7 @@ from flask import current_app as app from flask_sqlalchemy import SQLAlchemy import click -from sqlalchemy import func +from sqlalchemy import func, and_ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.serializer import loads, dumps from timetomodel.forecasting import make_rolling_forecasts @@ -34,75 +34,105 @@ infl_eng = inflect.engine() -def add_data_sources(db: SQLAlchemy): - db.session.add(DataSource(name="Seita", type="demo script")) - db.session.add(DataSource(name="Seita", type="forecasting script")) - db.session.add(DataSource(name="Seita", type="scheduling script")) +def add_default_data_sources(db: SQLAlchemy): + for source_name, source_type in ( + ("Seita", "demo script"), + ("Seita", "forecasting script"), + ("Seita", "scheduling script"), + ): + source = DataSource.query.filter( + and_(DataSource.name == source_name, DataSource.type == source_type) + ).one_or_none() + if source: + click.echo(f"Source {source_name} ({source_type}) already exists.") + else: + db.session.add(DataSource(name=source_name, type=source_type)) -def add_asset_types(db: SQLAlchemy): +def add_default_asset_types(db: SQLAlchemy) -> Dict[str, GenericAssetType]: """ Add a few useful asset types. """ - db.session.add( - GenericAssetType( - name="solar", - description="solar panel(s)", - ) - ) - db.session.add( - GenericAssetType( - name="wind", - description="wind turbine", - ) - ) - db.session.add( - GenericAssetType( - name="one-way_evse", - description="uni-directional Electric Vehicle Supply Equipment", - ) - ) - db.session.add( - GenericAssetType( - name="two-way_evse", - description="bi-directional Electric Vehicle Supply Equipment", - ) - ) - db.session.add( - GenericAssetType( - name="battery", - description="stationary battery", - ) - ) - db.session.add( - GenericAssetType( - name="building", - description="building", - ) - ) - - -def add_user_roles(db: SQLAlchemy): + types = {} + for type_name, type_description in ( + ("solar", "solar panel(s)"), + ("wind", "wind turbine"), + ("one-way_evse", "uni-directional Electric Vehicle Supply Equipment"), + ("two-way_evse", "bi-directional Electric Vehicle Supply Equipment"), + ("battery", "stationary battery"), + ("building", "building"), + ): + _type = GenericAssetType.query.filter( + GenericAssetType.name == type_name + ).one_or_none() + if _type: + click.echo(f"Asset type {type_name} already exists.") + else: + _type = GenericAssetType(name=type_name, description=type_description) + db.session.add(_type) + types[type_name] = _type + return types + + +def add_default_user_roles(db: SQLAlchemy): """ Add a few useful user roles. """ - db.session.add(Role(name="admin", description="Super user")) - db.session.add(Role(name="admin-reader", description="Can read everything")) + for role_name, role_description in ( + ("admin", "Super user"), + ("admin-reader", "Can read everything"), + ): + role = Role.query.filter(Role.name == role_name).one_or_none() + if role: + click.echo(f"Role {role_name} already exists.") + else: + db.session.add(Role(name=role_name, description=role_description)) -def add_account_roles(db: SQLAlchemy): +def add_default_account_roles(db: SQLAlchemy): """ Add a few useful account roles, inspired by USEF. """ - db.session.add( - AccountRole(name="Prosumer", description="A consumer who might also produce") - ) - db.session.add(AccountRole(name="MDC", description="Metering Data Company")) - db.session.add(AccountRole(name="Supplier", description="Supplier of energy")) - db.session.add( - AccountRole(name="Aggregator", description="Aggregator of energy flexibility") - ) - db.session.add(AccountRole(name="ESCO", description="Energy Service Company")) + for role_name, role_description in ( + ("Prosumer", "A consumer who might also produce"), + ("MDC", "Metering Data Company"), + ("Supplier", "Supplier of energy"), + ("Aggregator", "Aggregator of energy flexibility"), + ("ESCO", "Energy Service Company"), + ): + role = AccountRole.query.filter(AccountRole.name == role_name).one_or_none() + if role: + click.echo(f"Account role {role_name} already exists.") + else: + db.session.add(AccountRole(name=role_name, description=role_description)) + + +def add_transmission_zone_asset(country_code: str, db: SQLAlchemy) -> GenericAsset: + """ + Ensure a GenericAsset exists to model a transmission zone for a country. + """ + transmission_zone_type = GenericAssetType.query.filter( + GenericAssetType.name == "transmission zone" + ).one_or_none() + if not transmission_zone_type: + click.echo("Adding transmission zone type ...") + transmission_zone_type = GenericAssetType( + name="transmission zone", + description="A grid regulated & balanced as a whole, usually a national grid.", + ) + db.session.add(transmission_zone_type) + ga_name = f"{country_code} transmission zone" + transmission_zone = GenericAsset.query.filter( + GenericAsset.name == ga_name + ).one_or_none() + if not transmission_zone: + click.echo(f"Adding {ga_name} ...") + transmission_zone = GenericAsset( + name=ga_name, + generic_asset_type=transmission_zone_type, + account_id=None, # public + ) + return transmission_zone # ------------ Main functions -------------------------------- @@ -110,18 +140,15 @@ def add_account_roles(db: SQLAlchemy): @as_transaction -def populate_structure(db: SQLAlchemy): +def populate_initial_structure(db: SQLAlchemy): """ Add initial structural data for assets, markets, data sources - - TODO: add user roles (they can get created on-the-fly, but we should be - more pro-active) """ click.echo("Populating the database %s with structural data ..." % db.engine) - add_data_sources(db) - add_user_roles(db) - add_account_roles(db) - add_asset_types(db) + add_default_data_sources(db) + add_default_user_roles(db) + add_default_account_roles(db) + add_default_asset_types(db) click.echo("DB now has %d DataSource(s)" % db.session.query(DataSource).count()) click.echo( "DB now has %d AssetType(s)" % db.session.query(GenericAssetType).count() diff --git a/flexmeasures/data/services/asset_grouping.py b/flexmeasures/data/services/asset_grouping.py index 8d418ec3d..6de95c94a 100644 --- a/flexmeasures/data/services/asset_grouping.py +++ b/flexmeasures/data/services/asset_grouping.py @@ -178,7 +178,7 @@ def display_name(self) -> str: def is_eligible_for_comparing_individual_traces(self, max_traces: int = 7) -> bool: """ - Decide whether comparing individual traces for assets in this resource + Decide whether comparing individual traces for assets in this asset group is a useful feature. The number of assets that can be compared is parametrizable with max_traces. Plot colors are reused if max_traces > 7, and run out if max_traces > 105. diff --git a/flexmeasures/data/services/users.py b/flexmeasures/data/services/users.py index aeadf14a1..d4a134dd7 100644 --- a/flexmeasures/data/services/users.py +++ b/flexmeasures/data/services/users.py @@ -143,13 +143,12 @@ def create_user( # noqa: C901 print(f"Creating account {account_name} ...") account = Account(name=account_name) db.session.add(account) - db.session.flush() user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) kwargs.update(password=hash_password(password), email=email, username=username) user = user_datastore.create_user(**kwargs) - user.account_id = account.id + user.account = account # add roles to user (creating new roles if necessary) if user_roles: @@ -168,7 +167,6 @@ def create_user( # noqa: C901 user_datastore.add_role_to_user(user, role) # create data source - db.session.flush() db.session.add(DataSource(user=user)) return user diff --git a/flexmeasures/ui/charts/latest_state.py b/flexmeasures/ui/charts/latest_state.py index 4fd70f039..2292c84fd 100644 --- a/flexmeasures/ui/charts/latest_state.py +++ b/flexmeasures/ui/charts/latest_state.py @@ -60,7 +60,10 @@ def get_latest_power_as_plot(sensor: Sensor, small: bool = False) -> Tuple[str, "Capacity in use": [latest_power_value], "Remaining capacity": [capacity_in_mw - latest_power_value], } - percentage_capacity = latest_power_value / capacity_in_mw + if capacity_in_mw > 0: + percentage_capacity = latest_power_value / capacity_in_mw + else: + percentage_capacity = 0 df = pd.DataFrame(data) p = df.plot_bokeh( kind="bar", diff --git a/flexmeasures/ui/views/new_dashboard.py b/flexmeasures/ui/views/new_dashboard.py index cdf560609..c030cf61e 100644 --- a/flexmeasures/ui/views/new_dashboard.py +++ b/flexmeasures/ui/views/new_dashboard.py @@ -36,12 +36,7 @@ def new_dashboard_view(): map_asset_groups = {} for asset_group_name, asset_group_query in asset_groups.items(): asset_group = AssetGroup(asset_group_name, asset_query=asset_group_query) - if any( - [ - a.location and (a.has_power_sensors or a.has_energy_sensors) - for a in asset_group.assets - ] - ): + if any([a.location for a in asset_group.assets]): map_asset_groups[asset_group_name] = asset_group # Pack CDN resources (from pandas_bokeh/base.py)