Group sensors by (generic) asset (#157)

Set up two new tables to be used for our ongoing migration to a new data model, with corresponding classes GenericAsset and GenericAssetType. It serves to let Sensors be grouped by a GenericAsset. * Create draft PR for #155 * Two db migrations introducing the new tables GenericAssetType and GenericAsset. Check out the revision files for details on how to set up user defined table contents while migrating. * Fix tests * Update CLI command for adding a sensor * Resolve type warnings * Add CLI commands to create generic assets and generic asset types, and add corresponding Marshmallow schemas to validate input parameters * allow to generate db schema pics (with --uml) including the new dev models * Fix typos * Fix name * Add hint to setup documentation * Revert to explicit calls to superclasses * Changelog entry * Refactor setup of GenericAssetType * Rename GenericAssetType table column hover_label to description * Add upgrade warning to changelog Co-authored-by: Flix6x <Flix6x@users.noreply.github.com> Co-authored-by: F.N. Claessen <felix@seita.nl> Co-authored-by: Nicolas Höning <nicolas@seita.nl> Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com>
FlexMeasures · Aug 25, 2021 · 0ad268d · 0ad268d
1 parent 43dbe80
commit 0ad268d
Show file tree

Hide file tree

Showing 16 changed files with 726 additions and 33 deletions.
diff --git a/documentation/changelog.rst b/documentation/changelog.rst
@@ -5,6 +5,9 @@ FlexMeasures Changelog
 v0.6.0 | July XX, 2021
 ===========================
 
+.. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``).
+             In case you are using experimental developer features and have previously set up sensors, be sure to check out the upgrade instructions in `PR #157 <https://github.com/SeitaBV/flexmeasures/pull/157>`_.
+
 New features
 -----------
 * Analytics view offers grouping of all assets by location [see `PR #148 <http://www.github.com/SeitaBV/flexmeasures/pull/148>`_]
@@ -19,6 +22,7 @@ Infrastructure / Support
 * Add possibility to send errors to Sentry [see `PR #143 <http://www.github.com/SeitaBV/flexmeasures/pull/143>`_]
 * Add CLI task to monitor if tasks ran successfully and recently enough [see `PR #146 <http://www.github.com/SeitaBV/flexmeasures/pull/146>`_]
 * Document how to use a custom favicon in plugins [see `PR #152 <http://www.github.com/SeitaBV/flexmeasures/pull/152>`_]
+* Continue experimental integration with `timely beliefs <https://github.com/SeitaBV/timely-beliefs>`_ lib: link multiple sensors to a single asset [see `PR #157 <https://github.com/SeitaBV/flexmeasures/pull/157>`_]
 
 
 v0.5.0 | June 7, 2021

diff --git a/documentation/dev/data.rst b/documentation/dev/data.rst
@@ -61,6 +61,10 @@ Find the ``timezone`` setting and set it to 'UTC'.
 
 Then restart the postgres server.
 
+.. code-block:: bash
+
+    service postgresql restart
+
 
 Setup the "flexmeasures" Unix user
 ^^^^^^^^^^^^^

diff --git a/flexmeasures/api/common/schemas/tests/test_sensors.py b/flexmeasures/api/common/schemas/tests/test_sensors.py
@@ -14,7 +14,7 @@
             build_entity_address(dict(sensor_id=1), "sensor"),
             "sensor",
             "fm1",
-            "my daughter's height",
+            "height",
         ),
         (
             build_entity_address(

diff --git a/flexmeasures/api/dev/tests/conftest.py b/flexmeasures/api/dev/tests/conftest.py
@@ -3,6 +3,7 @@
 from flask_security import SQLAlchemySessionUserDatastore
 import pytest
 
+from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset
 from flexmeasures.data.models.time_series import Sensor
 
 
@@ -28,13 +29,25 @@ def setup_api_fresh_test_data(fresh_db, setup_roles_users_fresh_db):
     give_prosumer_the_MDC_role(fresh_db)
 
 
-def add_gas_sensor(the_db, test_supplier):
+def add_gas_sensor(db, test_supplier):
+    incineration_type = GenericAssetType(
+        name="waste incinerator",
+    )
+    db.session.add(incineration_type)
+    db.session.flush()
+    incineration_asset = GenericAsset(
+        name="incineration line",
+        generic_asset_type=incineration_type,
+    )
+    db.session.add(incineration_asset)
+    db.session.flush()
     gas_sensor = Sensor(
         name="some gas sensor",
         unit="m³/h",
         event_resolution=timedelta(minutes=10),
+        generic_asset=incineration_asset,
     )
-    the_db.session.add(gas_sensor)
+    db.session.add(gas_sensor)
     gas_sensor.owner = test_supplier
 
 

diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py
@@ -23,6 +23,7 @@
 from flexmeasures.utils.time_utils import as_server_time
 from flexmeasures.data.services.users import create_user
 from flexmeasures.data.models.assets import AssetType, Asset, Power
+from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset
 from flexmeasures.data.models.data_sources import DataSource
 from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorType
 from flexmeasures.data.models.markets import Market, MarketType, Price
@@ -186,6 +187,27 @@ def setup_asset_types_fresh_db(fresh_db) -> Dict[str, AssetType]:
     return create_test_asset_types(fresh_db)
 
 
+@pytest.fixture(scope="module")
+def setup_generic_asset(db, setup_generic_asset_type) -> Dict[str, AssetType]:
+    """Make some generic assets used throughout."""
+    troposphere = GenericAsset(
+        name="troposphere", generic_asset_type=setup_generic_asset_type["public_good"]
+    )
+    db.session.add(troposphere)
+    return dict(troposphere=troposphere)
+
+
+@pytest.fixture(scope="module")
+def setup_generic_asset_type(db) -> Dict[str, AssetType]:
+    """Make some generic asset types used throughout."""
+
+    public_good = GenericAssetType(
+        name="public good",
+    )
+    db.session.add(public_good)
+    return dict(public_good=public_good)
+
+
 def create_test_asset_types(db) -> Dict[str, AssetType]:
     """Make some asset types used throughout."""
 
@@ -489,11 +511,10 @@ def create_weather_sensors(db: SQLAlchemy):
 
 
 @pytest.fixture(scope="module")
-def add_sensors(db: SQLAlchemy):
+def add_sensors(db: SQLAlchemy, setup_generic_asset):
     """Add some generic sensors."""
     height_sensor = Sensor(
-        name="my daughter's height",
-        unit="m",
+        name="height", unit="m", generic_asset=setup_generic_asset["troposphere"]
     )
     db.session.add(height_sensor)
     return height_sensor

diff --git a/flexmeasures/data/migrations/versions/565e092a6c5e_introduce_the_GenericAssetType_table.py b/flexmeasures/data/migrations/versions/565e092a6c5e_introduce_the_GenericAssetType_table.py
@@ -0,0 +1,169 @@
+"""introduce the GenericAssetType table
+
+Revision ID: 565e092a6c5e
+Revises: 04f0e2d2924a
+Create Date: 2021-07-20 16:16:50.872449
+
+"""
+import json
+
+from alembic import context, op
+from sqlalchemy import orm
+import sqlalchemy as sa
+
+from flexmeasures.data.models.generic_assets import GenericAssetType
+
+# revision identifiers, used by Alembic.
+revision = "565e092a6c5e"
+down_revision = "04f0e2d2924a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    """Add GenericAssetType table
+
+    A GenericAssetType is created for each AssetType, MarketType and WeatherSensorType.
+    Optionally, additional GenericAssetTypes can be created using:
+
+    flexmeasures db upgrade +1 -x '{"name": "waste power plant"}' -x '{"name": "EVSE", "description": "Electric Vehicle Supply Equipment"}'
+
+    The +1 makes sure we only upgrade by 1 revision, as these arguments are only meant to be used by this upgrade function.
+    """
+
+    upgrade_schema()
+    upgrade_data()
+
+
+def downgrade():
+    op.drop_table("generic_asset_type")
+
+
+def upgrade_data():
+    """Data migration adding 1 generic asset type for each user defined generic asset type,
+    plus 1 generic asset type for each AssetType, MarketType and WeatherSensorType.
+    """
+
+    # Get user defined generic asset types
+    generic_asset_types = context.get_x_argument()
+
+    # Declare ORM table views
+    t_asset_types = sa.Table(
+        "asset_type",
+        sa.MetaData(),
+        sa.Column("name", sa.String(80)),
+        sa.Column("display_name", sa.String(80)),
+    )
+    t_market_types = sa.Table(
+        "market_type",
+        sa.MetaData(),
+        sa.Column("name", sa.String(80)),
+        sa.Column("display_name", sa.String(80)),
+    )
+    t_weather_sensor_types = sa.Table(
+        "weather_sensor_type",
+        sa.MetaData(),
+        sa.Column("name", sa.String(80)),
+        sa.Column("display_name", sa.String(80)),
+    )
+
+    # Use SQLAlchemy's connection and transaction to go through the data
+    connection = op.get_bind()
+    session = orm.Session(bind=connection)
+
+    # Select all existing ids that need migrating, while keeping names intact
+    asset_type_results = connection.execute(
+        sa.select(
+            [
+                t_asset_types.c.name,
+                t_asset_types.c.display_name,
+            ]
+        )
+    ).fetchall()
+    market_type_results = connection.execute(
+        sa.select(
+            [
+                t_market_types.c.name,
+                t_market_types.c.display_name,
+            ]
+        )
+    ).fetchall()
+    weather_sensor_type_results = connection.execute(
+        sa.select(
+            [
+                t_weather_sensor_types.c.name,
+                t_weather_sensor_types.c.display_name,
+            ]
+        )
+    ).fetchall()
+
+    # Prepare to build a list of new generic assets
+    new_generic_asset_types = []
+
+    # Construct generic asset type for each user defined generic asset type
+    asset_type_results_dict = {k: v for k, v in asset_type_results}
+    market_type_results_dict = {k: v for k, v in market_type_results}
+    weather_sensor_type_results_dict = {k: v for k, v in weather_sensor_type_results}
+    for i, generic_asset_type in enumerate(generic_asset_types):
+        generic_asset_type_dict = json.loads(generic_asset_type)
+        print(
+            f"Constructing one generic asset type according to: {generic_asset_type_dict}"
+        )
+        if generic_asset_type_dict["name"] in asset_type_results_dict.keys():
+            raise ValueError(
+                f"User defined generic asset type named '{generic_asset_type_dict['name']}' already exists as asset type."
+            )
+        if generic_asset_type_dict["name"] in market_type_results_dict.keys():
+            raise ValueError(
+                f"User defined generic asset type named '{generic_asset_type_dict['name']}' already exists as market type."
+            )
+        if generic_asset_type_dict["name"] in weather_sensor_type_results_dict.keys():
+            raise ValueError(
+                f"User defined generic asset type named '{generic_asset_type_dict['name']}' already exists as weather sensor type."
+            )
+        new_generic_asset_type = GenericAssetType(
+            name=generic_asset_type_dict["name"],
+            description=generic_asset_type_dict.get("description", None),
+        )
+        new_generic_asset_types.append(new_generic_asset_type)
+
+    # Construct generic asset types for each AssetType
+    print(
+        f"Constructing generic asset types for each of the following asset types: {asset_type_results_dict}"
+    )
+    for name, display_name in asset_type_results_dict.items():
+        # Create new GenericAssets with matching names
+        new_generic_asset_type = GenericAssetType(name=name, description=display_name)
+        new_generic_asset_types.append(new_generic_asset_type)
+
+    # Construct generic asset types for each MarketType
+    print(
+        f"Constructing generic asset types for each of the following market types: {market_type_results_dict}"
+    )
+    for name, display_name in market_type_results_dict.items():
+        # Create new GenericAssets with matching names
+        new_generic_asset_type = GenericAssetType(name=name, description=display_name)
+        new_generic_asset_types.append(new_generic_asset_type)
+
+    # Construct generic asset types for each WeatherSensorType
+    print(
+        f"Constructing generic asset types for each of the following weather sensor types: {weather_sensor_type_results_dict}"
+    )
+    for name, display_name in weather_sensor_type_results_dict.items():
+        # Create new GenericAssets with matching names
+        new_generic_asset_type = GenericAssetType(name=name, description=display_name)
+        new_generic_asset_types.append(new_generic_asset_type)
+
+    # Add the new generic asset types
+    session.add_all(new_generic_asset_types)
+    session.commit()
+
+
+def upgrade_schema():
+    op.create_table(
+        "generic_asset_type",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(length=80), nullable=True),
+        sa.Column("description", sa.String(length=80), nullable=True),
+        sa.PrimaryKeyConstraint("id", name=op.f("generic_asset_type_pkey")),
+    )