diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 8d2dcdcd4..b3b9830e5 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -5,6 +5,9 @@ FlexMeasures Changelog v0.6.0 | July XX, 2021 =========================== +.. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``). + In case you are using experimental developer features and have previously set up sensors, be sure to check out the upgrade instructions in `PR #157 `_. + New features ----------- * Analytics view offers grouping of all assets by location [see `PR #148 `_] @@ -19,6 +22,7 @@ Infrastructure / Support * Add possibility to send errors to Sentry [see `PR #143 `_] * Add CLI task to monitor if tasks ran successfully and recently enough [see `PR #146 `_] * Document how to use a custom favicon in plugins [see `PR #152 `_] +* Continue experimental integration with `timely beliefs `_ lib: link multiple sensors to a single asset [see `PR #157 `_] v0.5.0 | June 7, 2021 diff --git a/documentation/dev/data.rst b/documentation/dev/data.rst index 1f398f03d..dc846b1e8 100644 --- a/documentation/dev/data.rst +++ b/documentation/dev/data.rst @@ -61,6 +61,10 @@ Find the ``timezone`` setting and set it to 'UTC'. Then restart the postgres server. +.. code-block:: bash + + service postgresql restart + Setup the "flexmeasures" Unix user ^^^^^^^^^^^^^ diff --git a/flexmeasures/api/common/schemas/tests/test_sensors.py b/flexmeasures/api/common/schemas/tests/test_sensors.py index 829daf91b..e4f22f82f 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensors.py +++ b/flexmeasures/api/common/schemas/tests/test_sensors.py @@ -14,7 +14,7 @@ build_entity_address(dict(sensor_id=1), "sensor"), "sensor", "fm1", - "my daughter's height", + "height", ), ( build_entity_address( diff --git a/flexmeasures/api/dev/tests/conftest.py b/flexmeasures/api/dev/tests/conftest.py index 1fe88e2b7..00f2d49c5 100644 --- a/flexmeasures/api/dev/tests/conftest.py +++ b/flexmeasures/api/dev/tests/conftest.py @@ -3,6 +3,7 @@ from flask_security import SQLAlchemySessionUserDatastore import pytest +from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset from flexmeasures.data.models.time_series import Sensor @@ -28,13 +29,25 @@ def setup_api_fresh_test_data(fresh_db, setup_roles_users_fresh_db): give_prosumer_the_MDC_role(fresh_db) -def add_gas_sensor(the_db, test_supplier): +def add_gas_sensor(db, test_supplier): + incineration_type = GenericAssetType( + name="waste incinerator", + ) + db.session.add(incineration_type) + db.session.flush() + incineration_asset = GenericAsset( + name="incineration line", + generic_asset_type=incineration_type, + ) + db.session.add(incineration_asset) + db.session.flush() gas_sensor = Sensor( name="some gas sensor", unit="m³/h", event_resolution=timedelta(minutes=10), + generic_asset=incineration_asset, ) - the_db.session.add(gas_sensor) + db.session.add(gas_sensor) gas_sensor.owner = test_supplier diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index c937cc17c..4125f02d1 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -23,6 +23,7 @@ from flexmeasures.utils.time_utils import as_server_time from flexmeasures.data.services.users import create_user from flexmeasures.data.models.assets import AssetType, Asset, Power +from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorType from flexmeasures.data.models.markets import Market, MarketType, Price @@ -186,6 +187,27 @@ def setup_asset_types_fresh_db(fresh_db) -> Dict[str, AssetType]: return create_test_asset_types(fresh_db) +@pytest.fixture(scope="module") +def setup_generic_asset(db, setup_generic_asset_type) -> Dict[str, AssetType]: + """Make some generic assets used throughout.""" + troposphere = GenericAsset( + name="troposphere", generic_asset_type=setup_generic_asset_type["public_good"] + ) + db.session.add(troposphere) + return dict(troposphere=troposphere) + + +@pytest.fixture(scope="module") +def setup_generic_asset_type(db) -> Dict[str, AssetType]: + """Make some generic asset types used throughout.""" + + public_good = GenericAssetType( + name="public good", + ) + db.session.add(public_good) + return dict(public_good=public_good) + + def create_test_asset_types(db) -> Dict[str, AssetType]: """Make some asset types used throughout.""" @@ -489,11 +511,10 @@ def create_weather_sensors(db: SQLAlchemy): @pytest.fixture(scope="module") -def add_sensors(db: SQLAlchemy): +def add_sensors(db: SQLAlchemy, setup_generic_asset): """Add some generic sensors.""" height_sensor = Sensor( - name="my daughter's height", - unit="m", + name="height", unit="m", generic_asset=setup_generic_asset["troposphere"] ) db.session.add(height_sensor) return height_sensor diff --git a/flexmeasures/data/migrations/versions/565e092a6c5e_introduce_the_GenericAssetType_table.py b/flexmeasures/data/migrations/versions/565e092a6c5e_introduce_the_GenericAssetType_table.py new file mode 100644 index 000000000..99450384e --- /dev/null +++ b/flexmeasures/data/migrations/versions/565e092a6c5e_introduce_the_GenericAssetType_table.py @@ -0,0 +1,169 @@ +"""introduce the GenericAssetType table + +Revision ID: 565e092a6c5e +Revises: 04f0e2d2924a +Create Date: 2021-07-20 16:16:50.872449 + +""" +import json + +from alembic import context, op +from sqlalchemy import orm +import sqlalchemy as sa + +from flexmeasures.data.models.generic_assets import GenericAssetType + +# revision identifiers, used by Alembic. +revision = "565e092a6c5e" +down_revision = "04f0e2d2924a" +branch_labels = None +depends_on = None + + +def upgrade(): + """Add GenericAssetType table + + A GenericAssetType is created for each AssetType, MarketType and WeatherSensorType. + Optionally, additional GenericAssetTypes can be created using: + + flexmeasures db upgrade +1 -x '{"name": "waste power plant"}' -x '{"name": "EVSE", "description": "Electric Vehicle Supply Equipment"}' + + The +1 makes sure we only upgrade by 1 revision, as these arguments are only meant to be used by this upgrade function. + """ + + upgrade_schema() + upgrade_data() + + +def downgrade(): + op.drop_table("generic_asset_type") + + +def upgrade_data(): + """Data migration adding 1 generic asset type for each user defined generic asset type, + plus 1 generic asset type for each AssetType, MarketType and WeatherSensorType. + """ + + # Get user defined generic asset types + generic_asset_types = context.get_x_argument() + + # Declare ORM table views + t_asset_types = sa.Table( + "asset_type", + sa.MetaData(), + sa.Column("name", sa.String(80)), + sa.Column("display_name", sa.String(80)), + ) + t_market_types = sa.Table( + "market_type", + sa.MetaData(), + sa.Column("name", sa.String(80)), + sa.Column("display_name", sa.String(80)), + ) + t_weather_sensor_types = sa.Table( + "weather_sensor_type", + sa.MetaData(), + sa.Column("name", sa.String(80)), + sa.Column("display_name", sa.String(80)), + ) + + # Use SQLAlchemy's connection and transaction to go through the data + connection = op.get_bind() + session = orm.Session(bind=connection) + + # Select all existing ids that need migrating, while keeping names intact + asset_type_results = connection.execute( + sa.select( + [ + t_asset_types.c.name, + t_asset_types.c.display_name, + ] + ) + ).fetchall() + market_type_results = connection.execute( + sa.select( + [ + t_market_types.c.name, + t_market_types.c.display_name, + ] + ) + ).fetchall() + weather_sensor_type_results = connection.execute( + sa.select( + [ + t_weather_sensor_types.c.name, + t_weather_sensor_types.c.display_name, + ] + ) + ).fetchall() + + # Prepare to build a list of new generic assets + new_generic_asset_types = [] + + # Construct generic asset type for each user defined generic asset type + asset_type_results_dict = {k: v for k, v in asset_type_results} + market_type_results_dict = {k: v for k, v in market_type_results} + weather_sensor_type_results_dict = {k: v for k, v in weather_sensor_type_results} + for i, generic_asset_type in enumerate(generic_asset_types): + generic_asset_type_dict = json.loads(generic_asset_type) + print( + f"Constructing one generic asset type according to: {generic_asset_type_dict}" + ) + if generic_asset_type_dict["name"] in asset_type_results_dict.keys(): + raise ValueError( + f"User defined generic asset type named '{generic_asset_type_dict['name']}' already exists as asset type." + ) + if generic_asset_type_dict["name"] in market_type_results_dict.keys(): + raise ValueError( + f"User defined generic asset type named '{generic_asset_type_dict['name']}' already exists as market type." + ) + if generic_asset_type_dict["name"] in weather_sensor_type_results_dict.keys(): + raise ValueError( + f"User defined generic asset type named '{generic_asset_type_dict['name']}' already exists as weather sensor type." + ) + new_generic_asset_type = GenericAssetType( + name=generic_asset_type_dict["name"], + description=generic_asset_type_dict.get("description", None), + ) + new_generic_asset_types.append(new_generic_asset_type) + + # Construct generic asset types for each AssetType + print( + f"Constructing generic asset types for each of the following asset types: {asset_type_results_dict}" + ) + for name, display_name in asset_type_results_dict.items(): + # Create new GenericAssets with matching names + new_generic_asset_type = GenericAssetType(name=name, description=display_name) + new_generic_asset_types.append(new_generic_asset_type) + + # Construct generic asset types for each MarketType + print( + f"Constructing generic asset types for each of the following market types: {market_type_results_dict}" + ) + for name, display_name in market_type_results_dict.items(): + # Create new GenericAssets with matching names + new_generic_asset_type = GenericAssetType(name=name, description=display_name) + new_generic_asset_types.append(new_generic_asset_type) + + # Construct generic asset types for each WeatherSensorType + print( + f"Constructing generic asset types for each of the following weather sensor types: {weather_sensor_type_results_dict}" + ) + for name, display_name in weather_sensor_type_results_dict.items(): + # Create new GenericAssets with matching names + new_generic_asset_type = GenericAssetType(name=name, description=display_name) + new_generic_asset_types.append(new_generic_asset_type) + + # Add the new generic asset types + session.add_all(new_generic_asset_types) + session.commit() + + +def upgrade_schema(): + op.create_table( + "generic_asset_type", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=80), nullable=True), + sa.Column("description", sa.String(length=80), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("generic_asset_type_pkey")), + ) diff --git a/flexmeasures/data/migrations/versions/b6d49ed7cceb_introduce_the_GenericAsset_table.py b/flexmeasures/data/migrations/versions/b6d49ed7cceb_introduce_the_GenericAsset_table.py new file mode 100644 index 000000000..a12da2133 --- /dev/null +++ b/flexmeasures/data/migrations/versions/b6d49ed7cceb_introduce_the_GenericAsset_table.py @@ -0,0 +1,196 @@ +"""introduce the GenericAsset table + +Revision ID: b6d49ed7cceb +Revises: 565e092a6c5e +Create Date: 2021-07-20 20:15:28.019102 + +""" +import json + +from alembic import context, op +from sqlalchemy import orm +import sqlalchemy as sa + +from flexmeasures.data.models.assets import Asset +from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset +from flexmeasures.data.models.markets import Market +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.models.weather import WeatherSensor + + +# revision identifiers, used by Alembic. +revision = "b6d49ed7cceb" +down_revision = "565e092a6c5e" +branch_labels = None +depends_on = None + + +def upgrade(): + """Add GenericAsset table and link with Sensor table + + For Sensors with corresponding Assets, Markets or WeatherSensors, a GenericAsset is created with matching name. + For Sensors without, a GenericAsset is created with matching name. + Optionally, sensors that do not correspond to an existing Asset, Market or WeatherSensor can be grouped using + + flexmeasures db upgrade +1 -x '{"asset_type_name": "waste power plant", "sensor_ids": [2, 4], "asset_name": "Afval Energie Centrale", "owner_id": 2}' -x '{"asset_type_name": "EVSE", "sensor_ids": [7, 8], "asset_name": "Laadstation Rijksmuseum - charger 2", "owner_id": 2}' + + The +1 makes sure we only upgrade by 1 revision, as these arguments are only meant to be used by this upgrade function. + """ + + upgrade_schema() + upgrade_data() + op.alter_column("generic_asset", "generic_asset_type_id", nullable=False) + op.alter_column("sensor", "generic_asset_id", nullable=False) + + +def downgrade(): + op.drop_constraint( + op.f("sensor_generic_asset_id_generic_asset_fkey"), "sensor", type_="foreignkey" + ) + op.drop_column("sensor", "generic_asset_id") + op.drop_table("generic_asset") + + +def upgrade_data(): + """Data migration adding 1 generic asset for each user defined group of sensors, + plus 1 generic asset for each remaining sensor (i.e. those not part of a user defined group). + """ + + # Get user defined sensor groups + sensor_groups = context.get_x_argument() + + # Declare ORM table views + t_sensors = sa.Table( + "sensor", + sa.MetaData(), + sa.Column("id", sa.Integer), + sa.Column("name", sa.String(80)), + ) + + # Use SQLAlchemy's connection and transaction to go through the data + connection = op.get_bind() + session = orm.Session(bind=connection) + + # Select all existing ids that need migrating, while keeping names intact + sensor_results = connection.execute( + sa.select( + [ + t_sensors.c.id, + t_sensors.c.name, + ] + ) + ).fetchall() + sensors = session.query(Sensor).all() + + # Prepare to build a list of new generic assets + new_generic_assets = [] + + # Construct generic asset for each user defined sensor group + sensor_results_dict = {k: v for k, v in sensor_results} + for i, sensor_group in enumerate(sensor_groups): + sensor_group_dict = json.loads(sensor_group) + print(f"Constructing one generic asset according to: {sensor_group_dict}") + if not set(sensor_group_dict["sensor_ids"]).issubset( + set(sensor_results_dict.keys()) + ): + raise ValueError( + f"At least some of these sensor ids {sensor_group_dict['sensor_ids']} do not exist." + ) + generic_asset_type = ( + session.query(GenericAssetType) + .filter_by(name=sensor_group_dict["asset_type_name"]) + .one_or_none() + ) + if generic_asset_type is None: + raise ValueError( + f"Asset type name '{sensor_group_dict['asset_type_name']}' does not exist." + ) + group_sensors = [ + sensor for sensor in sensors if sensor.id in sensor_group_dict["sensor_ids"] + ] + new_generic_asset = GenericAsset( + name=sensor_group_dict["asset_name"], + generic_asset_type=generic_asset_type, + sensors=group_sensors, + owner_id=sensor_group_dict["owner_id"], + ) + new_generic_assets.append(new_generic_asset) + for id in sensor_group_dict["sensor_ids"]: + sensor_results_dict.pop(id) + + # Construct generic assets for all remaining sensors + print( + f"Constructing generic assets for each of the following sensors: {sensor_results_dict}" + ) + for id_, name in sensor_results_dict.items(): + _sensors = [sensor for sensor in sensors if sensor.id == id_] + + asset = session.query(Asset).filter_by(id=id_).one_or_none() + if asset is not None: + asset_type_name = asset.asset_type_name + else: + market = session.query(Market).filter_by(id=id_).one_or_none() + if market is not None: + asset_type_name = market.market_type_name + else: + weather_sensor = ( + session.query(WeatherSensor).filter_by(id=id_).one_or_none() + ) + if weather_sensor is not None: + asset_type_name = weather_sensor.weather_sensor_type_name + else: + raise ValueError( + f"Cannot find an Asset, Market or WeatherSensor with id {id_}" + ) + + generic_asset_type = ( + session.query(GenericAssetType) + .filter_by(name=asset_type_name) + .one_or_none() + ) + + # Create new GenericAssets with matching names + new_generic_asset = GenericAsset( + name=name, generic_asset_type=generic_asset_type, sensors=_sensors + ) + new_generic_assets.append(new_generic_asset) + + # Add the new generic assets + session.add_all(new_generic_assets) + session.commit() + + +def upgrade_schema(): + op.create_table( + "generic_asset", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=80), nullable=True), + sa.Column("latitude", sa.Float(), nullable=True), + sa.Column("longitude", sa.Float(), nullable=True), + sa.Column( + "generic_asset_type_id", sa.Integer(), nullable=True + ), # we set nullable=False after data migration + sa.Column("owner_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["generic_asset_type_id"], + ["generic_asset_type.id"], + name=op.f("generic_asset_generic_asset_type_id_generic_asset_type_fkey"), + ), + sa.ForeignKeyConstraint( + ["owner_id"], + ["fm_user.id"], + name=op.f("generic_asset_owner_id_fm_user_fkey"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("generic_asset_pkey")), + ) + op.add_column( + "sensor", sa.Column("generic_asset_id", sa.Integer(), nullable=True) + ) # we set nullable=False after data migration + op.create_foreign_key( + op.f("sensor_generic_asset_id_generic_asset_fkey"), + "sensor", + "generic_asset", + ["generic_asset_id"], + ["id"], + ) diff --git a/flexmeasures/data/models/assets.py b/flexmeasures/data/models/assets.py index 7dd30712e..079f7fb23 100644 --- a/flexmeasures/data/models/assets.py +++ b/flexmeasures/data/models/assets.py @@ -6,6 +6,10 @@ from flexmeasures.data.config import db from flexmeasures.data.models.time_series import Sensor, TimedValue +from flexmeasures.data.models.generic_assets import ( + create_generic_asset, + GenericAssetType, +) from flexmeasures.utils.entity_address_utils import build_entity_address from flexmeasures.utils.flexmeasures_inflection import humanize, pluralize @@ -27,6 +31,10 @@ class AssetType(db.Model): yearly_seasonality = db.Column(db.Boolean(), nullable=False, default=False) def __init__(self, **kwargs): + generic_asset_type = GenericAssetType( + name=kwargs["name"], description=kwargs.get("hover_label", None) + ) + db.session.add(generic_asset_type) super(AssetType, self).__init__(**kwargs) self.name = self.name.replace(" ", "_").lower() if "display_name" not in kwargs: @@ -102,7 +110,8 @@ def __init__(self, **kwargs): # Create a new Sensor with unique id across assets, markets and weather sensors if "id" not in kwargs: - new_sensor = Sensor(name=kwargs["name"]) + new_generic_asset = create_generic_asset("asset", **kwargs) + new_sensor = Sensor(name=kwargs["name"], generic_asset=new_generic_asset) db.session.add(new_sensor) db.session.flush() # generates the pkey for new_sensor sensor_id = new_sensor.id diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py new file mode 100644 index 000000000..40b30dc64 --- /dev/null +++ b/flexmeasures/data/models/generic_assets.py @@ -0,0 +1,86 @@ +from typing import Optional, Tuple + +from flexmeasures.data import db + + +class GenericAssetType(db.Model): + """An asset type defines what type an asset belongs to. + + Examples of asset types: WeatherStation, Market, CP, EVSE, WindTurbine, SolarPanel, Building. + """ + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(80), default="") + description = db.Column(db.String(80), nullable=True, unique=False) + + +class GenericAsset(db.Model): + """An asset is something that has economic value. + + Examples of tangible assets: a house, a ship, a weather station. + Examples of intangible assets: a market, a country, a copyright. + """ + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(80), default="") + latitude = db.Column(db.Float, nullable=True) + longitude = db.Column(db.Float, nullable=True) + + generic_asset_type_id = db.Column( + db.Integer, db.ForeignKey("generic_asset_type.id"), nullable=False + ) + generic_asset_type = db.relationship( + "GenericAssetType", + foreign_keys=[generic_asset_type_id], + backref=db.backref("generic_assets", lazy=True), + ) + + owner_id = db.Column( + db.Integer, db.ForeignKey("fm_user.id", ondelete="CASCADE"), nullable=True + ) # if null, asset is public + owner = db.relationship( + "User", + backref=db.backref( + "generic_assets", + foreign_keys=[owner_id], + lazy=True, + cascade="all, delete-orphan", + passive_deletes=True, + ), + ) + + @property + def location(self) -> Optional[Tuple[float, float]]: + if None not in (self.latitude, self.longitude): + return self.latitude, self.longitude + return None + + +def create_generic_asset(generic_asset_type: str, **kwargs) -> GenericAsset: + """Create a GenericAsset and assigns it an id. + + :param generic_asset_type: "asset", "market" or "weather_sensor" + :param kwargs: should have values for keys "name", and: + - "asset_type_name" or "asset_type" when generic_asset_type is "asset" + - "market_type_name" or "market_type" when generic_asset_type is "market" + - "weather_sensor_type_name" or "weather_sensor_type" when generic_asset_type is "weather_sensor" + - alternatively, "sensor_type" is also fine + :returns: the created GenericAsset + """ + asset_type_name = kwargs.pop(f"{generic_asset_type}_type_name", None) + if asset_type_name is None: + if f"{generic_asset_type}_type" in kwargs: + asset_type_name = kwargs.pop(f"{generic_asset_type}_type").name + else: + asset_type_name = kwargs.pop("sensor_type").name + generic_asset_type = GenericAssetType.query.filter_by( + name=asset_type_name + ).one_or_none() + if generic_asset_type is None: + raise ValueError(f"Cannot find GenericAssetType {asset_type_name} in database.") + new_generic_asset = GenericAsset( + name=kwargs["name"], generic_asset_type_id=generic_asset_type.id + ) + db.session.add(new_generic_asset) + db.session.flush() # generates the pkey for new_generic_asset + return new_generic_asset diff --git a/flexmeasures/data/models/markets.py b/flexmeasures/data/models/markets.py index dd32431c2..5d59e8640 100644 --- a/flexmeasures/data/models/markets.py +++ b/flexmeasures/data/models/markets.py @@ -5,6 +5,10 @@ from sqlalchemy.orm import Query from flexmeasures.data.config import db +from flexmeasures.data.models.generic_assets import ( + create_generic_asset, + GenericAssetType, +) from flexmeasures.data.models.time_series import Sensor, TimedValue from flexmeasures.utils.entity_address_utils import build_entity_address from flexmeasures.utils.flexmeasures_inflection import humanize @@ -23,6 +27,10 @@ class MarketType(db.Model): yearly_seasonality = db.Column(db.Boolean(), nullable=False, default=False) def __init__(self, **kwargs): + generic_asset_type = GenericAssetType( + name=kwargs["name"], description=kwargs.get("hover_label", None) + ) + db.session.add(generic_asset_type) super(MarketType, self).__init__(**kwargs) self.name = self.name.replace(" ", "_").lower() if "display_name" not in kwargs: @@ -65,7 +73,8 @@ def __init__(self, **kwargs): # Create a new Sensor with unique id across assets, markets and weather sensors if "id" not in kwargs: - new_sensor = Sensor(name=kwargs["name"]) + new_generic_asset = create_generic_asset("market", **kwargs) + new_sensor = Sensor(name=kwargs["name"], generic_asset=new_generic_asset) db.session.add(new_sensor) db.session.flush() # generates the pkey for new_sensor new_sensor_id = new_sensor.id diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 5f2a4d601..febc4490c 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -18,6 +18,7 @@ from flexmeasures.data.services.time_series import collect_time_series_data from flexmeasures.utils.entity_address_utils import build_entity_address from flexmeasures.data.models.charts import chart_type_to_chart_specs +from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.utils.time_utils import server_now from flexmeasures.utils.flexmeasures_inflection import capitalize @@ -25,9 +26,19 @@ class Sensor(db.Model, tb.SensorDBMixin): """A sensor measures events. """ - def __init__(self, name: str, **kwargs): + generic_asset_id = db.Column( + db.Integer, db.ForeignKey("generic_asset.id"), nullable=False + ) + generic_asset = db.relationship( + "GenericAsset", + foreign_keys=[generic_asset_id], + backref=db.backref("sensors", lazy=True), + ) + + def __init__(self, name: str, generic_asset: GenericAsset, **kwargs): tb.SensorDBMixin.__init__(self, name, **kwargs) tb_utils.remove_class_init_kwargs(tb.SensorDBMixin, kwargs) + kwargs["generic_asset"] = generic_asset db.Model.__init__(self, **kwargs) @property diff --git a/flexmeasures/data/models/weather.py b/flexmeasures/data/models/weather.py index 42c68b856..3d64a61cf 100644 --- a/flexmeasures/data/models/weather.py +++ b/flexmeasures/data/models/weather.py @@ -9,6 +9,10 @@ from flexmeasures.data.config import db from flexmeasures.data.models.time_series import Sensor, TimedValue +from flexmeasures.data.models.generic_assets import ( + create_generic_asset, + GenericAssetType, +) from flexmeasures.utils.geo_utils import parse_lat_lng from flexmeasures.utils.entity_address_utils import build_entity_address from flexmeasures.utils.flexmeasures_inflection import humanize @@ -27,6 +31,10 @@ class WeatherSensorType(db.Model): yearly_seasonality = True def __init__(self, **kwargs): + generic_asset_type = GenericAssetType( + name=kwargs["name"], description=kwargs.get("hover_label", None) + ) + db.session.add(generic_asset_type) super(WeatherSensorType, self).__init__(**kwargs) self.name = self.name.replace(" ", "_").lower() if "display_name" not in kwargs: @@ -67,7 +75,8 @@ def __init__(self, **kwargs): # Create a new Sensor with unique id across assets, markets and weather sensors if "id" not in kwargs: - new_sensor = Sensor(name=kwargs["name"]) + new_generic_asset = create_generic_asset("weather_sensor", **kwargs) + new_sensor = Sensor(name=kwargs["name"], generic_asset=new_generic_asset) db.session.add(new_sensor) db.session.flush() # generates the pkey for new_sensor new_sensor_id = new_sensor.id diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py new file mode 100644 index 000000000..d246a9a9d --- /dev/null +++ b/flexmeasures/data/schemas/generic_assets.py @@ -0,0 +1,70 @@ +from typing import Optional + +from marshmallow import validates, ValidationError, fields + +from flexmeasures.data import ma +from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType + + +class GenericAssetSchema(ma.SQLAlchemySchema): + """ + GenericAsset schema, with validations. + """ + + id = ma.auto_field() + name = fields.Str() + latitude = ma.auto_field() + longitude = ma.auto_field() + generic_asset_type_id = fields.Integer() + + class Meta: + model = GenericAsset + + @validates("generic_asset_type_id") + def validate_generic_asset_type(self, generic_asset_type_id: int): + generic_asset_type = GenericAssetType.query.get(generic_asset_type_id) + if not generic_asset_type: + raise ValidationError( + f"GenericAssetType with id {generic_asset_type_id} doesn't exist." + ) + + @validates("latitude") + def validate_latitude(self, latitude: Optional[float]): + """Validate optional latitude.""" + if latitude is None: + return + if latitude < -90: + raise ValidationError( + f"Latitude {latitude} exceeds the minimum latitude of -90 degrees." + ) + if latitude > 90: + raise ValidationError( + f"Latitude {latitude} exceeds the maximum latitude of 90 degrees." + ) + + @validates("longitude") + def validate_longitude(self, longitude: Optional[float]): + """Validate optional longitude.""" + if longitude is None: + return + if longitude < -180: + raise ValidationError( + f"Longitude {longitude} exceeds the minimum longitude of -180 degrees." + ) + if longitude > 180: + raise ValidationError( + f"Longitude {longitude} exceeds the maximum longitude of 180 degrees." + ) + + +class GenericAssetTypeSchema(ma.SQLAlchemySchema): + """ + GenericAssetType schema, with validations. + """ + + id = ma.auto_field() + name = fields.Str() + description = ma.auto_field() + + class Meta: + model = GenericAssetType diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 80fb23ecd..2641645fa 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -1,6 +1,7 @@ -from marshmallow import Schema, fields +from marshmallow import Schema, fields, validates, ValidationError from flexmeasures.data import ma +from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.models.time_series import Sensor @@ -29,5 +30,15 @@ class SensorSchema(SensorSchemaMixin, ma.SQLAlchemySchema): Sensor schema, with validations. """ + generic_asset_id = fields.Integer(required=True) + + @validates("generic_asset_id") + def validate_generic_asset(self, generic_asset_id: int): + generic_asset = GenericAsset.query.get(generic_asset_id) + if not generic_asset: + raise ValidationError( + f"Generic asset with id {generic_asset_id} doesn't exist." + ) + class Meta: model = Sensor diff --git a/flexmeasures/data/scripts/cli_tasks/data_add.py b/flexmeasures/data/scripts/cli_tasks/data_add.py old mode 100644 new mode 100755 index 96f0221b6..267b9387d --- a/flexmeasures/data/scripts/cli_tasks/data_add.py +++ b/flexmeasures/data/scripts/cli_tasks/data_add.py @@ -1,7 +1,7 @@ """CLI Tasks for (de)populating the database - most useful in development""" from datetime import timedelta -from typing import List, Optional +from typing import Dict, List, Optional import pandas as pd import pytz @@ -18,8 +18,13 @@ from flexmeasures.data.services.users import create_user from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.schemas.sensors import SensorSchema +from flexmeasures.data.schemas.generic_assets import ( + GenericAssetSchema, + GenericAssetTypeSchema, +) from flexmeasures.data.models.assets import Asset from flexmeasures.data.schemas.assets import AssetSchema +from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType from flexmeasures.data.models.markets import Market from flexmeasures.data.models.weather import WeatherSensor from flexmeasures.data.schemas.weather import WeatherSensorSchema @@ -75,7 +80,7 @@ def new_user(username: str, email: str, roles: List[str], timezone: str): user_roles=roles, check_deliverability=False, ) - app.db.session.commit() + db.session.commit() print(f"Successfully created user {created_user}") @@ -94,17 +99,70 @@ def new_user(username: str, email: str, roles: List[str], timezone: str): required=True, help="timezone as string, e.g. 'UTC' or 'Europe/Amsterdam'", ) +@click.option( + "--generic-asset-id", + required=True, + type=int, + help="Generic asset to assign this sensor to", +) def add_sensor(**args): """Add a sensor.""" check_timezone(args["timezone"]) check_errors(SensorSchema().validate(args)) args["event_resolution"] = timedelta(minutes=args["event_resolution"]) sensor = Sensor(**args) - app.db.session.add(sensor) - app.db.session.commit() + db.session.add(sensor) + db.session.commit() print(f"Successfully created sensor with ID {sensor.id}") - # TODO: uncomment when #66 has landed - # print(f"You can access it at its entity address {sensor.entity_address}") + print(f"You can access it at its entity address {sensor.entity_address}") + + +@fm_dev_add_data.command("generic-asset-type") +@with_appcontext +@click.option("--name", required=True) +@click.option( + "--hover-label", + type=str, + help="Label visible when hovering over the name in the UI.\n" + "Useful to explain acronyms, for example.", +) +def add_generic_asset_type(**args): + """Add a generic asset type.""" + check_errors(GenericAssetTypeSchema().validate(args)) + generic_asset_type = GenericAssetType(**args) + db.session.add(generic_asset_type) + db.session.commit() + print(f"Successfully created generic asset type with ID {generic_asset_type.id}") + print("You can now assign generic assets to it") + + +@fm_dev_add_data.command("generic-asset") +@with_appcontext +@click.option("--name", required=True) +@click.option( + "--latitude", + type=float, + help="Latitude of the asset's location", +) +@click.option( + "--longitude", + type=float, + help="Longitude of the asset's location", +) +@click.option( + "--generic-asset-type-id", + required=True, + type=int, + help="Generic asset type to assign to this asset", +) +def add_generic_asset(**args): + """Add a generic asset.""" + check_errors(GenericAssetSchema().validate(args)) + generic_asset = GenericAsset(**args) + db.session.add(generic_asset) + db.session.commit() + print(f"Successfully created generic asset with ID {generic_asset.id}") + print("You can now assign sensors to it") @fm_add_data.command("asset") @@ -171,8 +229,8 @@ def new_asset(**args): check_errors(AssetSchema().validate(args)) args["event_resolution"] = timedelta(minutes=args["event_resolution"]) asset = Asset(**args) - app.db.session.add(asset) - app.db.session.commit() + db.session.add(asset) + db.session.commit() print(f"Successfully created asset with ID {asset.id}") print(f"You can access it at its entity address {asset.entity_address}") @@ -211,8 +269,8 @@ def add_weather_sensor(**args): check_errors(WeatherSensorSchema().validate(args)) args["event_resolution"] = timedelta(minutes=args["event_resolution"]) sensor = WeatherSensor(**args) - app.db.session.add(sensor) - app.db.session.commit() + db.session.add(sensor) + db.session.commit() print(f"Successfully created weather sensor with ID {sensor.id}") print(f" You can access it at its entity address {sensor.entity_address}") @@ -223,7 +281,7 @@ def add_initial_structure(): """Initialize structural data like asset types, market types and weather sensor types.""" from flexmeasures.data.scripts.data_gen import populate_structure - populate_structure(app.db) + populate_structure(db) @fm_dev_add_data.command("beliefs") @@ -488,7 +546,7 @@ def create_forecasts( from flexmeasures.data.scripts.data_gen import populate_time_series_forecasts populate_time_series_forecasts( - app.db, horizons, from_date, to_date, asset_type, asset_id + db, horizons, from_date, to_date, asset_type, asset_id ) @@ -550,7 +608,7 @@ def check_timezone(timezone): raise click.Abort -def check_errors(errors: list): +def check_errors(errors: Dict[str, List[str]]): if errors: print( f"Please correct the following errors:\n{errors}.\n Use the --help flag to learn more." diff --git a/flexmeasures/data/scripts/visualize_data_model.py b/flexmeasures/data/scripts/visualize_data_model.py index 3341674fa..b7e9eea1b 100755 --- a/flexmeasures/data/scripts/visualize_data_model.py +++ b/flexmeasures/data/scripts/visualize_data_model.py @@ -30,6 +30,17 @@ FALLBACK_VIEWER_CMD = "gwenview" # Use this program if none of the standard viewers # (e.g. display) can be found. Can be overwritten as env var. +RELEVANT_MODULES = [ + "task_runs", + "data_sources", + "markets", + "assets", + "generic_assets", + "weather", + "user", + "time_series", +] + RELEVANT_TABLES = [ "asset", "asset_type", @@ -45,10 +56,15 @@ "weather_sensor", "weather_sensor_type", ] +RELEVANT_TABLES_DEV = [ + "generic_asset_type", + "generic_asset", + "sensor", + "timed_belief", + "timed_value", +] IGNORED_TABLES = ["alembic_version", "roles_users"] -RELEVANT_MODULES = ["task_runs", "data_sources", "markets", "assets", "weather", "user"] - def check_sqlalchemy_schemadisplay_installation(): """Make sure the library which translates the model into a graph structure @@ -101,7 +117,7 @@ def create_schema_pic(pg_url, pg_user, pg_pwd, store: bool = False): print( f"Connecting to database {pg_url} as user {pg_user} and loading schema metadata ..." ) - db_metadata = MetaData(f"postgres://{pg_user}:{pg_pwd}@{pg_url}") + db_metadata = MetaData(f"postgresql://{pg_user}:{pg_pwd}@{pg_url}") kwargs = dict( metadata=db_metadata, show_datatypes=False, # The image would get nasty big if we'd show the datatypes @@ -120,7 +136,7 @@ def create_schema_pic(pg_url, pg_user, pg_pwd, store: bool = False): @uses_dot -def create_uml_pic(store: bool = False): +def create_uml_pic(store: bool = False, dev: bool = False): print("CREATING UML CODE DIAGRAM ...") print("Finding all the relevant mappers in our model...") mappers = [] @@ -138,11 +154,14 @@ def create_uml_pic(store: bool = False): if inspect.isclass(mclass) and issubclass(mclass, flexmeasures_db.Model) } ) + relevant_tables = RELEVANT_TABLES + if dev: + relevant_tables += RELEVANT_TABLES_DEV if DEBUG: - print(f"Relevant tables: {RELEVANT_TABLES}") + print(f"Relevant tables: {relevant_tables}") print(f"Relevant models: {relevant_models}") matched_models = { - m: c for (m, c) in relevant_models.items() if c.__tablename__ in RELEVANT_TABLES + m: c for (m, c) in relevant_models.items() if c.__tablename__ in relevant_tables } for model_name, model_class in matched_models.items(): if DEBUG: @@ -215,9 +234,8 @@ def show_image(graph, fb_viewer_command: str): parser.add_argument( "--store", action="store_true", - help="Store the images a files, instead of showing them directly (which requires pillow).", + help="Store the images as files, instead of showing them directly (which requires pillow).", ) - parser.add_argument( "--pg_url", help="Postgres URL (needed if --schema is on).", @@ -228,6 +246,11 @@ def show_image(graph, fb_viewer_command: str): help="Postgres user (needed if --schema is on).", default="flexmeasures", ) + parser.add_argument( + "--dev", + action="store_false", + help="If true (and --uml is used), include the parts of the new data model which are in development.", + ) args = parser.parse_args() @@ -245,4 +268,4 @@ def show_image(graph, fb_viewer_command: str): f"We need flexmeasures.data to be in the path, so we can read the data model. Error: '{ie}''." ) sys.exit(0) - create_uml_pic(store=args.store) + create_uml_pic(store=args.store, dev=args.dev)