CLI: flexmeasures show beliefs

documentation/changelog.rst

@@ -28,7 +28,6 @@ Bugfixes
 Infrastructure / Support
 ----------------------
 * Plugins can import common FlexMeasures classes (like ``Asset`` and ``Sensor``) from a central place, using ``from flexmeasures import Asset, Sensor`` [see `PR #354 <http://www.github.com/FlexMeasures/flexmeasures/pull/354>`_]
-
 * Adapt CLI command for entering some initial structure (``flexmeasures add structure``) to new datamodel [see `PR #349 <http://www.github.com/FlexMeasures/flexmeasures/pull/349>`_]
 * Align documentation requirements with pip-tools [see `PR #384 <http://www.github.com/FlexMeasures/flexmeasures/pull/384>`_]
 

flexmeasures/cli/data_show.py

@@ -1,18 +1,24 @@
 """CLI Tasks for listing database contents - most useful in development"""
 
 from typing import Optional, List
+from datetime import datetime, timedelta
+
 import click
 from flask import current_app as app
 from flask.cli import with_appcontext
 from tabulate import tabulate
 from humanize import naturaldelta, naturaltime
-import isodate
 import uniplot
 
 from flexmeasures.data.models.user import Account, AccountRole, User, Role
 from flexmeasures.data.models.data_sources import DataSource
 from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
+from flexmeasures.data.schemas.generic_assets import GenericAssetIdField
+from flexmeasures.data.schemas.sensors import SensorIdField
+from flexmeasures.data.schemas.account import AccountIdField
+from flexmeasures.data.schemas.sources import DataSourceIdField
+from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField
 
 
 @click.group("show")
@@ -75,16 +81,11 @@ def list_roles():
 
 @fm_show_data.command("account")
 @with_appcontext
-@click.option("--id", "account_id", type=int, required=True)
-def show_account(account_id):
+@click.option("--id", "account", type=AccountIdField(), required=True)
+def show_account(account):
     """
     Show information about an account, including users and assets.
     """
-    account: Account = Account.query.get(account_id)
-    if not account:
-        click.echo(f"No account with id {account_id} known.")
-        raise click.Abort
-
     click.echo(f"========{len(account.name) * '='}==========")
     click.echo(f"Account {account.name} (ID:{account.id}):")
     click.echo(f"========{len(account.name) * '='}==========\n")
@@ -97,7 +98,7 @@ def show_account(account_id):
         click.echo("Account has no roles.")
     click.echo()
 
-    users = User.query.filter_by(account_id=account_id).order_by(User.username).all()
+    users = User.query.filter_by(account_id=account.id).order_by(User.username).all()
     if not users:
         click.echo("No users in account ...")
     else:
@@ -118,7 +119,7 @@ def show_account(account_id):
 
     click.echo()
     assets = (
-        GenericAsset.query.filter_by(account_id=account_id)
+        GenericAsset.query.filter_by(account_id=account.id)
         .order_by(GenericAsset.name)
         .all()
     )
@@ -153,16 +154,11 @@ def list_asset_types():
 
 @fm_show_data.command("asset")
 @with_appcontext
-@click.option("--id", "asset_id", type=int, required=True)
-def show_generic_asset(asset_id):
+@click.option("--id", "asset", type=GenericAssetIdField(), required=True)
+def show_generic_asset(asset):
     """
     Show asset info and list sensors
     """
-    asset = GenericAsset.query.get(asset_id)
-    if not asset:
-        click.echo(f"No asset with id {asset_id} known.")
-        raise click.Abort
-
     click.echo(f"======{len(asset.name) * '='}==========")
     click.echo(f"Asset {asset.name} (ID:{asset.id}):")
     click.echo(f"======{len(asset.name) * '='}==========\n")
@@ -178,7 +174,7 @@ def show_generic_asset(asset_id):
 
     click.echo()
     sensors = (
-        Sensor.query.filter_by(generic_asset_id=asset_id).order_by(Sensor.name).all()
+        Sensor.query.filter_by(generic_asset_id=asset.id).order_by(Sensor.name).all()
     )
     if not sensors:
         click.echo("No sensors in asset ...")
@@ -225,75 +221,58 @@ def list_data_sources():
 @with_appcontext
 @click.option(
     "--sensor-id",
-    "sensor_ids",
-    type=int,
+    "sensors",
     required=True,
     multiple=True,
+    type=SensorIdField(),
     help="ID of sensor(s). This argument can be given multiple times.",
 )
 @click.option(
     "--from",
-    "start_str",
+    "start",
+    type=AwareDateTimeField(),
     required=True,
     help="Plot starting at this datetime. Follow up with a timezone-aware datetime in ISO 6801 format.",
 )
 @click.option(
     "--duration",
-    "duration_str",
+    "duration",
+    type=DurationField(),
     required=True,
     help="Duration of the plot, after --from. Follow up with a duration in ISO 6801 format, e.g. PT1H (1 hour) or PT45M (45 minutes).",
 )
 @click.option(
     "--belief-time-before",
-    "belief_time_before_str",
+    "belief_time_before",
+    type=AwareDateTimeField(),
     required=False,
     help="Time at which beliefs had been known. Follow up with a timezone-aware datetime in ISO 6801 format.",
 )
 @click.option(
     "--source-id",
-    "source_id",
+    "source",
     required=False,
-    type=int,
+    type=DataSourceIdField(),
     help="Source of the beliefs (an existing source id).",
 )
 def plot_beliefs(
-    sensor_ids: List[int],
-    start_str: str,
-    duration_str: str,
-    belief_time_before_str: Optional[str],
-    source_id: Optional[int],
+    sensors: List[Sensor],
+    start: datetime,
+    duration: timedelta,
+    belief_time_before: Optional[datetime],
+    source: Optional[DataSource],
 ):
     """
     Show a simple plot of belief data directly in the terminal.
     """
-    # handle required params: sensor, start, duration
-    sensor_names: List[str] = []
-    for sensor_id in sensor_ids:
-        sensor: Sensor = Sensor.query.get(sensor_id)
-        if not sensor:
-            click.echo(f"No sensor with id {sensor_id} known.")
-            raise click.Abort
-        sensor_names.append(sensor.name)
-    start = isodate.parse_datetime(start_str)  # TODO: make sure it has a tz
-    duration = isodate.parse_duration(duration_str)
-    # handle belief time
-    belief_time_before = None
-    if belief_time_before_str:
-        belief_time_before = isodate.parse_datetime(belief_time_before_str)
-    # handle source
-    source: DataSource = None
-    if source_id:
-        source = DataSource.query.get(source_id)
-        if not source:
-            click.echo(f"No source with id {source_id} known.")
-            raise click.Abort
     # query data
     beliefs_by_sensor = TimedBelief.search(
-        sensors=list(sensor_ids),
+        sensors=list(sensors),
         event_starts_after=start,
         event_ends_before=start + duration,
         beliefs_before=belief_time_before,
         source=source,
+        one_deterministic_belief_per_event=True,
         sum_multiple=False,
     )
     # only keep non-empty
@@ -305,32 +284,34 @@ def plot_beliefs(
     if len(beliefs_by_sensor.keys()) == 0:
         click.echo("No data found!")
         raise click.Abort()
-    sensor_names = list(beliefs_by_sensor.keys())
-    first_df = list(beliefs_by_sensor.values())[0]
+    first_df = beliefs_by_sensor[sensors[0].name]
 
     # Build title
-    if len(sensor_ids) == 1:
-        title = f"Beliefs for Sensor '{sensor_names[0]}' (Id {sensor_ids[0]}).\n"
+    if len(sensors) == 1:
+        title = f"Beliefs for Sensor '{sensors[0].name}' (Id {sensors[0].id}).\n"
     else:
-        title = f"Beliefs for Sensor(s) [{','.join(sensor_names)}], (Id(s): [{','.join([str(sid) for sid in sensor_ids])}]).\n"
+        title = f"Beliefs for Sensor(s) [{','.join([s.name for s in sensors])}], (Id(s): [{','.join([str(s.id) for s in sensors])}]).\n"
     title += f"Data spans {naturaldelta(duration)} and starts at {start}."
     if belief_time_before:
         title += f"\nOnly beliefs made before: {belief_time_before}."
     if source:
-        title += f"\nSource: {source.name}"
-    if len(beliefs_by_sensor.values()) == 1:
+        title += f"\nSource: {source.description}"
+    if len(beliefs_by_sensor) == 1:
         title += f"\nThe time resolution (x-axis) is {naturaldelta(first_df.sensor.event_resolution)}."
 
     uniplot.plot(
         [
             beliefs.event_value
-            for beliefs in [beliefs_by_sensor[sn] for sn in sensor_names]
+            for beliefs in [beliefs_by_sensor[sn] for sn in [s.name for s in sensors]]
         ],
         title=title,
         color=True,
         lines=True,
-        y_unit=first_df.sensor.unit if len(beliefs_by_sensor.values()) == 1 else "",
-        legend_labels=sensor_names,
+        y_unit=first_df.sensor.unit
+        if len(beliefs_by_sensor) == 1
+        or all(sensor.unit == first_df.sensor.unit for sensor in sensors)
+        else "",
+        legend_labels=[s.name for s in sensors],
     )
 
 

flexmeasures/conftest.py

@@ -427,15 +427,23 @@ def setup_assets(
 
 @pytest.fixture(scope="module")
 def setup_beliefs(db, setup_markets, setup_sources) -> int:
-    """Make some beliefs."""
+    """
+    Make some beliefs.
+
+    :returns: the number of beliefs set up
+    """
     return create_beliefs(db, setup_markets, setup_sources)
 
 
 @pytest.fixture(scope="function")
 def setup_beliefs_fresh_db(
     fresh_db, setup_markets_fresh_db, setup_sources_fresh_db
 ) -> int:
-    """Make some beliefs."""
+    """
+    Make some beliefs.
+
+    :returns: the number of beliefs set up
+    """
     return create_beliefs(fresh_db, setup_markets_fresh_db, setup_sources_fresh_db)
 
 

flexmeasures/data/schemas/account.py

@@ -0,0 +1,22 @@
+from flask.cli import with_appcontext
+from marshmallow import fields
+
+from flexmeasures.data.models.user import Account
+from flexmeasures.data.schemas.utils import FMValidationError, MarshmallowClickMixin
+
+
+class AccountIdField(fields.Int, MarshmallowClickMixin):
+    """Field that de-serializes to a Sensor and serializes back to an integer."""
+
+    @with_appcontext
+    def _deserialize(self, value, attr, obj, **kwargs) -> Account:
+        """Turn a source id into a DataSource."""
+        account = Account.query.get(value)
+        account.account_roles  # lazy loading now
+        if account is None:
+            raise FMValidationError(f"No account found with id {value}.")
+        return account
+
+    def _serialize(self, account, attr, data, **kwargs):
+        """Turn a DataSource into a source id."""
+        return account.id

flexmeasures/data/schemas/generic_assets.py

@@ -94,16 +94,17 @@ class Meta:
 
 
 class GenericAssetIdField(fields.Int, MarshmallowClickMixin):
-    """Field that deserializes to a GenericAsset and serializes back to an integer."""
+    """Field that de-serializes to a GenericAsset and serializes back to an integer."""
 
     @with_appcontext
     def _deserialize(self, value, attr, obj, **kwargs) -> GenericAsset:
         """Turn a generic asset id into a GenericAsset."""
         generic_asset = GenericAsset.query.get(value)
+        generic_asset.generic_asset_type  # lazy loading now
         if generic_asset is None:
             raise FMValidationError(f"No asset found with id {value}.")
         return generic_asset
 
-    def _serialize(self, value, attr, data, **kwargs):
+    def _serialize(self, asset, attr, data, **kwargs):
         """Turn a GenericAsset into a generic asset id."""
-        return value.id
+        return asset.id

flexmeasures/data/schemas/sensors.py

@@ -53,7 +53,7 @@ class Meta:
 
 
 class SensorIdField(fields.Int, MarshmallowClickMixin):
-    """Field that deserializes to a Sensor and serializes back to an integer."""
+    """Field that de-serializes to a Sensor and serializes back to an integer."""
 
     @with_appcontext
     def _deserialize(self, value, attr, obj, **kwargs) -> Sensor:

flexmeasures/data/schemas/sources.py

@@ -0,0 +1,21 @@
+from flask.cli import with_appcontext
+from marshmallow import fields
+
+from flexmeasures.data.models.data_sources import DataSource
+from flexmeasures.data.schemas.utils import FMValidationError, MarshmallowClickMixin
+
+
+class DataSourceIdField(fields.Int, MarshmallowClickMixin):
+    """Field that de-serializes to a Sensor and serializes back to an integer."""
+
+    @with_appcontext
+    def _deserialize(self, value, attr, obj, **kwargs) -> DataSource:
+        """Turn a source id into a DataSource."""
+        source = DataSource.query.get(value)
+        if source is None:
+            raise FMValidationError(f"No data source found with id {value}.")
+        return source
+
+    def _serialize(self, source, attr, data, **kwargs):
+        """Turn a DataSource into a source id."""
+        return source.id

flexmeasures/utils/config_utils.py

@@ -153,8 +153,9 @@ def read_env_vars(app: Flask):
     - Logging settings
     - plugins (handled in plugin utils)
     """
-    for var in required + ["LOGGING_LEVEL", "DEBUG"]:
+    for var in required + ["LOGGING_LEVEL"]:
         app.config[var] = os.getenv(var, app.config.get(var, None))
+    app.config["DEBUG"] = int(bool(os.getenv("DEBUG", app.config.get("DEBUG", False))))
 
 
 def are_required_settings_complete(app) -> bool: