642 add report cli command

flexmeasures/cli/data_add.py

@@ -2,7 +2,10 @@
 from __future__ import annotations
 
 from datetime import datetime, timedelta
+from typing import Optional, Type
 import json
+from pathlib import Path
+from io import TextIOBase
 
 from marshmallow import validate
 import pandas as pd
@@ -12,6 +15,7 @@
 import click
 import getpass
 from sqlalchemy.exc import IntegrityError
+from sqlalchemy import func
 from timely_beliefs.sensors.func_store.knowledge_horizons import x_days_ago_at_y_oclock
 import timely_beliefs as tb
 import timely_beliefs.utils as tb_utils
@@ -60,6 +64,10 @@
 from flexmeasures.utils import flexmeasures_inflection
 from flexmeasures.utils.time_utils import server_now
 from flexmeasures.utils.unit_utils import convert_units, ur
+from flexmeasures.data.utils import save_to_db
+from flexmeasures.cli.utils import get_timerange_from_flag
+from flexmeasures.data.models.reporting import Reporter
+from timely_beliefs import BeliefsDataFrame
 
 
 @click.group("add")
@@ -1117,6 +1125,242 @@ def add_schedule_for_storage(
             click.secho("New schedule is stored.", **MsgStyle.SUCCESS)
 
 
+@fm_add_data.command("report")
+@with_appcontext
+@click.option(
+    "--reporter",
+    "reporter_class",
+    required=True,
+    type=click.STRING,
+    help="Reporter class registered in flexmeasures.data.models.reporting or in an available flexmeasures plugin .",
+)
+@click.option(
+    "--sensor-id",
+    "sensor",
+    type=SensorIdField(),
+    required=True,
+    help="ID of the report sensor. Needs to exist in advanced.",
+)
+@click.option(
+    "--reporter-config-file",
+    "reporter_config_file",
+    required=True,
+    type=click.File("r"),
+    help="Path to the JSON file with the reporter configuration.",
+)
+@click.option(
+    "--start",
+    "start",
+    type=AwareDateTimeField(format="iso"),
+    required=False,
+    help="Report start time. Either `--last-X` flag can be used instead. Follow up with a timezone-aware datetime in ISO 6801 format.",
+)
+@click.option(
+    "--end",
+    "end",
+    type=AwareDateTimeField(format="iso"),
+    required=False,
+    help="Report end time. Either `--last-X` flag can be used instead. Follow up with a timezone-aware datetime in ISO 6801 format.",
+)
+@click.option(
+    "--resolution",
+    "resolution",
+    type=DurationField(format="iso"),
+    required=False,
+    help="Time resolution of the input time series to employ for the calculations. Follow up with a ISO 8601 duration string",
+)
+@click.option(
+    "--output-file",
+    "output_file",
+    required=False,
+    type=click.Path(),
+    help="Path to the output file of the results of the report."
+    "Use the `.csv` suffix to save the results as Comma Separated Values and `.xlsx` to export them as Excel sheets.",
+)
+@click.option(
+    "--timezone",
+    "timezone",
+    required=False,
+    default="UTC",
+    help="timezone as string, e.g. 'UTC' or 'Europe/Amsterdam' (defaults to FLEXMEASURES_TIMEZONE config setting)",
+)
+@click.option(
+    "--last-hour",
+    "last_hour",
+    is_flag=True,
+    help="Use the last known 1 hour period for the scope of the report.",
+)
+@click.option(
+    "--last-day",
+    "last_day",
+    is_flag=True,
+    help="Use the last known 1 day period for the scope of the report.",
+)
+@click.option(
+    "--last-week",
+    "last_week",
+    is_flag=True,
+    help="Use the last known 7 days period for the scope of the report.",
+)
+@click.option(
+    "--last-month",
+    "last_month",
+    is_flag=True,
+    help="Use the last known 1 month period for the scope of the report.",
+)
+@click.option(
+    "--last-year",
+    "last_year",
+    is_flag=True,
+    help="Use the last known 1 year period for the scope of the report.",
+)
+@click.option(
+    "--dry-run",
+    "dry_run",
+    is_flag=True,
+    help="Add this flag to avoid saving the results to the database.",
+)
+def add_report(
+    reporter_class: str,
+    sensor: Sensor,
+    reporter_config_file: TextIOBase,
+    start: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    resolution: Optional[timedelta] = None,
+    output_file: Optional[Path] = None,
+    dry_run: bool = True,
+    last_hour: bool = False,
+    last_day: bool = False,
+    last_week: bool = False,
+    last_month: bool = False,
+    last_year: bool = False,
+    timezone: str = "UTC",
+):
+    """
+    Create a new report using the Reporter class and save the results
+    to the database or export them as csv or excel file.
+    """
+    check_timezone(timezone)
+
+    tz = pytz.timezone(zone=timezone)
+
+    # check that only 1 flag is provided
+    last_x_flags = [last_hour, last_day, last_week, last_month, last_year]
+    last_x_flag_given = last_x_flags.count(True) == 1
+
+    if ((start is None) or (end is None)) and not last_x_flag_given:
+        click.secho(
+            "Either --start and --end, any of the --last-X flags should be provided."
+            "Trying to use the the latest datapoint of the report sensor as the start time and "
+            "the current time as the end...",
+            **MsgStyle.WARN,
+        )
+
+        last_value_datetime = (
+            db.session.query(func.max(TimedBelief.event_start))
+            .filter(TimedBelief.sensor_id == sensor.id)
+            .one_or_none()
+        )
+
+        # If there's data saved to the reporter sensors, we use:
+        # - The latest date as the start.
+        # - The current time as the end.
+        if last_value_datetime is not None:
+            start = last_value_datetime[0]
+            end = datetime.now(tz=tz)
+        else:
+            click.secho(
+                f"Could not find any data for the report sensor {sensor}.",
+                **MsgStyle.ERROR,
+            )
+            raise click.Abort()
+
+    # if any of the last-X flag is provided
+    if last_x_flag_given:
+        start, end = get_timerange_from_flag(
+            last_hour=last_hour,
+            last_day=last_day,
+            last_week=last_week,
+            last_month=last_month,
+            last_year=last_year,
+        )
+
+    click.echo(f"Report scope:\n\tstart={start}\n\tend={end}")
+
+    click.echo(
+        f"Looking for the Reporter {reporter_class} among all the registered reporters...",
+    )
+
+    # get reporter class
+    ReporterClass = app.reporters.get(reporter_class)
+
+    # check if it exists
+    if ReporterClass is None:
+        click.secho(
+            f"Reporter class `{reporter_class}` not available.",
+            **MsgStyle.ERROR,
+        )
+        raise click.Abort()
+
+    click.secho(f"Reporter {reporter_class} found", **MsgStyle.SUCCESS)
+
+    reporter_config_raw = json.load(reporter_config_file)
+
+    # initialize reporter class with the reporter sensor and reporter config
+    reporter: Type[Reporter] = ReporterClass(
+        sensor=sensor, reporter_config_raw=reporter_config_raw
+    )
+
+    click.echo("Report computation is running....")
+
+    # compute the report
+    result: Type[BeliefsDataFrame] = reporter.compute(
+        start=start, end=end, input_resolution=resolution
+    )
+
+    click.secho("Report computation done.", **MsgStyle.SUCCESS)
+
+    # save the report it's not running in dry mode
+    if not dry_run:
+        click.echo("Storing report to the database...")
+        save_to_db(result)
+        db.session.commit()
+        click.secho(
+            "Success. The report has been saved to the database.",
+            **MsgStyle.SUCCESS,
+        )
+
+    # if an output file path is provided, save the results
+    if output_file:
+        suffix = str(output_file).split(".")[-1] if "." in str(output_file) else ""
+
+        if suffix == "xlsx":  # save to EXCEL
+            result.to_excel(output_file)
+            click.secho(
+                f"Success. The report has been exported as EXCEL to the file `{output_file}`",
+                **MsgStyle.SUCCESS,
+            )
+
+        elif suffix == "csv":  # save to CSV
+            result.to_csv(output_file)
+            click.secho(
+                f"Success. The report has been exported as CSV to the file `{output_file}`",
+                **MsgStyle.SUCCESS,
+            )
+
+        else:  # default output format: CSV.
+            click.secho(
+                f"File suffix not provided. Exporting results as CSV to file {output_file}",
+                **MsgStyle.WARN,
+            )
+            reporter.to_csv(output_file)
+    else:
+        click.secho(
+            "Success.",
+            **MsgStyle.SUCCESS,
+        )
+
+
 @fm_add_data.command("toy-account")
 @with_appcontext
 @click.option(

flexmeasures/cli/tests/conftest.py

@@ -0,0 +1,98 @@
+import pytest
+
+from datetime import datetime, timedelta
+from pytz import utc
+
+from flexmeasures.data.models.data_sources import DataSource
+from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType
+from flexmeasures.data.models.time_series import Sensor, TimedBelief
+
+
+@pytest.fixture(scope="module")
+@pytest.mark.skip_github
+def setup_dummy_data(db, app):
+    """
+    Create an asset with two sensors (1 and 2), and add the same set of 200 beliefs with an hourly resolution to each of them.
+    Return the two sensors and a result sensor (which has no data).
+    """
+
+    dummy_asset_type = GenericAssetType(name="DummyGenericAssetType")
+    report_asset_type = GenericAssetType(name="ReportAssetType")
+
+    db.session.add_all([dummy_asset_type, report_asset_type])
+
+    dummy_asset = GenericAsset(
+        name="DummyGenericAsset", generic_asset_type=dummy_asset_type
+    )
+
+    pandas_report = GenericAsset(
+        name="PandasReport", generic_asset_type=report_asset_type
+    )
+
+    db.session.add_all([dummy_asset, pandas_report])
+
+    sensor1 = Sensor(
+        "sensor 1", generic_asset=dummy_asset, event_resolution=timedelta(hours=1)
+    )
+
+    db.session.add(sensor1)
+    sensor2 = Sensor(
+        "sensor 2", generic_asset=dummy_asset, event_resolution=timedelta(hours=1)
+    )
+    db.session.add(sensor2)
+    report_sensor = Sensor(
+        "report sensor",
+        generic_asset=pandas_report,
+        event_resolution=timedelta(hours=2),
+    )
+    db.session.add(report_sensor)
+
+    # Create 1 DataSources
+    source = DataSource("source1")
+
+    # Create TimedBeliefs
+    beliefs = []
+    for sensor in [sensor1, sensor2]:
+        for t in range(200):
+            beliefs.append(
+                TimedBelief(
+                    event_start=datetime(2023, 4, 10, tzinfo=utc) + timedelta(hours=t),
+                    belief_time=datetime(2023, 4, 9, tzinfo=utc),
+                    event_value=t,
+                    sensor=sensor,
+                    source=source,
+                )
+            )
+
+    db.session.add_all(beliefs)
+    db.session.commit()
+
+    yield sensor1, sensor2, report_sensor
+
+
+@pytest.fixture(scope="module")
+@pytest.mark.skip_github
+def reporter_config_raw(app, db, setup_dummy_data):
+    """
+    This reporter_config defines the operations to add up the
+    values of the sensors 1 and 2 and resamples the result to a
+    two hour resolution.
+    """
+
+    sensor1, sensor2, report_sensor = setup_dummy_data
+
+    reporter_config_raw = dict(
+        tb_query_config=[dict(sensor=sensor1.id), dict(sensor=sensor2.id)],
+        transformations=[
+            dict(
+                df_input="sensor_1",
+                method="add",
+                args=["@sensor_2"],
+                df_output="df_agg",
+            ),
+            dict(method="resample_events", args=["2h"]),
+        ],
+        final_df_output="df_agg",
+    )
+
+    return reporter_config_raw