Adapt test_add_report to use the new field of the PandasReporter schema

flexmeasures/cli/data_add.py

@@ -11,6 +11,7 @@
 import yaml
 from pathlib import Path
 from io import TextIOBase
+from string import Template
 
 from marshmallow import validate
 import pandas as pd
@@ -58,6 +59,7 @@
 from flexmeasures.data.schemas.times import TimeIntervalSchema
 from flexmeasures.data.schemas.scheduling.storage import EfficiencyField
 from flexmeasures.data.schemas.sensors import SensorSchema
+from flexmeasures.data.schemas.io import Output
 from flexmeasures.data.schemas.units import QuantityField
 from flexmeasures.data.schemas.generic_assets import (
     GenericAssetSchema,
@@ -1305,14 +1307,6 @@ def add_schedule_process(
 
 @fm_add_data.command("report")
 @with_appcontext
-@click.option(
-    "--sensor-id",
-    "sensor",
-    type=SensorIdField(),
-    required=False,
-    help="Sensor used to save the report. Follow up with the sensor's ID. Can be defined in the parameters file, as well"
-    " If needed, use `flexmeasures add sensor` to create a new sensor first.",
-)
 @click.option(
     "--config",
     "config_file",
@@ -1372,17 +1366,20 @@ def add_schedule_process(
 )
 @click.option(
     "--output-file",
-    "output_file",
+    "output_file_pattern",
     required=False,
     type=click.Path(),
-    help="Path to save the report to file. Will override any previous file contents."
-    " Use the `.csv` suffix to save the results as Comma Separated Values and `.xlsx` to export them as Excel sheets.",
+    help="Format of the output file. Use dollar sign ($) to interpolate values among the following ones:"
+    " now (current time), name (name of the output), sensor_id (id of the sensor), column (column of the output)."
+    " Example: 'result_file_$name_$now.csv'. "
+    "Use the `.csv` suffix to save the results as Comma Separated Values and `.xlsx` to export them as Excel sheets.",
 )
 @click.option(
     "--timezone",
     "timezone",
     required=False,
-    help="Timezone as string, e.g. 'UTC' or 'Europe/Amsterdam' (defaults to the timezone of the sensor used to save the report).",
+    help="Timezone as string, e.g. 'UTC' or 'Europe/Amsterdam' (defaults to the timezone of the sensor used to save the report)."
+    "The timezone of the first output sensor (specified in the parameters) is taken as a default.",
 )
 @click.option(
     "--dry-run",
@@ -1410,15 +1407,14 @@ def add_schedule_process(
 )
 def add_report(  # noqa: C901
     reporter_class: str,
-    sensor: Sensor | None = None,
     config_file: TextIOBase | None = None,
     parameters_file: TextIOBase | None = None,
     start: datetime | None = None,
     end: datetime | None = None,
     start_offset: str | None = None,
     end_offset: str | None = None,
     resolution: timedelta | None = None,
-    output_file: Path | None = None,
+    output_file_pattern: Path | None = None,
     dry_run: bool = False,
     edit_config: bool = False,
     edit_parameters: bool = False,
@@ -1446,26 +1442,22 @@ def add_report(  # noqa: C901
     if edit_parameters:
         parameters = launch_editor("/tmp/parameters.yml")
 
-    if sensor is not None:
-        parameters["sensor"] = sensor.id
-
-    # check if sensor is not provided either in the parameters or the CLI
-    # click parameter
-    if parameters.get("sensor") is None:
+    # check if sensor is not provided in the `parameters` description
+    if "output" not in parameters or len(parameters["output"]) == 0:
         click.secho(
-            "Report sensor needs to be defined, either on the `parameters` file or trough the --sensor CLI parameter...",
+            "At least one output sensor needs to be specified in the parameters description.",
             **MsgStyle.ERROR,
         )
         raise click.Abort()
 
-    sensor = Sensor.query.get(parameters.get("sensor"))
+    output = [Output().load(o) for o in parameters["output"]]
 
     # compute now in the timezone local to the output sensor
     if timezone is not None:
         check_timezone(timezone)
 
     now = pytz.timezone(
-        zone=timezone if timezone is not None else sensor.timezone
+        zone=timezone if timezone is not None else output[0]["sensor"].timezone
     ).localize(datetime.now())
 
     # apply offsets, if provided
@@ -1486,9 +1478,11 @@ def add_report(  # noqa: C901
             " Trying to use the latest datapoint of the report sensor as the start time...",
             **MsgStyle.WARN,
         )
+
+        # todo: get the oldest last_value among all the sensors
         last_value_datetime = (
             db.session.query(func.max(TimedBelief.event_start))
-            .filter(TimedBelief.sensor_id == sensor.id)
+            .filter(TimedBelief.sensor_id == output[0]["sensor"].id)
             .one_or_none()
         )
 
@@ -1497,7 +1491,8 @@ def add_report(  # noqa: C901
             start = last_value_datetime[0]
         else:
             click.secho(
-                f"Could not find any data for the report sensor {sensor}.",
+                "Could not find any data for the output sensors provided. Such data is needed to compute"
+                " a sensible default start for the report, so setting a start explicitly would resolve this issue.",
                 **MsgStyle.ERROR,
             )
             raise click.Abort()
@@ -1545,58 +1540,77 @@ def add_report(  # noqa: C901
         parameters["resolution"] = pd.Timedelta(resolution).isoformat()
 
     # compute the report
-    result: BeliefsDataFrame = reporter.compute(parameters=parameters)
+    results: BeliefsDataFrame = reporter.compute(parameters=parameters)
 
-    if not result.empty:
-        click.secho("Report computation done.", **MsgStyle.SUCCESS)
-    else:
-        click.secho(
-            "Report computation done, but the report is empty.", **MsgStyle.WARN
-        )
-
-    # save the report if it's not running in dry mode
-    if not dry_run:
-        click.echo("Saving report to the database...")
-        save_to_db(result.dropna())
-        db.session.commit()
-        click.secho(
-            "Success. The report has been saved to the database.",
-            **MsgStyle.SUCCESS,
-        )
-    else:
-        click.echo(
-            f"Not saving report to the database (because of --dry-run), but this is what I computed:\n{result}"
-        )
-
-    # if an output file path is provided, save the results
-    if output_file:
-        suffix = str(output_file).split(".")[-1] if "." in str(output_file) else ""
-
-        if suffix == "xlsx":  # save to EXCEL
-            result.to_excel(output_file)
+    for result in results:
+        data = result["data"]
+        sensor = result["sensor"]
+        if not data.empty:
             click.secho(
-                f"Success. The report has been exported as EXCEL to the file `{output_file}`",
-                **MsgStyle.SUCCESS,
+                f"Report computation done for sensor `{sensor}`.", **MsgStyle.SUCCESS
+            )
+        else:
+            click.secho(
+                f"Report computation done for sensor `{sensor}`, but the report is empty.",
+                **MsgStyle.WARN,
             )
 
-        elif suffix == "csv":  # save to CSV
-            result.to_csv(output_file)
+        # save the report if it's not running in dry mode
+        if not dry_run:
+            click.echo(f"Saving report for sensor `{sensor}` to the database...")
+            save_to_db(data.dropna())
+            db.session.commit()
             click.secho(
-                f"Success. The report has been exported as CSV to the file `{output_file}`",
+                f"Success. The report for sensor `{sensor}` has been saved to the database.",
                 **MsgStyle.SUCCESS,
             )
+        else:
+            click.echo(
+                f"Not saving report for sensor `{sensor}` to the database  (because of --dry-run), but this is what I computed:\n{data}"
+            )
+
+        # if an output file path is provided, save the data
+        if output_file_pattern:
+            suffix = (
+                str(output_file_pattern).split(".")[-1]
+                if "." in str(output_file_pattern)
+                else ""
+            )
+            template = Template(str(output_file_pattern))
+
+            filename = template.safe_substitute(
+                sensor_id=result["sensor"].id,
+                name=result.get("name", ""),
+                column=result.get("column", ""),
+                reporter_class=reporter_class,
+                now=now.strftime("%Y_%m_%dT%H%M%S"),
+            )
+
+            if suffix == "xlsx":  # save to EXCEL
+                data.to_excel(filename)
+                click.secho(
+                    f"Success. The report for sensor `{sensor}` has been exported as EXCEL to the file `{filename}`",
+                    **MsgStyle.SUCCESS,
+                )
 
-        else:  # default output format: CSV.
+            elif suffix == "csv":  # save to CSV
+                data.to_csv(filename)
+                click.secho(
+                    f"Success. The report for sensor `{sensor}` has been exported as CSV to the file `{filename}`",
+                    **MsgStyle.SUCCESS,
+                )
+
+            else:  # default output format: CSV.
+                click.secho(
+                    f"File suffix not provided. Exporting results for sensor `{sensor}` as CSV to file {filename}",
+                    **MsgStyle.WARN,
+                )
+                data.to_csv(filename)
+        else:
             click.secho(
-                f"File suffix not provided. Exporting results as CSV to file {output_file}",
-                **MsgStyle.WARN,
+                "Success.",
+                **MsgStyle.SUCCESS,
             )
-            result.to_csv(output_file)
-    else:
-        click.secho(
-            "Success.",
-            **MsgStyle.SUCCESS,
-        )
 
 
 def launch_editor(filename: str) -> dict:

flexmeasures/cli/tests/conftest.py

@@ -63,6 +63,13 @@ def setup_dummy_data(db, app, setup_dummy_asset):
     )
     db.session.add(report_sensor)
 
+    report_sensor_2 = Sensor(
+        "report sensor 2",
+        generic_asset=pandas_report,
+        event_resolution=timedelta(hours=2),
+    )
+    db.session.add(report_sensor_2)
+
     # Create 1 DataSources
     source = DataSource("source1")
 
@@ -83,35 +90,7 @@ def setup_dummy_data(db, app, setup_dummy_asset):
     db.session.add_all(beliefs)
     db.session.commit()
 
-    yield sensor1, sensor2, report_sensor
-
-
-@pytest.fixture(scope="module")
-@pytest.mark.skip_github
-def reporter_config(app, db, setup_dummy_data):
-    """
-    This reporter_config defines the operations to add up the
-    values of the sensors 1 and 2 and resamples the result to a
-    two hour resolution.
-    """
-
-    sensor1, sensor2, report_sensor = setup_dummy_data
-
-    reporter_config = dict(
-        required_input=[{"name": "sensor_1"}, {"name": "sensor_2"}],
-        required_output=[{"name": "df_agg"}],
-        transformations=[
-            dict(
-                df_input="sensor_1",
-                method="add",
-                args=["@sensor_2"],
-                df_output="df_agg",
-            ),
-            dict(method="resample_events", args=["2h"]),
-        ],
-    )
-
-    return reporter_config
+    yield sensor1.id, sensor2.id, report_sensor.id, report_sensor_2.id
 
 
 @pytest.mark.skip_github