feat: support YAML in flexmeasures add report command

ci/run_mypy.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 pip install --upgrade 'mypy>=0.902'
-pip install types-pytz types-requests types-Flask types-click types-redis types-tzlocal types-python-dateutil types-setuptools types-tabulate
+pip install types-pytz types-requests types-Flask types-click types-redis types-tzlocal types-python-dateutil types-setuptools types-tabulate types-PyYAML
 # We are checking python files which have type hints, and leave out bigger issues we made issues for
 # * data/scripts: We'll remove legacy code: https://trello.com/c/1wEnHOkK/7-remove-custom-data-scripts
 # * data/models and data/services: https://trello.com/c/rGxZ9h2H/540-makequery-call-signature-is-incoherent

flexmeasures/cli/data_add.py

@@ -7,6 +7,7 @@
 from datetime import datetime, timedelta
 from typing import Type
 import json
+import yaml
 from pathlib import Path
 from io import TextIOBase
 
@@ -1164,16 +1165,23 @@ def add_schedule_for_storage(
     "--sensor-id",
     "sensor",
     type=SensorIdField(),
-    required=True,
-    help="Sensor used to save the report. Follow up with the sensor's ID. "
+    required=False,
+    help="Sensor used to save the report. Follow up with the sensor's ID. Can be defined in the input file, as well"
     " If needed, use `flexmeasures add sensor` to create a new sensor first.",
 )
 @click.option(
-    "--reporter-config",
-    "reporter_config",
-    required=True,
+    "--config",
+    "config_file",
+    required=False,
+    type=click.File("r"),
+    help="Path to the JSON or YAML file with the configuration of the reporter.",
+)
+@click.option(
+    "--input",
+    "input_file",
+    required=False,
     type=click.File("r"),
-    help="Path to the JSON file with the reporter configuration.",
+    help="Path to the JSON or YAML file with the report inputs.",
 )
 @click.option(
     "--reporter",
@@ -1238,27 +1246,71 @@ def add_schedule_for_storage(
     is_flag=True,
     help="Add this flag to avoid saving the results to the database.",
 )
+@click.option(
+    "--edit-config",
+    "edit_config",
+    is_flag=True,
+    help="Add this flag to edit the configuration of the Reporter in your default text editor (e.g. nano).",
+)
+@click.option(
+    "--edit-input",
+    "edit_input",
+    is_flag=True,
+    help="Add this flag to edit the input to the Reporter in your default text editor (e.g. nano).",
+)
 def add_report(  # noqa: C901
     reporter_class: str,
-    sensor: Sensor,
-    reporter_config: TextIOBase,
+    sensor: Sensor | None = None,
+    config_file: TextIOBase | None = None,
+    input_file: TextIOBase | None = None,
     start: datetime | None = None,
     end: datetime | None = None,
     start_offset: str | None = None,
     end_offset: str | None = None,
     resolution: timedelta | None = None,
     output_file: Path | None = None,
     dry_run: bool = False,
+    edit_config: bool = False,
+    edit_input: bool = False,
     timezone: str | None = None,
 ):
     """
     Create a new report using the Reporter class and save the results
     to the database or export them as CSV or Excel file.
     """
 
+    config = dict()
+
+    if config_file:
+        config = yaml.safe_load(config_file)
+
+    if edit_config:
+        config = launch_editor("/tmp/config.yml")
+
+    input = dict()
+
+    if input_file:
+        input = yaml.safe_load(input_file)
+
+    if edit_input:
+        input = launch_editor("/tmp/input.yml")
+
+    if sensor is not None:
+        input["sensor"] = sensor.id
+
+    sensor = Sensor.query.get(input.get("sensor"))
+
+    if input.get("sensor") is None:
+        click.secho(
+            "Report sensor needs to be defined, either on the `input` file or trough the --sensor CLI parameter...",
+            **MsgStyle.ERROR,
+        )
+        raise click.Abort()
+
     # compute now in the timezone local to the output sensor
     if timezone is not None:
         check_timezone(timezone)
+
     now = pytz.timezone(
         zone=timezone if timezone is not None else sensor.timezone
     ).localize(datetime.now())
@@ -1313,7 +1365,9 @@ def add_report(  # noqa: C901
     )
 
     # get reporter class
-    ReporterClass: Type[Reporter] = app.reporters.get(reporter_class)
+    ReporterClass: Type[Reporter] = app.data_generators.get("reporter").get(
+        reporter_class
+    )
 
     # check if it exists
     if ReporterClass is None:
@@ -1325,19 +1379,20 @@ def add_report(  # noqa: C901
 
     click.secho(f"Reporter {reporter_class} found.", **MsgStyle.SUCCESS)
 
-    reporter_config_raw = json.load(reporter_config)
-
     # initialize reporter class with the reporter sensor and reporter config
-    reporter: Reporter = ReporterClass(
-        sensor=sensor, reporter_config_raw=reporter_config_raw
-    )
+    reporter: Reporter = ReporterClass(config=config)
 
     click.echo("Report computation is running...")
 
+    if ("start" not in input) and (start is not None):
+        input["start"] = start.isoformat()
+    if ("end" not in input) and (end is not None):
+        input["end"] = end.isoformat()
+    if ("resolution" not in input) and (resolution is not None):
+        input["resolution"] = pd.Timedelta.isoformat(resolution)
+
     # compute the report
-    result: BeliefsDataFrame = reporter.compute(
-        start=start, end=end, input_resolution=resolution
-    )
+    result: BeliefsDataFrame = reporter.compute(input=input)
 
     if not result.empty:
         click.secho("Report computation done.", **MsgStyle.SUCCESS)
@@ -1391,6 +1446,18 @@ def add_report(  # noqa: C901
         )
 
 
+def launch_editor(filename: str) -> dict:
+    """Launch editor to create/edit a json object"""
+    click.edit("{\n}", filename=filename)
+
+    with open(filename, "r") as f:
+        content = yaml.safe_load(f)
+        if content is None:
+            return dict()
+
+        return content
+
+
 @fm_add_data.command("toy-account")
 @with_appcontext
 @click.option(

flexmeasures/cli/tests/test_data_add.py

@@ -1,5 +1,6 @@
 import pytest
 import json
+import yaml
 import os
 
 
@@ -96,7 +97,7 @@ def test_cli_help(app):
 
 
 @pytest.mark.skip_github
-def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw):
+def test_add_reporter(app, db, setup_dummy_data, reporter_config):
     """
     The reporter aggregates input data from two sensors (both have 200 data points)
     to a two-hour resolution.
@@ -121,22 +122,32 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw):
     runner = app.test_cli_runner()
 
     cli_input_params = {
-        "sensor-id": report_sensor_id,
-        "reporter-config": "reporter_config.json",
+        "config": "reporter_config.yaml",
+        "input": "input.json",
         "reporter": "PandasReporter",
         "start": "2023-04-10T00:00:00 00:00",
         "end": "2023-04-10T10:00:00 00:00",
         "output-file": "test.csv",
     }
 
+    input = dict(
+        input_sensors=dict(
+            sensor_1=dict(sensor=sensor1.id), sensor_2=dict(sensor=sensor2.id)
+        ),
+        sensor=report_sensor_id,
+    )
+
     cli_input = to_flags(cli_input_params)
 
     # run test in an isolated file system
     with runner.isolated_filesystem():
 
         # save reporter_config to a json file
-        with open("reporter_config.json", "w") as f:
-            json.dump(reporter_config_raw, f)
+        with open("reporter_config.yaml", "w") as f:
+            yaml.dump(reporter_config, f)
+
+        with open("input.json", "w") as f:
+            json.dump(input, f)
 
         # call command
         result = runner.invoke(add_report, cli_input)
@@ -175,8 +186,8 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw):
     previous_command_end = cli_input_params.get("end").replace(" ", "+")
 
     cli_input_params = {
-        "sensor-id": report_sensor_id,
-        "reporter-config": "reporter_config.json",
+        "config": "reporter_config.json",
+        "input": "input.json",
         "reporter": "PandasReporter",
         "output-file": "test.csv",
         "timezone": "UTC",
@@ -188,7 +199,10 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw):
 
         # save reporter_config to a json file
         with open("reporter_config.json", "w") as f:
-            json.dump(reporter_config_raw, f)
+            json.dump(reporter_config, f)
+
+        with open("input.json", "w") as f:
+            json.dump(input, f)
 
         # call command
         result = runner.invoke(add_report, cli_input)

flexmeasures/data/tests/test_data_source.py

@@ -9,7 +9,6 @@
 def test_get_reporter_from_source(
     db, app, aggregator_reporter_data_source, add_nearby_weather_sensors
 ):
-
     reporter = aggregator_reporter_data_source.data_generator
 
     reporter_sensor = add_nearby_weather_sensors.get("farther_temperature")

requirements/app.in

@@ -1,4 +1,5 @@
 # see ui/utils/plotting_utils: separate_legend() and create_hover_tool()
+pyyaml
 altair
 colour
 pscript