feat: support YAML in flexmeasures add report command

ci/run_mypy.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 pip install --upgrade 'mypy>=0.902'
-pip install types-pytz types-requests types-Flask types-click types-redis types-tzlocal types-python-dateutil types-setuptools types-tabulate
+pip install types-pytz types-requests types-Flask types-click types-redis types-tzlocal types-python-dateutil types-setuptools types-tabulate types-PyYAML
 # We are checking python files which have type hints, and leave out bigger issues we made issues for
 # * data/scripts: We'll remove legacy code: https://trello.com/c/1wEnHOkK/7-remove-custom-data-scripts
 # * data/models and data/services: https://trello.com/c/rGxZ9h2H/540-makequery-call-signature-is-incoherent

flexmeasures/cli/data_add.py

@@ -7,6 +7,7 @@
 from datetime import datetime, timedelta
 from typing import Type
 import json
+import yaml
 from pathlib import Path
 from io import TextIOBase
 
@@ -1169,11 +1170,18 @@ def add_schedule_for_storage(
     " If needed, use `flexmeasures add sensor` to create a new sensor first.",
 )
 @click.option(
-    "--reporter-config",
-    "reporter_config",
-    required=True,
+    "--config",
+    "config_file",
+    required=False,
     type=click.File("r"),
-    help="Path to the JSON file with the reporter configuration.",
+    help="Path to the JSON or YAML file with the configuration of the reporter.",
+)
+@click.option(
+    "--inputs",
+    "inputs_file",
+    required=False,
+    type=click.File("r"),
+    help="Path to the JSON or YAML file with the report inputs.",
 )
 @click.option(
     "--reporter",
@@ -1238,18 +1246,33 @@ def add_schedule_for_storage(
     is_flag=True,
     help="Add this flag to avoid saving the results to the database.",
 )
+@click.option(
+    "--edit-config",
+    "edit_config",
+    is_flag=True,
+    help="Add this flag to edit the configuration of the Reporter in your default text editor (e.g. nano).",
+)
+@click.option(
+    "--edit-inputs",
+    "edit_inputs",
+    is_flag=True,
+    help="Add this flag to edit the inputs to the Reporter in your default text editor (e.g. nano).",
+)
 def add_report(  # noqa: C901
     reporter_class: str,
     sensor: Sensor,
-    reporter_config: TextIOBase,
-    start: datetime | None = None,
-    end: datetime | None = None,
-    start_offset: str | None = None,
-    end_offset: str | None = None,
-    resolution: timedelta | None = None,
-    output_file: Path | None = None,
+    config_file: Optional[TextIOBase],
+    inputs_file: Optional[TextIOBase],
+    start: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    start_offset: Optional[str] = None,
+    end_offset: Optional[str] = None,
+    resolution: Optional[timedelta] = None,
+    output_file: Optional[Path] = None,
     dry_run: bool = False,
-    timezone: str | None = None,
+    edit_config: bool = False,
+    edit_inputs: bool = False,
+    timezone: str | pytz.BaseTzInfo = get_timezone(),
 ):
     """
     Create a new report using the Reporter class and save the results
@@ -1325,19 +1348,44 @@ def add_report(  # noqa: C901
 
     click.secho(f"Reporter {reporter_class} found.", **MsgStyle.SUCCESS)
 
-    reporter_config_raw = json.load(reporter_config)
+    config = None  # dict()
+
+    if config_file:
+        config = yaml.safe_load(config_file)
+
+    if edit_config:
+        config = launch_editor("/tmp/config.yml")
+
+    if config is None:
+        config = dict()
+
+    config["sensor"] = sensor.id
 
     # initialize reporter class with the reporter sensor and reporter config
-    reporter: Reporter = ReporterClass(
-        sensor=sensor, reporter_config_raw=reporter_config_raw
-    )
+    reporter: Reporter = ReporterClass(config=config)
 
     click.echo("Report computation is running...")
 
+    inputs = None
+
+    if inputs_file:
+        inputs = yaml.safe_load(inputs_file)
+
+    if edit_inputs:
+        inputs = launch_editor("/tmp/inputs.yml")
+
+    if inputs is None:
+        inputs = dict()
+
+    if ("start" not in inputs) and (start is not None):
+        inputs["start"] = start.isoformat()
+    if ("end" not in inputs) and (end is not None):
+        inputs["end"] = end.isoformat()
+    if ("resolution" not in inputs) and (resolution is not None):
+        inputs["resolution"] = pd.Timedelta.isoformat(resolution)
+
     # compute the report
-    result: BeliefsDataFrame = reporter.compute(
-        start=start, end=end, input_resolution=resolution
-    )
+    result: BeliefsDataFrame = reporter.compute(inputs=inputs)
 
     if not result.empty:
         click.secho("Report computation done.", **MsgStyle.SUCCESS)
@@ -1391,6 +1439,14 @@ def add_report(  # noqa: C901
         )
 
 
+def launch_editor(filename: str) -> dict | None:
+    """Launch editor to create/edit a json object"""
+    click.edit("{\n}", filename=filename)
+
+    with open(filename, "r") as f:
+        return yaml.safe_load(f)
+
+
 @fm_add_data.command("toy-account")
 @with_appcontext
 @click.option(

flexmeasures/cli/tests/test_data_add.py

@@ -1,5 +1,6 @@
 import pytest
 import json
+import yaml
 import os
 
 
@@ -96,7 +97,7 @@ def test_cli_help(app):
 
 
 @pytest.mark.skip_github
-def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw):
+def test_add_reporter(app, db, setup_dummy_data, reporter_config):
     """
     The reporter aggregates input data from two sensors (both have 200 data points)
     to a two-hour resolution.
@@ -122,21 +123,31 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw):
 
     cli_input_params = {
         "sensor-id": report_sensor_id,
-        "reporter-config": "reporter_config.json",
+        "config": "reporter_config.yaml",
+        "inputs": "inputs.json",
         "reporter": "PandasReporter",
         "start": "2023-04-10T00:00:00 00:00",
         "end": "2023-04-10T10:00:00 00:00",
         "output-file": "test.csv",
     }
 
+    inputs = dict(
+        input_sensors=dict(
+            sensor_1=dict(sensor=sensor1.id), sensor_2=dict(sensor=sensor2.id)
+        )
+    )
+
     cli_input = to_flags(cli_input_params)
 
     # run test in an isolated file system
     with runner.isolated_filesystem():
 
         # save reporter_config to a json file
-        with open("reporter_config.json", "w") as f:
-            json.dump(reporter_config_raw, f)
+        with open("reporter_config.yaml", "w") as f:
+            yaml.dump(reporter_config, f)
+
+        with open("inputs.json", "w") as f:
+            json.dump(inputs, f)
 
         # call command
         result = runner.invoke(add_report, cli_input)
@@ -176,7 +187,8 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw):
 
     cli_input_params = {
         "sensor-id": report_sensor_id,
-        "reporter-config": "reporter_config.json",
+        "config": "reporter_config.json",
+        "inputs": "inputs.json",
         "reporter": "PandasReporter",
         "output-file": "test.csv",
         "timezone": "UTC",
@@ -188,7 +200,10 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw):
 
         # save reporter_config to a json file
         with open("reporter_config.json", "w") as f:
-            json.dump(reporter_config_raw, f)
+            json.dump(reporter_config, f)
+
+        with open("inputs.json", "w") as f:
+            json.dump(inputs, f)
 
         # call command
         result = runner.invoke(add_report, cli_input)

flexmeasures/data/tests/test_data_source.py

@@ -6,8 +6,9 @@
 from pytz import UTC
 
 
-def test_get_reporter_from_source(db, app, aggregator_reporter_data_source):
-
+def test_get_reporter_from_source(
+    db, app, aggregator_reporter_data_source, add_nearby_weather_sensors
+):
     reporter = aggregator_reporter_data_source.data_generator
 
     assert isinstance(reporter, Reporter)
@@ -23,22 +24,27 @@ def test_get_reporter_from_source(db, app, aggregator_reporter_data_source):
         reporter.compute(start=datetime(2023, 1, 1, tzinfo=UTC), end="not a date")
 
 
-def test_data_source(db, app, aggregator_reporter_data_source):
-    TestTeporter = app.data_generators.get("TestReporter")
+def test_creation_of_new_data_source(
+    db, app, aggregator_reporter_data_source, add_nearby_weather_sensors
+):
+    sensor1 = add_nearby_weather_sensors.get("temperature")
+    sensor2 = add_nearby_weather_sensors.get("farther_temperature")
+
+    TestReporter = app.data_generators["TestReporter"]
 
-    ds1 = TestTeporter(config={"sensor": 1})
+    ds1 = TestReporter(config={"sensor": sensor1.id})
 
     db.session.add(ds1.data_source)
     db.session.commit()
 
-    ds2 = TestTeporter(config={"sensor": 1})
+    ds2 = TestReporter(sensor=sensor1)
 
     assert ds1.data_source == ds2.data_source
     assert ds1.data_source.attributes.get("config") == ds2.data_source.attributes.get(
         "config"
     )
 
-    ds3 = TestTeporter(config={"sensor": 2})
+    ds3 = TestReporter(config={"sensor": sensor2.id})
 
     assert ds3.data_source != ds2.data_source
     assert ds3.data_source.attributes.get("config") != ds2.data_source.attributes.get(

requirements/app.in

@@ -1,4 +1,5 @@
 # see ui/utils/plotting_utils: separate_legend() and create_hover_tool()
+pyyaml
 altair
 colour
 pscript