feat: support YAML in flexmeasures add report command (#752)

* feat: add pyyaml to the requirements Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * feat: support YAML and add report_config Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * fix: move `types-PyYAML` dependency to the right place Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * fix: use a DataGenerator with defined schemas Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * fix: adapt tests of the schemas Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * feat: add option to open default editor Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * fix: move sensor to input Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * fix: parse resolution properly Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * fix: remove accidentally commited file Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * fix: avoid potential bug Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * rename input to parameters Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * add chagelog entry Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * add pyyaml to app.txt Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * add --save-config to the add_report command Signed-off-by: Victor Garcia Reolid <victor@seita.nl> * improve changelog files Signed-off-by: Victor Garcia Reolid <victor@seita.nl> --------- Signed-off-by: Victor Garcia Reolid <victor@seita.nl> Signed-off-by: Victor <victor@seita.nl>
FlexMeasures · Aug 3, 2023 · 1e8704e · 1e8704e
1 parent 4b676b0
commit 1e8704e
Show file tree

Hide file tree

Showing 7 changed files with 120 additions and 24 deletions.
diff --git a/ci/run_mypy.sh b/ci/run_mypy.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 pip install --upgrade 'mypy>=0.902'
-pip install types-pytz types-requests types-Flask types-click types-redis types-tzlocal types-python-dateutil types-setuptools types-tabulate
+pip install types-pytz types-requests types-Flask types-click types-redis types-tzlocal types-python-dateutil types-setuptools types-tabulate types-PyYAML
 # We are checking python files which have type hints, and leave out bigger issues we made issues for
 # * data/scripts: We'll remove legacy code: https://trello.com/c/1wEnHOkK/7-remove-custom-data-scripts
 # * data/models and data/services: https://trello.com/c/rGxZ9h2H/540-makequery-call-signature-is-incoherent

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
@@ -25,6 +25,8 @@ New features
 * Added API endpoints `/sensors/<id>` for fetching a single sensor, `/sensors` (POST) for adding a sensor, `/sensors/<id>` (PATCH) for updating a sensor and `/sensors/<id>` (DELETE) for deleting a sensor. [see `PR #759 <https://www.github.com/FlexMeasures/flexmeasures/pull/759>`_] and [see `PR #767 <https://www.github.com/FlexMeasures/flexmeasures/pull/767>`_] and [see `PR #773 <https://www.github.com/FlexMeasures/flexmeasures/pull/773>`_] and [see `PR #784 <https://www.github.com/FlexMeasures/flexmeasures/pull/784>`_]
 * The CLI now allows to set lists and dicts as asset & sensor attributes (formerly only single values) [see `PR #762 <https://www.github.com/FlexMeasures/flexmeasures/pull/762>`_]
 * Add `ProcessScheduler` class to optimize the starting time of processes one of the policies developed (INFLEXIBLE, SHIFTABLE and BREAKABLE), accessible via the CLI command `flexmeasures add schedule for-process` [see `PR #729 <https://www.github.com/FlexMeasures/flexmeasures/pull/729>`_ and `PR #768 <https://www.github.com/FlexMeasures/flexmeasures/pull/768>`_]
+* Users will be able to see (e.g. in the UI) exactly which reporter created the report (saved as sensor data), and hosts will be able to identify exactly which configuration was used to create a given report [see `PR #751 <https://www.github.com/FlexMeasures/flexmeasures/pull/751>`_]
+* The CLI `flexmeasures add report` now allows passing `config` and `parameters` in YAML format as files or editable via the system's default editor [see `PR #752 <https://www.github.com/FlexMeasures/flexmeasures/pull/752>`_]
 
 Bugfixes
 -----------

diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst
@@ -9,6 +9,7 @@ since v0.15.0 | July XX, 2023
 
 * Allow deleting multiple sensors with a single call to ``flexmeasures delete sensor`` by passing the ``--id`` option multiple times.
 * Add ``flexmeasures add schedule for-process`` to create a new process schedule for a given power sensor.
+* Add support for describing ``config`` and ``parameters`` in YAML for the command ``flexmeasures add report``, editable in user's code editor using the flags ``--edit-config`` or ``--edit-parameters``.
 * Add ``--kind process`` option to create the asset and sensors for the ``ProcessScheduler`` tutorial.
 
 since v0.14.1 | June XX, 2023

diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py
@@ -8,6 +8,7 @@
 from typing import Type, List
 import isodate
 import json
+import yaml
 from pathlib import Path
 from io import TextIOBase
 
@@ -1308,16 +1309,23 @@ def add_schedule_process(
     "--sensor-id",
     "sensor",
     type=SensorIdField(),
-    required=True,
-    help="Sensor used to save the report. Follow up with the sensor's ID. "
+    required=False,
+    help="Sensor used to save the report. Follow up with the sensor's ID. Can be defined in the parameters file, as well"
     " If needed, use `flexmeasures add sensor` to create a new sensor first.",
 )
 @click.option(
-    "--reporter-config",
-    "reporter_config",
-    required=True,
+    "--config",
+    "config_file",
+    required=False,
     type=click.File("r"),
-    help="Path to the JSON file with the reporter configuration.",
+    help="Path to the JSON or YAML file with the configuration of the reporter.",
+)
+@click.option(
+    "--parameters",
+    "parameters_file",
+    required=False,
+    type=click.File("r"),
+    help="Path to the JSON or YAML file with the report parameters (passed to the compute step).",
 )
 @click.option(
     "--reporter",
@@ -1382,27 +1390,80 @@ def add_schedule_process(
     is_flag=True,
     help="Add this flag to avoid saving the results to the database.",
 )
+@click.option(
+    "--edit-config",
+    "edit_config",
+    is_flag=True,
+    help="Add this flag to edit the configuration of the Reporter in your default text editor (e.g. nano).",
+)
+@click.option(
+    "--edit-parameters",
+    "edit_parameters",
+    is_flag=True,
+    help="Add this flag to edit the parameters passed to the Reporter in your default text editor (e.g. nano).",
+)
+@click.option(
+    "--save-config",
+    "save_config",
+    is_flag=True,
+    help="Add this flag to save the `config` in the attributes of the DataSource for future reference.",
+)
 def add_report(  # noqa: C901
     reporter_class: str,
-    sensor: Sensor,
-    reporter_config: TextIOBase,
+    sensor: Sensor | None = None,
+    config_file: TextIOBase | None = None,
+    parameters_file: TextIOBase | None = None,
     start: datetime | None = None,
     end: datetime | None = None,
     start_offset: str | None = None,
     end_offset: str | None = None,
     resolution: timedelta | None = None,
     output_file: Path | None = None,
     dry_run: bool = False,
+    edit_config: bool = False,
+    edit_parameters: bool = False,
+    save_config: bool = False,
     timezone: str | None = None,
 ):
     """
     Create a new report using the Reporter class and save the results
     to the database or export them as CSV or Excel file.
     """
 
+    config = dict()
+
+    if config_file:
+        config = yaml.safe_load(config_file)
+
+    if edit_config:
+        config = launch_editor("/tmp/config.yml")
+
+    parameters = dict()
+
+    if parameters_file:
+        parameters = yaml.safe_load(parameters_file)
+
+    if edit_parameters:
+        parameters = launch_editor("/tmp/parameters.yml")
+
+    if sensor is not None:
+        parameters["sensor"] = sensor.id
+
+    # check if sensor is not provided either in the parameters or the CLI
+    # click parameter
+    if parameters.get("sensor") is None:
+        click.secho(
+            "Report sensor needs to be defined, either on the `parameters` file or trough the --sensor CLI parameter...",
+            **MsgStyle.ERROR,
+        )
+        raise click.Abort()
+
+    sensor = Sensor.query.get(parameters.get("sensor"))
+
     # compute now in the timezone local to the output sensor
     if timezone is not None:
         check_timezone(timezone)
+
     now = pytz.timezone(
         zone=timezone if timezone is not None else sensor.timezone
     ).localize(datetime.now())
@@ -1457,7 +1518,9 @@ def add_report(  # noqa: C901
     )
 
     # get reporter class
-    ReporterClass: Type[Reporter] = app.reporters.get(reporter_class)
+    ReporterClass: Type[Reporter] = app.data_generators.get("reporter").get(
+        reporter_class
+    )
 
     # check if it exists
     if ReporterClass is None:
@@ -1469,19 +1532,20 @@ def add_report(  # noqa: C901
 
     click.secho(f"Reporter {reporter_class} found.", **MsgStyle.SUCCESS)
 
-    reporter_config_raw = json.load(reporter_config)
-
     # initialize reporter class with the reporter sensor and reporter config
-    reporter: Reporter = ReporterClass(
-        sensor=sensor, reporter_config_raw=reporter_config_raw
-    )
+    reporter: Reporter = ReporterClass(config=config, save_config=save_config)
 
     click.echo("Report computation is running...")
 
+    if ("start" not in parameters) and (start is not None):
+        parameters["start"] = start.isoformat()
+    if ("end" not in parameters) and (end is not None):
+        parameters["end"] = end.isoformat()
+    if ("resolution" not in parameters) and (resolution is not None):
+        parameters["resolution"] = pd.Timedelta(resolution).isoformat()
+
     # compute the report
-    result: BeliefsDataFrame = reporter.compute(
-        start=start, end=end, input_resolution=resolution
-    )
+    result: BeliefsDataFrame = reporter.compute(parameters=parameters)
 
     if not result.empty:
         click.secho("Report computation done.", **MsgStyle.SUCCESS)
@@ -1535,6 +1599,18 @@ def add_report(  # noqa: C901
         )
 
 
+def launch_editor(filename: str) -> dict:
+    """Launch editor to create/edit a json object"""
+    click.edit("{\n}", filename=filename)
+
+    with open(filename, "r") as f:
+        content = yaml.safe_load(f)
+        if content is None:
+            return dict()
+
+        return content
+
+
 @fm_add_data.command("toy-account")
 @with_appcontext
 @click.option(

diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py
@@ -1,5 +1,6 @@
 import pytest
 import json
+import yaml
 import os
 
 
@@ -121,22 +122,32 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config):
     runner = app.test_cli_runner()
 
     cli_input_params = {
-        "sensor-id": report_sensor_id,
-        "reporter-config": "reporter_config.json",
+        "config": "reporter_config.yaml",
+        "parameters": "parameters.json",
         "reporter": "PandasReporter",
         "start": "2023-04-10T00:00:00 00:00",
         "end": "2023-04-10T10:00:00 00:00",
         "output-file": "test.csv",
     }
 
+    parameters = dict(
+        input_variables=dict(
+            sensor_1=dict(sensor=sensor1.id), sensor_2=dict(sensor=sensor2.id)
+        ),
+        sensor=report_sensor_id,
+    )
+
     cli_input = to_flags(cli_input_params)
 
     # run test in an isolated file system
     with runner.isolated_filesystem():
 
         # save reporter_config to a json file
-        with open("reporter_config.json", "w") as f:
-            json.dump(reporter_config, f)
+        with open("reporter_config.yaml", "w") as f:
+            yaml.dump(reporter_config, f)
+
+        with open("parameters.json", "w") as f:
+            json.dump(parameters, f)
 
         # call command
         result = runner.invoke(add_report, cli_input)
@@ -175,8 +186,8 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config):
     previous_command_end = cli_input_params.get("end").replace(" ", "+")
 
     cli_input_params = {
-        "sensor-id": report_sensor_id,
-        "reporter-config": "reporter_config.json",
+        "config": "reporter_config.json",
+        "parameters": "parameters.json",
         "reporter": "PandasReporter",
         "output-file": "test.csv",
         "timezone": "UTC",
@@ -190,6 +201,9 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config):
         with open("reporter_config.json", "w") as f:
             json.dump(reporter_config, f)
 
+        with open("parameters.json", "w") as f:
+            json.dump(parameters, f)
+
         # call command
         result = runner.invoke(add_report, cli_input)
 

diff --git a/requirements/app.in b/requirements/app.in
@@ -1,4 +1,5 @@
 # see ui/utils/plotting_utils: separate_legend() and create_hover_tool()
+pyyaml
 altair
 colour
 pscript

diff --git a/requirements/app.txt b/requirements/app.txt
@@ -253,6 +253,8 @@ pytz==2023.3
     #   pandas
     #   timely-beliefs
     #   timetomodel
+pyyaml==6.0.1
+    # via -r requirements/app.in
 redis==4.6.0
     # via
     #   -r requirements/app.in