diff --git a/ci/run_mypy.sh b/ci/run_mypy.sh index 3b29de77b..88d5bd181 100755 --- a/ci/run_mypy.sh +++ b/ci/run_mypy.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e pip install --upgrade 'mypy>=0.902' -pip install types-pytz types-requests types-Flask types-click types-redis types-tzlocal types-python-dateutil types-setuptools types-tabulate +pip install types-pytz types-requests types-Flask types-click types-redis types-tzlocal types-python-dateutil types-setuptools types-tabulate types-PyYAML # We are checking python files which have type hints, and leave out bigger issues we made issues for # * data/scripts: We'll remove legacy code: https://trello.com/c/1wEnHOkK/7-remove-custom-data-scripts # * data/models and data/services: https://trello.com/c/rGxZ9h2H/540-makequery-call-signature-is-incoherent diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 2dd8a64cb..d9982e9d8 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -25,6 +25,8 @@ New features * Added API endpoints `/sensors/` for fetching a single sensor, `/sensors` (POST) for adding a sensor, `/sensors/` (PATCH) for updating a sensor and `/sensors/` (DELETE) for deleting a sensor. [see `PR #759 `_] and [see `PR #767 `_] and [see `PR #773 `_] and [see `PR #784 `_] * The CLI now allows to set lists and dicts as asset & sensor attributes (formerly only single values) [see `PR #762 `_] * Add `ProcessScheduler` class to optimize the starting time of processes one of the policies developed (INFLEXIBLE, SHIFTABLE and BREAKABLE), accessible via the CLI command `flexmeasures add schedule for-process` [see `PR #729 `_ and `PR #768 `_] +* Users will be able to see (e.g. in the UI) exactly which reporter created the report (saved as sensor data), and hosts will be able to identify exactly which configuration was used to create a given report [see `PR #751 `_] +* The CLI `flexmeasures add report` now allows passing `config` and `parameters` in YAML format as files or editable via the system's default editor [see `PR #752 `_] Bugfixes ----------- diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 2d5112445..7ea243638 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -9,6 +9,7 @@ since v0.15.0 | July XX, 2023 * Allow deleting multiple sensors with a single call to ``flexmeasures delete sensor`` by passing the ``--id`` option multiple times. * Add ``flexmeasures add schedule for-process`` to create a new process schedule for a given power sensor. +* Add support for describing ``config`` and ``parameters`` in YAML for the command ``flexmeasures add report``, editable in user's code editor using the flags ``--edit-config`` or ``--edit-parameters``. * Add ``--kind process`` option to create the asset and sensors for the ``ProcessScheduler`` tutorial. since v0.14.1 | June XX, 2023 diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index cc60491cf..4fe509f45 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -8,6 +8,7 @@ from typing import Type, List import isodate import json +import yaml from pathlib import Path from io import TextIOBase @@ -1308,16 +1309,23 @@ def add_schedule_process( "--sensor-id", "sensor", type=SensorIdField(), - required=True, - help="Sensor used to save the report. Follow up with the sensor's ID. " + required=False, + help="Sensor used to save the report. Follow up with the sensor's ID. Can be defined in the parameters file, as well" " If needed, use `flexmeasures add sensor` to create a new sensor first.", ) @click.option( - "--reporter-config", - "reporter_config", - required=True, + "--config", + "config_file", + required=False, type=click.File("r"), - help="Path to the JSON file with the reporter configuration.", + help="Path to the JSON or YAML file with the configuration of the reporter.", +) +@click.option( + "--parameters", + "parameters_file", + required=False, + type=click.File("r"), + help="Path to the JSON or YAML file with the report parameters (passed to the compute step).", ) @click.option( "--reporter", @@ -1382,10 +1390,29 @@ def add_schedule_process( is_flag=True, help="Add this flag to avoid saving the results to the database.", ) +@click.option( + "--edit-config", + "edit_config", + is_flag=True, + help="Add this flag to edit the configuration of the Reporter in your default text editor (e.g. nano).", +) +@click.option( + "--edit-parameters", + "edit_parameters", + is_flag=True, + help="Add this flag to edit the parameters passed to the Reporter in your default text editor (e.g. nano).", +) +@click.option( + "--save-config", + "save_config", + is_flag=True, + help="Add this flag to save the `config` in the attributes of the DataSource for future reference.", +) def add_report( # noqa: C901 reporter_class: str, - sensor: Sensor, - reporter_config: TextIOBase, + sensor: Sensor | None = None, + config_file: TextIOBase | None = None, + parameters_file: TextIOBase | None = None, start: datetime | None = None, end: datetime | None = None, start_offset: str | None = None, @@ -1393,6 +1420,9 @@ def add_report( # noqa: C901 resolution: timedelta | None = None, output_file: Path | None = None, dry_run: bool = False, + edit_config: bool = False, + edit_parameters: bool = False, + save_config: bool = False, timezone: str | None = None, ): """ @@ -1400,9 +1430,40 @@ def add_report( # noqa: C901 to the database or export them as CSV or Excel file. """ + config = dict() + + if config_file: + config = yaml.safe_load(config_file) + + if edit_config: + config = launch_editor("/tmp/config.yml") + + parameters = dict() + + if parameters_file: + parameters = yaml.safe_load(parameters_file) + + if edit_parameters: + parameters = launch_editor("/tmp/parameters.yml") + + if sensor is not None: + parameters["sensor"] = sensor.id + + # check if sensor is not provided either in the parameters or the CLI + # click parameter + if parameters.get("sensor") is None: + click.secho( + "Report sensor needs to be defined, either on the `parameters` file or trough the --sensor CLI parameter...", + **MsgStyle.ERROR, + ) + raise click.Abort() + + sensor = Sensor.query.get(parameters.get("sensor")) + # compute now in the timezone local to the output sensor if timezone is not None: check_timezone(timezone) + now = pytz.timezone( zone=timezone if timezone is not None else sensor.timezone ).localize(datetime.now()) @@ -1457,7 +1518,9 @@ def add_report( # noqa: C901 ) # get reporter class - ReporterClass: Type[Reporter] = app.reporters.get(reporter_class) + ReporterClass: Type[Reporter] = app.data_generators.get("reporter").get( + reporter_class + ) # check if it exists if ReporterClass is None: @@ -1469,19 +1532,20 @@ def add_report( # noqa: C901 click.secho(f"Reporter {reporter_class} found.", **MsgStyle.SUCCESS) - reporter_config_raw = json.load(reporter_config) - # initialize reporter class with the reporter sensor and reporter config - reporter: Reporter = ReporterClass( - sensor=sensor, reporter_config_raw=reporter_config_raw - ) + reporter: Reporter = ReporterClass(config=config, save_config=save_config) click.echo("Report computation is running...") + if ("start" not in parameters) and (start is not None): + parameters["start"] = start.isoformat() + if ("end" not in parameters) and (end is not None): + parameters["end"] = end.isoformat() + if ("resolution" not in parameters) and (resolution is not None): + parameters["resolution"] = pd.Timedelta(resolution).isoformat() + # compute the report - result: BeliefsDataFrame = reporter.compute( - start=start, end=end, input_resolution=resolution - ) + result: BeliefsDataFrame = reporter.compute(parameters=parameters) if not result.empty: click.secho("Report computation done.", **MsgStyle.SUCCESS) @@ -1535,6 +1599,18 @@ def add_report( # noqa: C901 ) +def launch_editor(filename: str) -> dict: + """Launch editor to create/edit a json object""" + click.edit("{\n}", filename=filename) + + with open(filename, "r") as f: + content = yaml.safe_load(f) + if content is None: + return dict() + + return content + + @fm_add_data.command("toy-account") @with_appcontext @click.option( diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py index 5c174bdb7..5c07f0224 100644 --- a/flexmeasures/cli/tests/test_data_add.py +++ b/flexmeasures/cli/tests/test_data_add.py @@ -1,5 +1,6 @@ import pytest import json +import yaml import os @@ -121,22 +122,32 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config): runner = app.test_cli_runner() cli_input_params = { - "sensor-id": report_sensor_id, - "reporter-config": "reporter_config.json", + "config": "reporter_config.yaml", + "parameters": "parameters.json", "reporter": "PandasReporter", "start": "2023-04-10T00:00:00 00:00", "end": "2023-04-10T10:00:00 00:00", "output-file": "test.csv", } + parameters = dict( + input_variables=dict( + sensor_1=dict(sensor=sensor1.id), sensor_2=dict(sensor=sensor2.id) + ), + sensor=report_sensor_id, + ) + cli_input = to_flags(cli_input_params) # run test in an isolated file system with runner.isolated_filesystem(): # save reporter_config to a json file - with open("reporter_config.json", "w") as f: - json.dump(reporter_config, f) + with open("reporter_config.yaml", "w") as f: + yaml.dump(reporter_config, f) + + with open("parameters.json", "w") as f: + json.dump(parameters, f) # call command result = runner.invoke(add_report, cli_input) @@ -175,8 +186,8 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config): previous_command_end = cli_input_params.get("end").replace(" ", "+") cli_input_params = { - "sensor-id": report_sensor_id, - "reporter-config": "reporter_config.json", + "config": "reporter_config.json", + "parameters": "parameters.json", "reporter": "PandasReporter", "output-file": "test.csv", "timezone": "UTC", @@ -190,6 +201,9 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config): with open("reporter_config.json", "w") as f: json.dump(reporter_config, f) + with open("parameters.json", "w") as f: + json.dump(parameters, f) + # call command result = runner.invoke(add_report, cli_input) diff --git a/requirements/app.in b/requirements/app.in index b26e2a80a..0babfb74b 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -1,4 +1,5 @@ # see ui/utils/plotting_utils: separate_legend() and create_hover_tool() +pyyaml altair colour pscript diff --git a/requirements/app.txt b/requirements/app.txt index 340a23f4d..53917b19d 100644 --- a/requirements/app.txt +++ b/requirements/app.txt @@ -253,6 +253,8 @@ pytz==2023.3 # pandas # timely-beliefs # timetomodel +pyyaml==6.0.1 + # via -r requirements/app.in redis==4.6.0 # via # -r requirements/app.in