Skip to content

Commit

Permalink
Adapt test_add_report to use the new field of the PandasReporter …
Browse files Browse the repository at this point in the history 
…schema (#789)

* fix: typo

Signed-off-by: F.N. Claessen <felix@seita.nl>

* fix: fetch output sensor only from parameters dict

Signed-off-by: F.N. Claessen <felix@seita.nl>

* adapt the CLI to deal with multiple output

Signed-off-by: Victor Garcia Reolid <victor@seita.nl>

* fix typos

Signed-off-by: Victor Garcia Reolid <victor@seita.nl>

---------

Signed-off-by: F.N. Claessen <felix@seita.nl>
Signed-off-by: Victor Garcia Reolid <victor@seita.nl>
Co-authored-by: F.N. Claessen <felix@seita.nl>
  • Loading branch information
@victorgarcia98 @Flix6x
victorgarcia98 and Flix6x committed Aug 7, 2023
1 parent a6829d1 commit 5bf20ac
Show file tree
Hide file tree
Showing 3 changed files with 211 additions and 111 deletions.
148 changes: 81 additions & 67 deletions flexmeasures/cli/data_add.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import yaml
from pathlib import Path
from io import TextIOBase
from string import Template

from marshmallow import validate
import pandas as pd
Expand Down Expand Up @@ -58,6 +59,7 @@
from flexmeasures.data.schemas.times import TimeIntervalSchema
from flexmeasures.data.schemas.scheduling.storage import EfficiencyField
from flexmeasures.data.schemas.sensors import SensorSchema
from flexmeasures.data.schemas.io import Output
from flexmeasures.data.schemas.units import QuantityField
from flexmeasures.data.schemas.generic_assets import (
GenericAssetSchema,
Expand Down Expand Up @@ -1305,14 +1307,6 @@ def add_schedule_process(

@fm_add_data.command("report")
@with_appcontext
@click.option(
"--sensor-id",
"sensor",
type=SensorIdField(),
required=False,
help="Sensor used to save the report. Follow up with the sensor's ID. Can be defined in the parameters file, as well"
" If needed, use `flexmeasures add sensor` to create a new sensor first.",
)
@click.option(
"--config",
"config_file",
Expand Down Expand Up @@ -1372,17 +1366,20 @@ def add_schedule_process(
)
@click.option(
"--output-file",
"output_file",
"output_file_pattern",
required=False,
type=click.Path(),
help="Path to save the report to file. Will override any previous file contents."
" Use the `.csv` suffix to save the results as Comma Separated Values and `.xlsx` to export them as Excel sheets.",
help="Format of the output file. Use dollar sign ($) to interpolate values among the following ones:"
" now (current time), name (name of the output), sensor_id (id of the sensor), column (column of the output)."
" Example: 'result_file_$name_$now.csv'. "
"Use the `.csv` suffix to save the results as Comma Separated Values and `.xlsx` to export them as Excel sheets.",
)
@click.option(
"--timezone",
"timezone",
required=False,
help="Timezone as string, e.g. 'UTC' or 'Europe/Amsterdam' (defaults to the timezone of the sensor used to save the report).",
help="Timezone as string, e.g. 'UTC' or 'Europe/Amsterdam' (defaults to the timezone of the sensor used to save the report)."
"The timezone of the first output sensor (specified in the parameters) is taken as a default.",
)
@click.option(
"--dry-run",
Expand Down Expand Up @@ -1410,15 +1407,14 @@ def add_schedule_process(
)
def add_report( # noqa: C901
reporter_class: str,
sensor: Sensor | None = None,
config_file: TextIOBase | None = None,
parameters_file: TextIOBase | None = None,
start: datetime | None = None,
end: datetime | None = None,
start_offset: str | None = None,
end_offset: str | None = None,
resolution: timedelta | None = None,
output_file: Path | None = None,
output_file_pattern: Path | None = None,
dry_run: bool = False,
edit_config: bool = False,
edit_parameters: bool = False,
Expand Down Expand Up @@ -1446,26 +1442,22 @@ def add_report( # noqa: C901
if edit_parameters:
parameters = launch_editor("/tmp/parameters.yml")

if sensor is not None:
parameters["sensor"] = sensor.id

# check if sensor is not provided either in the parameters or the CLI
# click parameter
if parameters.get("sensor") is None:
# check if sensor is not provided in the `parameters` description
if "output" not in parameters or len(parameters["output"]) == 0:
click.secho(
"Report sensor needs to be defined, either on the `parameters` file or trough the --sensor CLI parameter...",
"At least one output sensor needs to be specified in the parameters description.",
**MsgStyle.ERROR,
)
raise click.Abort()

sensor = Sensor.query.get(parameters.get("sensor"))
output = [Output().load(o) for o in parameters["output"]]

# compute now in the timezone local to the output sensor
if timezone is not None:
check_timezone(timezone)

now = pytz.timezone(
zone=timezone if timezone is not None else sensor.timezone
zone=timezone if timezone is not None else output[0]["sensor"].timezone
).localize(datetime.now())

# apply offsets, if provided
Expand All @@ -1486,9 +1478,11 @@ def add_report( # noqa: C901
" Trying to use the latest datapoint of the report sensor as the start time...",
**MsgStyle.WARN,
)

# todo: get the oldest last_value among all the sensors
last_value_datetime = (
db.session.query(func.max(TimedBelief.event_start))
.filter(TimedBelief.sensor_id == sensor.id)
.filter(TimedBelief.sensor_id == output[0]["sensor"].id)
.one_or_none()
)

Expand All @@ -1497,7 +1491,8 @@ def add_report( # noqa: C901
start = last_value_datetime[0]
else:
click.secho(
f"Could not find any data for the report sensor {sensor}.",
"Could not find any data for the output sensors provided. Such data is needed to compute"
" a sensible default start for the report, so setting a start explicitly would resolve this issue.",
**MsgStyle.ERROR,
)
raise click.Abort()
Expand Down Expand Up @@ -1545,58 +1540,77 @@ def add_report( # noqa: C901
parameters["resolution"] = pd.Timedelta(resolution).isoformat()

# compute the report
result: BeliefsDataFrame = reporter.compute(parameters=parameters)
results: BeliefsDataFrame = reporter.compute(parameters=parameters)

if not result.empty:
click.secho("Report computation done.", **MsgStyle.SUCCESS)
else:
click.secho(
"Report computation done, but the report is empty.", **MsgStyle.WARN
)

# save the report if it's not running in dry mode
if not dry_run:
click.echo("Saving report to the database...")
save_to_db(result.dropna())
db.session.commit()
click.secho(
"Success. The report has been saved to the database.",
**MsgStyle.SUCCESS,
)
else:
click.echo(
f"Not saving report to the database (because of --dry-run), but this is what I computed:\n{result}"
)

# if an output file path is provided, save the results
if output_file:
suffix = str(output_file).split(".")[-1] if "." in str(output_file) else ""

if suffix == "xlsx": # save to EXCEL
result.to_excel(output_file)
for result in results:
data = result["data"]
sensor = result["sensor"]
if not data.empty:
click.secho(
f"Success. The report has been exported as EXCEL to the file `{output_file}`",
**MsgStyle.SUCCESS,
f"Report computation done for sensor `{sensor}`.", **MsgStyle.SUCCESS
)
else:
click.secho(
f"Report computation done for sensor `{sensor}`, but the report is empty.",
**MsgStyle.WARN,
)

elif suffix == "csv": # save to CSV
result.to_csv(output_file)
# save the report if it's not running in dry mode
if not dry_run:
click.echo(f"Saving report for sensor `{sensor}` to the database...")
save_to_db(data.dropna())
db.session.commit()
click.secho(
f"Success. The report has been exported as CSV to the file `{output_file}`",
f"Success. The report for sensor `{sensor}` has been saved to the database.",
**MsgStyle.SUCCESS,
)
else:
click.echo(
f"Not saving report for sensor `{sensor}` to the database (because of --dry-run), but this is what I computed:\n{data}"
)

# if an output file path is provided, save the data
if output_file_pattern:
suffix = (
str(output_file_pattern).split(".")[-1]
if "." in str(output_file_pattern)
else ""
)
template = Template(str(output_file_pattern))

filename = template.safe_substitute(
sensor_id=result["sensor"].id,
name=result.get("name", ""),
column=result.get("column", ""),
reporter_class=reporter_class,
now=now.strftime("%Y_%m_%dT%H%M%S"),
)

if suffix == "xlsx": # save to EXCEL
data.to_excel(filename)
click.secho(
f"Success. The report for sensor `{sensor}` has been exported as EXCEL to the file `{filename}`",
**MsgStyle.SUCCESS,
)

else: # default output format: CSV.
elif suffix == "csv": # save to CSV
data.to_csv(filename)
click.secho(
f"Success. The report for sensor `{sensor}` has been exported as CSV to the file `{filename}`",
**MsgStyle.SUCCESS,
)

else: # default output format: CSV.
click.secho(
f"File suffix not provided. Exporting results for sensor `{sensor}` as CSV to file {filename}",
**MsgStyle.WARN,
)
data.to_csv(filename)
else:
click.secho(
f"File suffix not provided. Exporting results as CSV to file {output_file}",
**MsgStyle.WARN,
"Success.",
**MsgStyle.SUCCESS,
)
result.to_csv(output_file)
else:
click.secho(
"Success.",
**MsgStyle.SUCCESS,
)


def launch_editor(filename: str) -> dict:
Expand Down
37 changes: 8 additions & 29 deletions flexmeasures/cli/tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,13 @@ def setup_dummy_data(db, app, setup_dummy_asset):
)
db.session.add(report_sensor)

report_sensor_2 = Sensor(
"report sensor 2",
generic_asset=pandas_report,
event_resolution=timedelta(hours=2),
)
db.session.add(report_sensor_2)

# Create 1 DataSources
source = DataSource("source1")

Expand All @@ -83,35 +90,7 @@ def setup_dummy_data(db, app, setup_dummy_asset):
db.session.add_all(beliefs)
db.session.commit()

yield sensor1, sensor2, report_sensor


@pytest.fixture(scope="module")
@pytest.mark.skip_github
def reporter_config(app, db, setup_dummy_data):
"""
This reporter_config defines the operations to add up the
values of the sensors 1 and 2 and resamples the result to a
two hour resolution.
"""

sensor1, sensor2, report_sensor = setup_dummy_data

reporter_config = dict(
required_input=[{"name": "sensor_1"}, {"name": "sensor_2"}],
required_output=[{"name": "df_agg"}],
transformations=[
dict(
df_input="sensor_1",
method="add",
args=["@sensor_2"],
df_output="df_agg",
),
dict(method="resample_events", args=["2h"]),
],
)

return reporter_config
yield sensor1.id, sensor2.id, report_sensor.id, report_sensor_2.id


@pytest.mark.skip_github
Expand Down

0 comments on commit 5bf20ac

Please sign in to comment.