From 1d803c169777d47e3637c7f2eea96f5c4063373b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 27 Nov 2022 23:29:01 +0100 Subject: [PATCH 01/38] Upgrade timely-beliefs Signed-off-by: F.N. Claessen --- requirements/app.in | 2 +- requirements/app.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/app.in b/requirements/app.in index df6a9a220..e09d1a5a2 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -28,7 +28,7 @@ tldextract pyomo>=5.6 tabulate timetomodel>=0.7.1 -timely-beliefs[forecast]>=1.14 +timely-beliefs[forecast]>=1.15 python-dotenv # a backport, not needed in Python3.8 importlib_metadata diff --git a/requirements/app.txt b/requirements/app.txt index 480874aa5..9aded522c 100644 --- a/requirements/app.txt +++ b/requirements/app.txt @@ -314,7 +314,7 @@ tabulate==0.8.10 # via -r requirements/app.in threadpoolctl==3.1.0 # via scikit-learn -timely-beliefs[forecast]==1.14.0 +timely-beliefs[forecast]==1.15.0 # via -r requirements/app.in timetomodel==0.7.1 # via -r requirements/app.in From c1fe8b8ab6319a1cb43af4b1b85c8693e183cac9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 10:17:52 +0100 Subject: [PATCH 02/38] Use requested event frequency instead of df.event_resolution, and come up with default for instantaneous sensors Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index b44fc6738..7a7a5e3a7 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -17,7 +17,11 @@ from flexmeasures.data.models.planning.utils import initialize_index from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField from flexmeasures.data.services.time_series import simplify_index -from flexmeasures.utils.time_utils import duration_isoformat, server_now +from flexmeasures.utils.time_utils import ( + decide_resolution, + duration_isoformat, + server_now, +) from flexmeasures.utils.unit_utils import ( convert_units, units_are_convertible, @@ -153,6 +157,10 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: unit = sensor_data_description["unit"] resolution = sensor_data_description.get("resolution") + # Post-load default frequency for instantaneous sensors + if resolution is None and sensor.event_resolution == timedelta(hours=0): + resolution = decide_resolution(start, end) + # Post-load configuration of belief timing against message type horizons_at_least = sensor_data_description.get("horizon", None) horizons_at_most = None @@ -180,7 +188,7 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: ) # Convert to desired time range - index = initialize_index(start=start, end=end, resolution=df.event_resolution) + index = initialize_index(start=start, end=end, resolution=resolution) df = df.reindex(index) # Convert to desired unit From d4f5ac3904c180994384e48f15c50bdc9f4887e0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 10:18:22 +0100 Subject: [PATCH 03/38] Extend docstring for TimedBelief.search Signed-off-by: F.N. Claessen --- flexmeasures/data/models/time_series.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index d1acf04f6..17c8f7fe6 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -611,7 +611,10 @@ def search( * If user_source_ids is specified, the "user" source type is automatically included (and not excluded). Somewhat redundant, though still allowed, is to set both source_types and exclude_source_types. - ** Note that timely-beliefs converts string resolutions to datetime.timedelta objects (see https://github.com/SeitaBV/timely-beliefs/issues/13). + ** Note that: + - timely-beliefs converts string resolutions to datetime.timedelta objects (see https://github.com/SeitaBV/timely-beliefs/issues/13). + - for sensors recording non-instantaneous data: updates both the event frequency and the event resolution + - for sensors recording instantaneous data: updates only the event frequency (and event resolution remains 0) """ # todo: deprecate the 'sensor' argument in favor of 'sensors' (announced v0.8.0) sensors = tb_utils.replace_deprecated_argument( From 1f633da35c687dac9e5ccd9e42d65b69a4c20f22 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 11:58:38 +0100 Subject: [PATCH 04/38] Return event resolution explicitly, because it may differ from the event frequency Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 7a7a5e3a7..b8e15e3bf 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -207,6 +207,7 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: start=datetime_isoformat(start), duration=duration_isoformat(duration), unit=unit, + resolution=duration_isoformat(df.event_resolution), ) return response From aa8033617ecd84c7e9ba80144edcc1ebad52993b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 12:10:03 +0100 Subject: [PATCH 05/38] Make sure reindex is pass a (non-zero) timedelta instead of a possible None value Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index b8e15e3bf..c72307d5f 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -157,9 +157,13 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: unit = sensor_data_description["unit"] resolution = sensor_data_description.get("resolution") - # Post-load default frequency for instantaneous sensors - if resolution is None and sensor.event_resolution == timedelta(hours=0): - resolution = decide_resolution(start, end) + # Post-load configuration of event frequency + if resolution is None: + if sensor.event_resolution != timedelta(hours=0): + resolution = sensor.event_resolution + else: + # For instantaneous sensors, choose a default resolution given the requested time window + resolution = decide_resolution(start, end) # Post-load configuration of belief timing against message type horizons_at_least = sensor_data_description.get("horizon", None) From 60fc05ed111548af06ff8fbd6a1d0aa49d322561 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 12:14:36 +0100 Subject: [PATCH 06/38] Remove old documentation notes concerning the use of the fm0 schema for UDI events Signed-off-by: F.N. Claessen --- documentation/api/notation.rst | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index e989f1757..0449b73ce 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -92,17 +92,9 @@ It uses the fact that all FlexMeasures sensors have unique IDs. ea1.2021-01.io.flexmeasures:fm1.42 ea1.2021-01.io.flexmeasures:fm1. -.. todo:: UDI events are not yet modelled in the fm1 scheme - The ``fm0`` scheme is the original scheme. It identified different types of sensors (such as grid connections, weather sensors and markets) in different ways. -The ``fm0`` scheme has been deprecated for the most part and is no longer supported officially. -Only UDI events still need to be sent using the fm0 scheme. - -.. code-block:: - - ea1.2021-01.io.flexmeasures:fm0.40:30:302:soc - ea1.2021-01.io.flexmeasures:fm0.::: +The ``fm0`` scheme has been deprecated and is no longer supported officially. Timeseries From 21d3157e58e40e13d24cea0eea780830992ac134 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 12:37:20 +0100 Subject: [PATCH 07/38] Clarify resolution check for posting sensor data Signed-off-by: F.N. Claessen --- .../api/common/schemas/sensor_data.py | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index c72307d5f..ed963050b 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -266,12 +266,21 @@ def check_schema_unit_against_type(self, data, **kwargs): ) @validates_schema - def check_resolution_compatibility_of_values(self, data, **kwargs): - inferred_resolution = data["duration"] / len(data["values"]) + def check_resolution_compatibility_of_sensor_data(self, data, **kwargs): + """Ensure event frequency is compatible with the sensor's event resolution. + + For a sensor recording instantaneous values, any event frequency is compatible. + For a sensor recording non-instantaneous values, the event frequency must fit the sensor's event resolution. + Currently, only upsampling is supported (e.g. converting hourly events to 15-minute events). + """ required_resolution = data["sensor"].event_resolution - # TODO: we don't yet have a good policy w.r.t. zero-resolution (direct measurement) if required_resolution == timedelta(hours=0): + # For instantaneous sensors, any event frequency is compatible return + + # The event frequency is inferred by assuming sequential, equidistant values within a time interval. + # The event resolution is assumed to be equal to the event frequency. + inferred_resolution = data["duration"] / len(data["values"]) if inferred_resolution % required_resolution != timedelta(hours=0): raise ValidationError( f"Resolution of {inferred_resolution} is incompatible with the sensor's required resolution of {required_resolution}." @@ -315,13 +324,15 @@ def possibly_upsample_values(data): Upsample the data if needed, to fit to the sensor's resolution. Marshmallow runs this after validation. """ - inferred_resolution = data["duration"] / len(data["values"]) required_resolution = data["sensor"].event_resolution - - # TODO: we don't yet have a good policy w.r.t. zero-resolution (direct measurement) if required_resolution == timedelta(hours=0): + # For instantaneous sensors, no need to upsample return data + # The event frequency is inferred by assuming sequential, equidistant values within a time interval. + # The event resolution is assumed to be equal to the event frequency. + inferred_resolution = data["duration"] / len(data["values"]) + # we already know resolutions are compatible (see validation) if inferred_resolution != required_resolution: data["values"] = upsample_values( From b267694a284900b290fed981145b00eced354740 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 13:25:09 +0100 Subject: [PATCH 08/38] Update notation in API documentation Signed-off-by: F.N. Claessen --- documentation/api/notation.rst | 36 ++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index 0449b73ce..a8481c0c7 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -100,7 +100,8 @@ The ``fm0`` scheme has been deprecated and is no longer supported officially. Timeseries ^^^^^^^^^^ -Timestamps and durations are consistent with the ISO 8601 standard. The resolution of the data is implicit (from duration and number of values), see :ref:`resolutions`. +Timestamps and durations are consistent with the ISO 8601 standard. +The frequency of the data is implicit (from duration and number of values), while the resolution of the data is explicit, see :ref:`frequency_and_resolution`. All timestamps in requests to the API must be timezone-aware. For instance, in the below example, the timezone indication "Z" indicates a zero offset from UTC. @@ -259,19 +260,34 @@ For example, the following message implies that all prognosed values were made 1 Note that, for a horizon indicating a belief 10 minutes after the *start* of each 15-minute interval, the "horizon" would have been "PT5M". This denotes that the prognosed interval has 5 minutes left to be concluded. -.. _resolutions: +.. _frequency_and_resolution: -Resolutions -^^^^^^^^^^^ +Frequency and resolution +^^^^^^^^^^^^^^^^^^^^^^^^ -Specifying a resolution is redundant for POST requests that contain both "values" and a "duration" ― FlexMeasures computes the resolution by dividing the duration by the number of values. +FlexMeasures handles two types of time series, which can be distinguished by defining the following timing properties for events recorded by sensors: -When POSTing data, FlexMeasures checks this computed resolution against the required resolution of the sensors which are posted to. If these can't be matched (through upsampling), an error will occur. +- Frequency: how far apart events occur (a constant duration between event starts) +- Resolution: how long an event lasts (a constant duration between the start and end of an event) -GET requests (such as *getMeterData*) return data in the resolution which the sensor is configured for. -A "resolution" may be specified explicitly to obtain the data in downsampled form, -which can be very beneficial for download speed. The specified resolution needs to be a multiple -of the sensor's resolution, e.g. hourly or daily values if the sensor's resolution is 15 minutes. +.. note:: FlexMeasures runs on Pandas, and follows Pandas terminology accordingly. + The term frequency as used by Pandas is the reciprocal of the `SI quantity for frequency `_. + +1. The first type of time series describes non-instantaneous events such as average hourly wind speed. + For this case, it is commonly assumed that ``frequency == resolution``. + That is, events follow each other sequentially and without delay. + +2. The second type of time series describes instantaneous events (zero resolution) such as temperature at a given time. + For this case, we have ``frequency != resolution``. + +Specifying a frequency and resolution is redundant for POST requests that contain both "values" and a "duration" ― FlexMeasures computes the frequency by dividing the duration by the number of values, and, for sensors that record non-instantaneous events, assumes the resolution of the data is equal to the frequency. + +When POSTing data, FlexMeasures checks this inferred resolution against the required resolution of the sensors that are posted to. +If these can't be matched (through upsampling), an error will occur. + +GET requests (such as *getMeterData*) return data with a frequency either equal to the resolution that the sensor is configured for (for non-instantaneous sensors), or a default frequency fitting (in our opinion) the requested time interval. +A "resolution" may be specified explicitly to obtain the data in downsampled form, which can be very beneficial for download speed. +For non-instantaneous sensors, the specified resolution needs to be a multiple of the sensor's resolution, e.g. hourly or daily values if the sensor's resolution is 15 minutes. .. _sources: From 713dd78dde983ffbf2c98292e63f5d5681670847 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 13:29:37 +0100 Subject: [PATCH 09/38] Update example endpoint Signed-off-by: F.N. Claessen --- documentation/api/notation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index a8481c0c7..d5d05415b 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -285,7 +285,7 @@ Specifying a frequency and resolution is redundant for POST requests that contai When POSTing data, FlexMeasures checks this inferred resolution against the required resolution of the sensors that are posted to. If these can't be matched (through upsampling), an error will occur. -GET requests (such as *getMeterData*) return data with a frequency either equal to the resolution that the sensor is configured for (for non-instantaneous sensors), or a default frequency fitting (in our opinion) the requested time interval. +GET requests (such as */sensors/data*) return data with a frequency either equal to the resolution that the sensor is configured for (for non-instantaneous sensors), or a default frequency befitting (in our opinion) the requested time interval. A "resolution" may be specified explicitly to obtain the data in downsampled form, which can be very beneficial for download speed. For non-instantaneous sensors, the specified resolution needs to be a multiple of the sensor's resolution, e.g. hourly or daily values if the sensor's resolution is 15 minutes. From c852a9edf186435aeec7a2f8802d7661903b5744 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 14:17:35 +0100 Subject: [PATCH 10/38] Add more clarity about how the resolution field is interpreted for instantaneous sensors Signed-off-by: F.N. Claessen --- documentation/api/notation.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index d5d05415b..10a75fcfd 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -288,6 +288,8 @@ If these can't be matched (through upsampling), an error will occur. GET requests (such as */sensors/data*) return data with a frequency either equal to the resolution that the sensor is configured for (for non-instantaneous sensors), or a default frequency befitting (in our opinion) the requested time interval. A "resolution" may be specified explicitly to obtain the data in downsampled form, which can be very beneficial for download speed. For non-instantaneous sensors, the specified resolution needs to be a multiple of the sensor's resolution, e.g. hourly or daily values if the sensor's resolution is 15 minutes. +For instantaneous sensors, the specified resolution is interpreted as a request for data in a specific frequency. +The resolution of the underlying data will remain zero (and the returned message will say so). .. _sources: From a39c87ab83a1c6b4a0b559af88bb7a58a547ca06 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 14:19:35 +0100 Subject: [PATCH 11/38] Make SourceIdField importable and usable for API validation Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/__init__.py | 1 + flexmeasures/data/schemas/sources.py | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/schemas/__init__.py b/flexmeasures/data/schemas/__init__.py index 96cd296f4..2d72e2f1f 100644 --- a/flexmeasures/data/schemas/__init__.py +++ b/flexmeasures/data/schemas/__init__.py @@ -1,4 +1,5 @@ from .assets import LatitudeField, LongitudeField # noqa F401 from .generic_assets import GenericAssetIdField as AssetIdField # noqa F401 from .sensors import SensorIdField # noqa F401 +from .sources import DataSourceIdField as SourceIdField # noqa F401 from .times import AwareDateTimeField, DurationField # noqa F401 diff --git a/flexmeasures/data/schemas/sources.py b/flexmeasures/data/schemas/sources.py index bdcd9a9da..993f5f716 100644 --- a/flexmeasures/data/schemas/sources.py +++ b/flexmeasures/data/schemas/sources.py @@ -1,14 +1,17 @@ -from flask.cli import with_appcontext from marshmallow import fields from flexmeasures.data.models.data_sources import DataSource -from flexmeasures.data.schemas.utils import FMValidationError, MarshmallowClickMixin +from flexmeasures.data.schemas.utils import ( + with_appcontext_if_needed, + FMValidationError, + MarshmallowClickMixin, +) class DataSourceIdField(fields.Int, MarshmallowClickMixin): - """Field that deserializes to a Sensor and serializes back to an integer.""" + """Field that deserializes to a DataSource and serializes back to an integer.""" - @with_appcontext + @with_appcontext_if_needed() def _deserialize(self, value, attr, obj, **kwargs) -> DataSource: """Turn a source id into a DataSource.""" source = DataSource.query.get(value) From a6c84db8c724c28c71a8137da1281252952881a3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 14:21:20 +0100 Subject: [PATCH 12/38] Allow filtering sensor data by source in API Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index ed963050b..6a53ed06a 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -15,7 +15,7 @@ from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.api.common.utils.api_utils import upsample_values from flexmeasures.data.models.planning.utils import initialize_index -from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField +from flexmeasures.data.schemas import AwareDateTimeField, DurationField, SourceIdField from flexmeasures.data.services.time_series import simplify_index from flexmeasures.utils.time_utils import ( decide_resolution, @@ -103,6 +103,7 @@ def check_schema_unit_against_sensor_unit(self, data, **kwargs): class GetSensorDataSchema(SensorDataDescriptionSchema): resolution = DurationField(required=False) + source = SourceIdField(required=False) # Optional field that can be used for extra validation type = fields.Str( @@ -156,6 +157,7 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: end = sensor_data_description["start"] + duration unit = sensor_data_description["unit"] resolution = sensor_data_description.get("resolution") + source = sensor_data_description.get("source") # Post-load configuration of event frequency if resolution is None: @@ -184,6 +186,7 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: event_ends_before=end, horizons_at_least=horizons_at_least, horizons_at_most=horizons_at_most, + source=source, beliefs_before=sensor_data_description.get("prior", None), one_deterministic_belief_per_event=True, resolution=resolution, From 83dfe540ccfa7690a2576167805dc65263f57984 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 14:19:35 +0100 Subject: [PATCH 13/38] Make SourceIdField importable and usable for API validation Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/__init__.py | 1 + flexmeasures/data/schemas/sources.py | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/schemas/__init__.py b/flexmeasures/data/schemas/__init__.py index 96cd296f4..2d72e2f1f 100644 --- a/flexmeasures/data/schemas/__init__.py +++ b/flexmeasures/data/schemas/__init__.py @@ -1,4 +1,5 @@ from .assets import LatitudeField, LongitudeField # noqa F401 from .generic_assets import GenericAssetIdField as AssetIdField # noqa F401 from .sensors import SensorIdField # noqa F401 +from .sources import DataSourceIdField as SourceIdField # noqa F401 from .times import AwareDateTimeField, DurationField # noqa F401 diff --git a/flexmeasures/data/schemas/sources.py b/flexmeasures/data/schemas/sources.py index bdcd9a9da..993f5f716 100644 --- a/flexmeasures/data/schemas/sources.py +++ b/flexmeasures/data/schemas/sources.py @@ -1,14 +1,17 @@ -from flask.cli import with_appcontext from marshmallow import fields from flexmeasures.data.models.data_sources import DataSource -from flexmeasures.data.schemas.utils import FMValidationError, MarshmallowClickMixin +from flexmeasures.data.schemas.utils import ( + with_appcontext_if_needed, + FMValidationError, + MarshmallowClickMixin, +) class DataSourceIdField(fields.Int, MarshmallowClickMixin): - """Field that deserializes to a Sensor and serializes back to an integer.""" + """Field that deserializes to a DataSource and serializes back to an integer.""" - @with_appcontext + @with_appcontext_if_needed() def _deserialize(self, value, attr, obj, **kwargs) -> DataSource: """Turn a source id into a DataSource.""" source = DataSource.query.get(value) From ca7d9a91cd5eab5ed7df2c47e7ffb0517c3b2356 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 14:21:20 +0100 Subject: [PATCH 14/38] Allow filtering sensor data by source in API Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index b44fc6738..bbb7cb45e 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -15,7 +15,7 @@ from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.api.common.utils.api_utils import upsample_values from flexmeasures.data.models.planning.utils import initialize_index -from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField +from flexmeasures.data.schemas import AwareDateTimeField, DurationField, SourceIdField from flexmeasures.data.services.time_series import simplify_index from flexmeasures.utils.time_utils import duration_isoformat, server_now from flexmeasures.utils.unit_utils import ( @@ -99,6 +99,7 @@ def check_schema_unit_against_sensor_unit(self, data, **kwargs): class GetSensorDataSchema(SensorDataDescriptionSchema): resolution = DurationField(required=False) + source = SourceIdField(required=False) # Optional field that can be used for extra validation type = fields.Str( @@ -152,6 +153,7 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: end = sensor_data_description["start"] + duration unit = sensor_data_description["unit"] resolution = sensor_data_description.get("resolution") + source = sensor_data_description.get("source") # Post-load configuration of belief timing against message type horizons_at_least = sensor_data_description.get("horizon", None) @@ -172,6 +174,7 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: event_ends_before=end, horizons_at_least=horizons_at_least, horizons_at_most=horizons_at_most, + source=source, beliefs_before=sensor_data_description.get("prior", None), one_deterministic_belief_per_event=True, resolution=resolution, From d00be86f4bd4cdd548ad671b04e8479a183dd207 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Nov 2022 15:30:29 +0100 Subject: [PATCH 15/38] Show source id in chart tooltip so users have somewhere to find them Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/belief_charts.py | 14 ++++++++++++-- flexmeasures/data/models/charts/defaults.py | 5 +++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index 3d064f9bd..6fb67d681 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -49,7 +49,7 @@ def bar_chart( **event_value_field_definition, **dict(title=f"{capitalize(sensor.sensor_type)}"), }, - FIELD_DEFINITIONS["source_name"], + FIELD_DEFINITIONS["source_name_and_id"], FIELD_DEFINITIONS["source_model"], ], }, @@ -58,6 +58,10 @@ def bar_chart( "calculate": f"datum.event_start + {resolution_in_ms}", "as": "event_end", }, + { + "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'", + "as": "source_name_and_id", + }, ], } for k, v in override_chart_specs.items(): @@ -110,7 +114,7 @@ def chart_for_multiple_sensors( **event_value_field_definition, **dict(title=f"{capitalize(sensor.sensor_type)}"), }, - FIELD_DEFINITIONS["source_name"], + FIELD_DEFINITIONS["source_name_and_id"], FIELD_DEFINITIONS["source_model"], ] line_layer = { @@ -226,6 +230,12 @@ def chart_for_multiple_sensors( chart_specs = dict( description="A vertically concatenated chart showing sensor data.", vconcat=[*sensors_specs], + transform=[ + { + "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'", + "as": "source_name_and_id", + }, + ], spacing=100, bounds="flush", ) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index f1ff22c2a..f4d097f42 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -51,6 +51,11 @@ type="nominal", title="Time and date", ), + "source_name_and_id": dict( + field="source_name_and_id", + type="nominal", + title="Source", + ), } SHADE_LAYER = { "mark": { From f4987dbb82f57cd8fb99621e563c8529fedecb1b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 09:39:10 +0100 Subject: [PATCH 16/38] Add test for /sensors/data [GET] Signed-off-by: F.N. Claessen --- .../api/common/schemas/sensor_data.py | 8 +++--- flexmeasures/api/v3_0/sensors.py | 3 +- flexmeasures/api/v3_0/tests/conftest.py | 24 ++++++++++++++-- .../api/v3_0/tests/test_sensor_data.py | 28 +++++++++++++++++++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index bbb7cb45e..82e8ca185 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -187,18 +187,18 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: df = df.reindex(index) # Convert to desired unit - values: pd.Series = convert_units( # type: ignore + event_values: pd.Series = convert_units( # type: ignore df["event_value"], from_unit=sensor.unit, to_unit=unit, ) - # Convert NaN to null - values = values.where(pd.notnull(values), None) + # Convert NaN (in Series) to None (in list), which JSON dumps as null values + values = [v if not pd.isnull(v) else None for v in event_values.tolist()] # Form the response response = dict( - values=values.tolist(), + values=values, start=datetime_isoformat(start), duration=duration_isoformat(duration), unit=unit, diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 901d63bb2..bab09259a 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -191,7 +191,8 @@ def get_data(self, response: dict): :status 403: INVALID_SENDER :status 422: UNPROCESSABLE_ENTITY """ - return json.dumps(response) + d, s = request_processed() + return dict(**response, **d), s @route("//schedules/trigger", methods=["POST"]) @use_kwargs( diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index dce437b50..b33fb33ac 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -1,10 +1,13 @@ from datetime import timedelta +import pandas as pd import pytest from flask_security import SQLAlchemySessionUserDatastore, hash_password +from flexmeasures import Source from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset -from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.models.time_series import Sensor, TimedBelief +from flexmeasures.data.utils import get_data_source @pytest.fixture(scope="module") @@ -13,7 +16,10 @@ def setup_api_test_data(db, setup_roles_users, setup_generic_assets): Set up data for API v3.0 tests. """ print("Setting up data for API v3.0 tests on %s" % db.engine) - add_gas_sensor(db, setup_roles_users["Test Supplier User"]) + gas_sensor = add_gas_sensor(db, setup_roles_users["Test Supplier User"]) + add_gas_measurements( + db, setup_roles_users["Test Supplier User"].data_source[0], gas_sensor + ) @pytest.fixture(scope="function") @@ -46,7 +52,7 @@ def setup_inactive_user(db, setup_accounts, setup_roles_users): ) -def add_gas_sensor(db, test_supplier_user): +def add_gas_sensor(db, test_supplier_user) -> Sensor: incineration_type = GenericAssetType( name="waste incinerator", ) @@ -67,3 +73,15 @@ def add_gas_sensor(db, test_supplier_user): ) db.session.add(gas_sensor) gas_sensor.owner = test_supplier_user.account + return gas_sensor + + +def add_gas_measurements(db, source: Source, gas_sensor: Sensor): + belief = TimedBelief( + sensor=gas_sensor, + source=source, + event_start=pd.Timestamp("2021-08-02T00:00:00+02:00"), + belief_horizon=timedelta(0), + event_value=91.3, + ) + db.session.add(belief) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 81180dba7..674f9c5ad 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -1,6 +1,10 @@ +from datetime import timedelta +import json + from flask import url_for import pytest +from flexmeasures import Sensor from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor @@ -97,3 +101,27 @@ def test_post_sensor_data_twice(client, setup_api_test_data): print(response.json) assert response.status_code == 403 assert "data represents a replacement" in response.json["message"] + + +def test_get_sensor_data(client, setup_api_test_data): + """Check the /sensors/data endpoint for fetching 1 hour of data of a 10-minute resolution sensor.""" + sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none() + assert sensor.event_resolution == timedelta(minutes=10) + message = { + "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", + "start": "2021-08-02T00:00:00+02:00", + "duration": "PT1H", + "horizon": "PT0H", + "unit": "m³/h", + } + auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") + response = client.get( + url_for("SensorAPI:get_data"), + query_string=message, + headers={"content-type": "application/json", "Authorization": auth_token}, + ) + print("Server responded with:\n%s" % response.json) + assert response.status_code == 200 + values = response.json["values"] + # We expect one data point (from conftest) followed by 5 null values (which are converted to None by .json) + assert all(a == b for a, b in zip(values, [91.3, None, None, None, None, None])) From 764b7cfcdf243819e07ece609b906c9773ba560f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 09:53:03 +0100 Subject: [PATCH 17/38] Test source field, and refactor test setup Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/conftest.py | 9 ++++++--- flexmeasures/api/v3_0/tests/test_sensor_data.py | 8 +++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index b33fb33ac..9d7cf1c78 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -4,14 +4,16 @@ import pytest from flask_security import SQLAlchemySessionUserDatastore, hash_password -from flexmeasures import Source +from flexmeasures import Sensor, Source from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.utils import get_data_source @pytest.fixture(scope="module") -def setup_api_test_data(db, setup_roles_users, setup_generic_assets): +def setup_api_test_data( + db, setup_roles_users, setup_generic_assets +) -> dict[str, Sensor]: """ Set up data for API v3.0 tests. """ @@ -20,6 +22,7 @@ def setup_api_test_data(db, setup_roles_users, setup_generic_assets): add_gas_measurements( db, setup_roles_users["Test Supplier User"].data_source[0], gas_sensor ) + return {gas_sensor.name: gas_sensor} @pytest.fixture(scope="function") @@ -64,7 +67,6 @@ def add_gas_sensor(db, test_supplier_user) -> Sensor: account_id=test_supplier_user.account_id, ) db.session.add(incineration_asset) - db.session.flush() gas_sensor = Sensor( name="some gas sensor", unit="m³/h", @@ -73,6 +75,7 @@ def add_gas_sensor(db, test_supplier_user) -> Sensor: ) db.session.add(gas_sensor) gas_sensor.owner = test_supplier_user.account + db.session.flush() # assign sensor id return gas_sensor diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 674f9c5ad..3f00da02f 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -4,7 +4,7 @@ from flask import url_for import pytest -from flexmeasures import Sensor +from flexmeasures import Sensor, Source, User from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor @@ -103,9 +103,10 @@ def test_post_sensor_data_twice(client, setup_api_test_data): assert "data represents a replacement" in response.json["message"] -def test_get_sensor_data(client, setup_api_test_data): +def test_get_sensor_data(client, setup_api_test_data, setup_roles_users): """Check the /sensors/data endpoint for fetching 1 hour of data of a 10-minute resolution sensor.""" - sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none() + sensor = setup_api_test_data["some gas sensor"] + source = setup_roles_users["Test Supplier User"].data_source[0] assert sensor.event_resolution == timedelta(minutes=10) message = { "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", @@ -113,6 +114,7 @@ def test_get_sensor_data(client, setup_api_test_data): "duration": "PT1H", "horizon": "PT0H", "unit": "m³/h", + "source": source.id, } auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") response = client.get( From d389c4ad70f458e72458c7e25ff89a53cac004a0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 09:54:03 +0100 Subject: [PATCH 18/38] Test resolution field Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_sensor_data.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 3f00da02f..602c02df5 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -115,6 +115,7 @@ def test_get_sensor_data(client, setup_api_test_data, setup_roles_users): "horizon": "PT0H", "unit": "m³/h", "source": source.id, + "resolution": "PT20M", } auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") response = client.get( @@ -125,5 +126,5 @@ def test_get_sensor_data(client, setup_api_test_data, setup_roles_users): print("Server responded with:\n%s" % response.json) assert response.status_code == 200 values = response.json["values"] - # We expect one data point (from conftest) followed by 5 null values (which are converted to None by .json) - assert all(a == b for a, b in zip(values, [91.3, None, None, None, None, None])) + # We expect one data point (from conftest) followed by 2 null values (which are converted to None by .json) + assert all(a == b for a, b in zip(values, [91.3, None, None])) From aa2eb22fc2d07fc9727510a8da3ad8ee7b534fed Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 10:02:31 +0100 Subject: [PATCH 19/38] Test averaging event values when using resolution field Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/conftest.py | 24 ++++++++++++------- .../api/v3_0/tests/test_sensor_data.py | 7 +++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index 9d7cf1c78..72a442275 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -80,11 +80,19 @@ def add_gas_sensor(db, test_supplier_user) -> Sensor: def add_gas_measurements(db, source: Source, gas_sensor: Sensor): - belief = TimedBelief( - sensor=gas_sensor, - source=source, - event_start=pd.Timestamp("2021-08-02T00:00:00+02:00"), - belief_horizon=timedelta(0), - event_value=91.3, - ) - db.session.add(belief) + event_starts = [ + pd.Timestamp("2021-08-02T00:00:00+02:00") + timedelta(minutes=minutes) + for minutes in range(0, 30, 10) + ] + event_values = [91.3, 91.7, 92.1] + beliefs = [ + TimedBelief( + sensor=gas_sensor, + source=source, + event_start=event_start, + belief_horizon=timedelta(0), + event_value=event_value, + ) + for event_start, event_value in zip(event_starts, event_values) + ] + db.session.add_all(beliefs) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 602c02df5..25aee1bb0 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -111,7 +111,7 @@ def test_get_sensor_data(client, setup_api_test_data, setup_roles_users): message = { "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", "start": "2021-08-02T00:00:00+02:00", - "duration": "PT1H", + "duration": "PT1H20M", "horizon": "PT0H", "unit": "m³/h", "source": source.id, @@ -126,5 +126,6 @@ def test_get_sensor_data(client, setup_api_test_data, setup_roles_users): print("Server responded with:\n%s" % response.json) assert response.status_code == 200 values = response.json["values"] - # We expect one data point (from conftest) followed by 2 null values (which are converted to None by .json) - assert all(a == b for a, b in zip(values, [91.3, None, None])) + # We expect two data point (from conftest) followed by 2 null values (which are converted to None by .json) + # The first data point averages 91.3 and 91.7, and the second data point averages 92.1 and None. + assert all(a == b for a, b in zip(values, [91.5, 92.1, None, None])) From d89c22c39feb55cba885a00c69867d9f03cd295d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 10:21:50 +0100 Subject: [PATCH 20/38] Add type annotations, flake8 Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 1 - flexmeasures/api/v3_0/tests/conftest.py | 3 +-- flexmeasures/api/v3_0/tests/test_sensor_data.py | 9 ++++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index bab09259a..f8f825264 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -1,5 +1,4 @@ from datetime import datetime, timedelta -import json from typing import List, Optional from flask import current_app diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index 72a442275..e68de8dfe 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -6,8 +6,7 @@ from flexmeasures import Sensor, Source from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset -from flexmeasures.data.models.time_series import Sensor, TimedBelief -from flexmeasures.data.utils import get_data_source +from flexmeasures.data.models.time_series import TimedBelief @pytest.fixture(scope="module") diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 25aee1bb0..80d98ebc9 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -1,5 +1,6 @@ +from __future__ import annotations + from datetime import timedelta -import json from flask import url_for import pytest @@ -103,10 +104,12 @@ def test_post_sensor_data_twice(client, setup_api_test_data): assert "data represents a replacement" in response.json["message"] -def test_get_sensor_data(client, setup_api_test_data, setup_roles_users): +def test_get_sensor_data( + client, setup_api_test_data: dict[str, Sensor], setup_roles_users: dict[str, User] +): """Check the /sensors/data endpoint for fetching 1 hour of data of a 10-minute resolution sensor.""" sensor = setup_api_test_data["some gas sensor"] - source = setup_roles_users["Test Supplier User"].data_source[0] + source: Source = setup_roles_users["Test Supplier User"].data_source[0] assert sensor.event_resolution == timedelta(minutes=10) message = { "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", From 62488c367ed5ec9181e92a7e0118c394509a3393 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 10:31:14 +0100 Subject: [PATCH 21/38] Remove double line breaks Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 0bf6aa869..d29d14bf2 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -10,7 +10,6 @@ v3.0-3 | 2022-08-28 - Introduced ``consumption_price_sensor``, ``production_price_sensor`` and ``inflexible_device_sensors`` fields to `/sensors//schedules/trigger` (POST) - v3.0-2 | 2022-07-08 """"""""""""""""""" @@ -77,8 +76,6 @@ v2.0-3 | 2021-06-07 - Updated all entity addresses in documentation according to the fm0 scheme, preserving backwards compatibility. - Introduced the fm1 scheme for entity addresses for connections, markets, weather sensors and sensors. - - v2.0-2 | 2021-04-02 """"""""""""""""""" @@ -103,7 +100,6 @@ v2.0-0 | 2020-11-14 - Added REST endpoints for managing assets: `/assets/` (GET, POST) and `/asset/` (GET, PATCH, DELETE). - v1.3-11 | 2022-01-05 """""""""""""""""""" @@ -178,7 +174,6 @@ v1.3-2 | 2020-03-11 - Fixed example entity addresses in simulation section - v1.3-1 | 2020-02-08 """"""""""""""""""" @@ -299,7 +294,6 @@ v1.0-1 | 2018-07-10 - Added sections listing all endpoints per version - Documentation includes specifications of **all** supported API versions (supported versions have a registered Flask blueprint) - v1.0-0 | 2018-07-10 """"""""""""""""""" From eb8a951402f3952ce16b2d48af26ea6342812a7b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 10:32:53 +0100 Subject: [PATCH 22/38] API changelog entry for introduction of 'source' field Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index d29d14bf2..e685b223c 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -5,6 +5,11 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace. +v3.0-4 | 2022-11-29 +""""""""""""""""""" + +- Introduced the ``source`` field to `/sensors/data` (GET) to obtain data for a given source (ID). + v3.0-3 | 2022-08-28 """"""""""""""""""" From 12537d35c80195f1b10a17ccdc41d9fc2464a7d2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 10:33:04 +0100 Subject: [PATCH 23/38] Missing punctuation Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index e685b223c..af6fbdb32 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -13,7 +13,7 @@ v3.0-4 | 2022-11-29 v3.0-3 | 2022-08-28 """"""""""""""""""" -- Introduced ``consumption_price_sensor``, ``production_price_sensor`` and ``inflexible_device_sensors`` fields to `/sensors//schedules/trigger` (POST) +- Introduced ``consumption_price_sensor``, ``production_price_sensor`` and ``inflexible_device_sensors`` fields to `/sensors//schedules/trigger` (POST). v3.0-2 | 2022-07-08 """"""""""""""""""" From 5cd066c77bf1f1e3bd49a65c6c69616abf82db97 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 10:34:38 +0100 Subject: [PATCH 24/38] API changelog entry for return message fix Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index af6fbdb32..7ce7f86f7 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -9,6 +9,7 @@ v3.0-4 | 2022-11-29 """"""""""""""""""" - Introduced the ``source`` field to `/sensors/data` (GET) to obtain data for a given source (ID). +- Fixed the JSON wrapping of the return message for `/sensors/data` (GET). v3.0-3 | 2022-08-28 """"""""""""""""""" From 30f5dbbac0c6275e158aadc5978511de2f79b3b7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 10:45:50 +0100 Subject: [PATCH 25/38] Add missing API documentation for optional fields Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index f8f825264..aeff59aac 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -181,6 +181,13 @@ def get_data(self, response: dict): The unit has to be convertible from the sensor's unit. + **Optional fields** + + - "resolution" (see :ref:`resolutions`) + - "horizon" (see :ref:`beliefs`) + - "prior" (see :ref:`beliefs`) + - "source" (see :ref:`sources`) + :reqheader Authorization: The authentication token :reqheader Content-Type: application/json :resheader Content-Type: application/json From d02dc94c6703e8991959115ae2d69c14b1dbf40c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 11:00:28 +0100 Subject: [PATCH 26/38] Update API documentation section on sources Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 3 +++ documentation/api/notation.rst | 11 ++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 7ce7f86f7..463394e0c 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -10,6 +10,9 @@ v3.0-4 | 2022-11-29 - Introduced the ``source`` field to `/sensors/data` (GET) to obtain data for a given source (ID). - Fixed the JSON wrapping of the return message for `/sensors/data` (GET). +- Changed the Notation section: + + - Rewrote the section on filtering by source (ID) with a deprecation notice on filtering by account role and user ID. v3.0-3 | 2022-08-28 """"""""""""""""""" diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index e989f1757..e96519d86 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -287,17 +287,18 @@ of the sensor's resolution, e.g. hourly or daily values if the sensor's resoluti Sources ------- -Requests for data may limit the data selection by specifying a source, for example, a specific user. -Account roles are also valid source selectors. -For example, to obtain data originating from either a meter data company or user 42, include the following: +Requests for data may filter by source. FlexMeasures keeps track of the data source (the data's author, for example, a user, forecaster or scheduler belonging to a given organisation) of time series data. +For example, to obtain data originating from data source 42, include the following: .. code-block:: json { - "sources": ["MDC", "42"], + "source": 42, } -Here, "MDC" is the name of the account role for meter data companies. +Data source IDs can be found by hovering over data in charts. + +.. note:: Older API version (< 3) accepted user IDs (integers), account roles (strings) and lists thereof, instead of data source IDs (integers). .. _units: From eeb2e85fb2709dedf72271bd139797e25acdb923 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 11:49:04 +0100 Subject: [PATCH 27/38] Move test to module using fresh db for each test to avoid session flush Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/conftest.py | 17 +++++++- .../api/v3_0/tests/test_sensor_data.py | 35 ---------------- .../v3_0/tests/test_sensor_data_fresh_db.py | 40 ++++++++++++++++++- 3 files changed, 55 insertions(+), 37 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index e68de8dfe..3e06b1f71 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -34,7 +34,22 @@ def setup_api_fresh_test_data( print("Setting up fresh data for API 3.0 tests on %s" % fresh_db.engine) for sensor in Sensor.query.all(): fresh_db.delete(sensor) - add_gas_sensor(fresh_db, setup_roles_users_fresh_db["Test Supplier User"]) + gas_sensor = add_gas_sensor( + fresh_db, setup_roles_users_fresh_db["Test Supplier User"] + ) + return {gas_sensor.name: gas_sensor} + + +@pytest.fixture(scope="function") +def setup_api_fresh_gas_measurements( + fresh_db, setup_api_fresh_test_data, setup_roles_users_fresh_db +): + """Set up some measurements for the gas sensor.""" + add_gas_measurements( + fresh_db, + setup_roles_users_fresh_db["Test Supplier User"].data_source[0], + setup_api_fresh_test_data["some gas sensor"], + ) @pytest.fixture(scope="module") diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 80d98ebc9..81180dba7 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -1,11 +1,6 @@ -from __future__ import annotations - -from datetime import timedelta - from flask import url_for import pytest -from flexmeasures import Sensor, Source, User from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor @@ -102,33 +97,3 @@ def test_post_sensor_data_twice(client, setup_api_test_data): print(response.json) assert response.status_code == 403 assert "data represents a replacement" in response.json["message"] - - -def test_get_sensor_data( - client, setup_api_test_data: dict[str, Sensor], setup_roles_users: dict[str, User] -): - """Check the /sensors/data endpoint for fetching 1 hour of data of a 10-minute resolution sensor.""" - sensor = setup_api_test_data["some gas sensor"] - source: Source = setup_roles_users["Test Supplier User"].data_source[0] - assert sensor.event_resolution == timedelta(minutes=10) - message = { - "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", - "start": "2021-08-02T00:00:00+02:00", - "duration": "PT1H20M", - "horizon": "PT0H", - "unit": "m³/h", - "source": source.id, - "resolution": "PT20M", - } - auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") - response = client.get( - url_for("SensorAPI:get_data"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % response.json) - assert response.status_code == 200 - values = response.json["values"] - # We expect two data point (from conftest) followed by 2 null values (which are converted to None by .json) - # The first data point averages 91.3 and 91.7, and the second data point averages 92.1 and None. - assert all(a == b for a, b in zip(values, [91.5, 92.1, None, None])) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index d55923d8d..cd003af95 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -1,9 +1,14 @@ +from __future__ import annotations + +from datetime import timedelta + import pytest from flask import url_for +from flexmeasures import Sensor, Source, User from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor -from flexmeasures.data.models.time_series import TimedBelief, Sensor +from flexmeasures.data.models.time_series import TimedBelief @pytest.mark.parametrize( @@ -50,3 +55,36 @@ def test_post_sensor_data( assert len(beliefs) == expected_num_values # check that values are scaled to the sensor unit correctly assert pytest.approx(beliefs[0].event_value - expected_value) == 0 + + +def test_get_sensor_data( + client, + setup_api_fresh_test_data: dict[str, Sensor], + setup_api_fresh_gas_measurements, + setup_roles_users_fresh_db: dict[str, User], +): + """Check the /sensors/data endpoint for fetching 1 hour of data of a 10-minute resolution sensor.""" + sensor = setup_api_fresh_test_data["some gas sensor"] + source: Source = setup_roles_users_fresh_db["Test Supplier User"].data_source[0] + assert sensor.event_resolution == timedelta(minutes=10) + message = { + "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", + "start": "2021-08-02T00:00:00+02:00", + "duration": "PT1H20M", + "horizon": "PT0H", + "unit": "m³/h", + "source": source.id, + "resolution": "PT20M", + } + auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") + response = client.get( + url_for("SensorAPI:get_data"), + query_string=message, + headers={"content-type": "application/json", "Authorization": auth_token}, + ) + print("Server responded with:\n%s" % response.json) + assert response.status_code == 200 + values = response.json["values"] + # We expect two data point (from conftest) followed by 2 null values (which are converted to None by .json) + # The first data point averages 91.3 and 91.7, and the second data point averages 92.1 and None. + assert all(a == b for a, b in zip(values, [91.5, 92.1, None, None])) From a5a150abde57b49d27da8235f7937fa4f0a6770c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 12:17:59 +0100 Subject: [PATCH 28/38] Faster conversion of NaN values (we were just missing the dtype conversion) Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 82e8ca185..d38099e4b 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -187,18 +187,18 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: df = df.reindex(index) # Convert to desired unit - event_values: pd.Series = convert_units( # type: ignore + values: pd.Series = convert_units( # type: ignore df["event_value"], from_unit=sensor.unit, to_unit=unit, ) # Convert NaN (in Series) to None (in list), which JSON dumps as null values - values = [v if not pd.isnull(v) else None for v in event_values.tolist()] + values = values.astype(object).where(pd.notnull(values), None) # Form the response response = dict( - values=values, + values=values.tolist(), start=datetime_isoformat(start), duration=duration_isoformat(duration), unit=unit, From a05034265c9c70e7dbaf971e5e4a61e3344145da Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 12:19:26 +0100 Subject: [PATCH 29/38] Update inline comment Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index d38099e4b..36cbe5b0b 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -193,7 +193,7 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: to_unit=unit, ) - # Convert NaN (in Series) to None (in list), which JSON dumps as null values + # Convert NaN to None, which JSON dumps as null values values = values.astype(object).where(pd.notnull(values), None) # Form the response From 51139b5e2b5ef867ca0c05eb4023fec13ab140d8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 13:09:31 +0100 Subject: [PATCH 30/38] Update documentation refs Signed-off-by: F.N. Claessen --- documentation/tut/posting_data.rst | 2 +- flexmeasures/api/v1/routes.py | 2 +- flexmeasures/api/v1_1/routes.py | 2 +- flexmeasures/api/v3_0/sensors.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index 809f0d6d6..36dbedf77 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -21,7 +21,7 @@ Prerequisites - FlexMeasures needs some structural meta data for data to be understood. For example, for adding weather data we need to define a weather sensor, and what kind of weather sensors there are. You also need a user account. If you host FlexMeasures yourself, you need to add this info first. Head over to :ref:`getting_started`, where these steps are covered, study our :ref:`cli` or look into plugins which do this like `flexmeasures-entsoe `_ or `flexmeasures-openweathermap `_. - You should be familiar with where to find your API endpoints (see :ref:`api_versions`) and how to authenticate against the API (see :ref:`api_auth`). -.. note:: For deeper explanations of the data and the meta fields we'll send here, You can always read the :ref:`api_introduction`, to the FlexMeasures API, e.g. :ref:`signs`, :ref:`resolutions`, :ref:`prognoses` and :ref:`units`. +.. note:: For deeper explanations of the data and the meta fields we'll send here, You can always read the :ref:`api_introduction`, to the FlexMeasures API, e.g. :ref:`signs`, :ref:`frequency_and_resolution`, :ref:`prognoses` and :ref:`units`. .. note:: To address assets and sensors, these tutorials assume entity addresses valid in the namespace ``fm1``. See :ref:`api_introduction` for more explanations. diff --git a/flexmeasures/api/v1/routes.py b/flexmeasures/api/v1/routes.py index 5fe5c3cd5..6727ccf42 100644 --- a/flexmeasures/api/v1/routes.py +++ b/flexmeasures/api/v1/routes.py @@ -38,7 +38,7 @@ def get_meter_data(): **Optional fields** - - "resolution" (see :ref:`resolutions`) + - "resolution" (see :ref:`frequency_and_resolution`) - "horizon" (see :ref:`beliefs`) - "prior" (see :ref:`beliefs`) - "source" (see :ref:`sources`) diff --git a/flexmeasures/api/v1_1/routes.py b/flexmeasures/api/v1_1/routes.py index b79fb6701..0c5b27dac 100644 --- a/flexmeasures/api/v1_1/routes.py +++ b/flexmeasures/api/v1_1/routes.py @@ -275,7 +275,7 @@ def get_prognosis(): **Optional fields** - - "resolution" (see :ref:`resolutions`) + - "resolution" (see :ref:`frequency_and_resolution`) - "horizon" (see :ref:`beliefs`) - "prior" (see :ref:`beliefs`) - "source" (see :ref:`sources`) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index aeff59aac..090605c8a 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -183,7 +183,7 @@ def get_data(self, response: dict): **Optional fields** - - "resolution" (see :ref:`resolutions`) + - "resolution" (see :ref:`frequency_and_resolution`) - "horizon" (see :ref:`beliefs`) - "prior" (see :ref:`beliefs`) - "source" (see :ref:`sources`) From 3224d530c5177b76dc68e742e8662e301677f037 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 14:00:28 +0100 Subject: [PATCH 31/38] Add test for fetching instantaneous sensor data Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/tests/conftest.py | 6 +- flexmeasures/api/v3_0/tests/conftest.py | 57 +++++++++++++++---- .../v3_0/tests/test_sensor_data_fresh_db.py | 35 +++++++++++- 3 files changed, 84 insertions(+), 14 deletions(-) diff --git a/flexmeasures/api/dev/tests/conftest.py b/flexmeasures/api/dev/tests/conftest.py index 6887661e2..9b98647af 100644 --- a/flexmeasures/api/dev/tests/conftest.py +++ b/flexmeasures/api/dev/tests/conftest.py @@ -1,6 +1,6 @@ import pytest -from flexmeasures.api.v3_0.tests.conftest import add_gas_sensor +from flexmeasures.api.v3_0.tests.conftest import add_incineration_line from flexmeasures.data.models.time_series import Sensor @@ -10,7 +10,7 @@ def setup_api_test_data(db, setup_roles_users, setup_generic_assets): Set up data for API dev tests. """ print("Setting up data for API dev tests on %s" % db.engine) - add_gas_sensor(db, setup_roles_users["Test Supplier User"]) + add_incineration_line(db, setup_roles_users["Test Supplier User"]) @pytest.fixture(scope="function") @@ -23,4 +23,4 @@ def setup_api_fresh_test_data( print("Setting up fresh data for API dev tests on %s" % fresh_db.engine) for sensor in Sensor.query.all(): fresh_db.delete(sensor) - add_gas_sensor(fresh_db, setup_roles_users_fresh_db["Test Supplier User"]) + add_incineration_line(fresh_db, setup_roles_users_fresh_db["Test Supplier User"]) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index 3e06b1f71..d287c69f4 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -17,11 +17,11 @@ def setup_api_test_data( Set up data for API v3.0 tests. """ print("Setting up data for API v3.0 tests on %s" % db.engine) - gas_sensor = add_gas_sensor(db, setup_roles_users["Test Supplier User"]) + sensors = add_incineration_line(db, setup_roles_users["Test Supplier User"]) add_gas_measurements( - db, setup_roles_users["Test Supplier User"].data_source[0], gas_sensor + db, setup_roles_users["Test Supplier User"].data_source[0], sensors["some gas sensor"] ) - return {gas_sensor.name: gas_sensor} + return sensors @pytest.fixture(scope="function") @@ -34,10 +34,10 @@ def setup_api_fresh_test_data( print("Setting up fresh data for API 3.0 tests on %s" % fresh_db.engine) for sensor in Sensor.query.all(): fresh_db.delete(sensor) - gas_sensor = add_gas_sensor( + sensors = add_incineration_line( fresh_db, setup_roles_users_fresh_db["Test Supplier User"] ) - return {gas_sensor.name: gas_sensor} + return sensors @pytest.fixture(scope="function") @@ -52,6 +52,17 @@ def setup_api_fresh_gas_measurements( ) +@pytest.fixture(scope="function") +def setup_api_fresh_temperature_measurements( + fresh_db, setup_api_fresh_test_data, setup_roles_users_fresh_db +): + """Set up some measurements for the gas sensor.""" + add_temperature_measurements( + fresh_db, + setup_roles_users_fresh_db["Test Supplier User"].data_source[0], + setup_api_fresh_test_data["some temperature sensor"], + ) + @pytest.fixture(scope="module") def setup_inactive_user(db, setup_accounts, setup_roles_users): """ @@ -69,7 +80,7 @@ def setup_inactive_user(db, setup_accounts, setup_roles_users): ) -def add_gas_sensor(db, test_supplier_user) -> Sensor: +def add_incineration_line(db, test_supplier_user) -> dict[str, Sensor]: incineration_type = GenericAssetType( name="waste incinerator", ) @@ -89,11 +100,18 @@ def add_gas_sensor(db, test_supplier_user) -> Sensor: ) db.session.add(gas_sensor) gas_sensor.owner = test_supplier_user.account - db.session.flush() # assign sensor id - return gas_sensor + temperature_sensor = Sensor( + name="some temperature sensor", + unit="°C", + event_resolution=timedelta(0), + generic_asset=incineration_asset, + ) + db.session.add(temperature_sensor) + db.session.flush() # assign sensor ids + return {gas_sensor.name: gas_sensor, temperature_sensor.name: temperature_sensor} -def add_gas_measurements(db, source: Source, gas_sensor: Sensor): +def add_gas_measurements(db, source: Source, sensor: Sensor): event_starts = [ pd.Timestamp("2021-08-02T00:00:00+02:00") + timedelta(minutes=minutes) for minutes in range(0, 30, 10) @@ -101,7 +119,26 @@ def add_gas_measurements(db, source: Source, gas_sensor: Sensor): event_values = [91.3, 91.7, 92.1] beliefs = [ TimedBelief( - sensor=gas_sensor, + sensor=sensor, + source=source, + event_start=event_start, + belief_horizon=timedelta(0), + event_value=event_value, + ) + for event_start, event_value in zip(event_starts, event_values) + ] + db.session.add_all(beliefs) + + +def add_temperature_measurements(db, source: Source, sensor: Sensor): + event_starts = [ + pd.Timestamp("2021-08-02T00:00:00+02:00") + timedelta(minutes=minutes) + for minutes in range(0, 30, 10) + ] + event_values = [815, 817, 818] + beliefs = [ + TimedBelief( + sensor=sensor, source=source, event_start=event_start, belief_horizon=timedelta(0), diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index cd003af95..a0c67890a 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -86,5 +86,38 @@ def test_get_sensor_data( assert response.status_code == 200 values = response.json["values"] # We expect two data point (from conftest) followed by 2 null values (which are converted to None by .json) - # The first data point averages 91.3 and 91.7, and the second data point averages 92.1 and None. + # The first data point averages [91.3, 91.7], and the second data point averages [92.1, None]. assert all(a == b for a, b in zip(values, [91.5, 92.1, None, None])) + + +def test_get_instantaneous_sensor_data( + client, + setup_api_fresh_test_data: dict[str, Sensor], + setup_api_fresh_temperature_measurements, + setup_roles_users_fresh_db: dict[str, User], +): + """Check the /sensors/data endpoint for fetching 1 hour of data of an instantaneous sensor.""" + sensor = setup_api_fresh_test_data["some temperature sensor"] + source: Source = setup_roles_users_fresh_db["Test Supplier User"].data_source[0] + assert sensor.event_resolution == timedelta(minutes=0) + message = { + "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", + "start": "2021-08-02T00:00:00+02:00", + "duration": "PT1H20M", + "horizon": "PT0H", + "unit": "°C", + "source": source.id, + "resolution": "PT20M", + } + auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") + response = client.get( + url_for("SensorAPI:get_data"), + query_string=message, + headers={"content-type": "application/json", "Authorization": auth_token}, + ) + print("Server responded with:\n%s" % response.json) + assert response.status_code == 200 + values = response.json["values"] + # We expect two data point (from conftest) followed by 2 null values (which are converted to None by .json) + # The first data point is 815 is the first of [815, 817], and the second data point is the first of [818, None]. + assert all(a == b for a, b in zip(values, [815, 818, None, None])) From a5a81b5c1503698e4c7e1f5dd05d01bc7d32b5c9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 14:01:58 +0100 Subject: [PATCH 32/38] Sensor ownership is determined via asset ownership Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index d287c69f4..ea74fbc4d 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -99,7 +99,6 @@ def add_incineration_line(db, test_supplier_user) -> dict[str, Sensor]: generic_asset=incineration_asset, ) db.session.add(gas_sensor) - gas_sensor.owner = test_supplier_user.account temperature_sensor = Sensor( name="some temperature sensor", unit="°C", From 0f6e4d639db884c5f384021fcae4b99c468317a9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 14:02:58 +0100 Subject: [PATCH 33/38] Remove redundant flush Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index ea74fbc4d..efcd623d5 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -85,11 +85,10 @@ def add_incineration_line(db, test_supplier_user) -> dict[str, Sensor]: name="waste incinerator", ) db.session.add(incineration_type) - db.session.flush() incineration_asset = GenericAsset( name="incineration line", generic_asset_type=incineration_type, - account_id=test_supplier_user.account_id, + owner=test_supplier_user.account, ) db.session.add(incineration_asset) gas_sensor = Sensor( From 03470ac9c86efabeb8584dcaa0912ffeee0b1fdb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Nov 2022 14:06:22 +0100 Subject: [PATCH 34/38] black Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index efcd623d5..d0e2ede85 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -19,7 +19,9 @@ def setup_api_test_data( print("Setting up data for API v3.0 tests on %s" % db.engine) sensors = add_incineration_line(db, setup_roles_users["Test Supplier User"]) add_gas_measurements( - db, setup_roles_users["Test Supplier User"].data_source[0], sensors["some gas sensor"] + db, + setup_roles_users["Test Supplier User"].data_source[0], + sensors["some gas sensor"], ) return sensors @@ -63,6 +65,7 @@ def setup_api_fresh_temperature_measurements( setup_api_fresh_test_data["some temperature sensor"], ) + @pytest.fixture(scope="module") def setup_inactive_user(db, setup_accounts, setup_roles_users): """ From 2a9e3f59b7cd3012fc16e59bb656e40e624ccb1b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 2 Dec 2022 11:57:25 +0100 Subject: [PATCH 35/38] Set up initial measurements together with the sensors Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/conftest.py | 33 ++++--------------- .../v3_0/tests/test_sensor_data_fresh_db.py | 14 ++++---- 2 files changed, 15 insertions(+), 32 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index 4d8e32cf7..be9ddd120 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -37,30 +37,6 @@ def setup_api_fresh_test_data( return sensors -@pytest.fixture(scope="function") -def setup_api_fresh_gas_measurements( - fresh_db, setup_api_fresh_test_data, setup_roles_users_fresh_db -): - """Set up some measurements for the gas sensor.""" - add_gas_measurements( - fresh_db, - setup_roles_users_fresh_db["Test Supplier User"].data_source[0], - setup_api_fresh_test_data["some gas sensor"], - ) - - -@pytest.fixture(scope="function") -def setup_api_fresh_temperature_measurements( - fresh_db, setup_api_fresh_test_data, setup_roles_users_fresh_db -): - """Set up some measurements for the temperature sensor.""" - add_temperature_measurements( - fresh_db, - setup_roles_users_fresh_db["Test Supplier User"].data_source[0], - setup_api_fresh_test_data["some temperature sensor"], - ) - - @pytest.fixture(scope="module") def setup_inactive_user(db, setup_accounts, setup_roles_users): """ @@ -96,6 +72,7 @@ def add_incineration_line(db, test_supplier_user) -> dict[str, Sensor]: generic_asset=incineration_asset, ) db.session.add(gas_sensor) + add_gas_measurements(db, test_supplier_user.data_source[0], gas_sensor) temperature_sensor = Sensor( name="some temperature sensor", unit="°C", @@ -103,13 +80,17 @@ def add_incineration_line(db, test_supplier_user) -> dict[str, Sensor]: generic_asset=incineration_asset, ) db.session.add(temperature_sensor) + add_temperature_measurements( + db, test_supplier_user.data_source[0], temperature_sensor + ) + db.session.flush() # assign sensor ids return {gas_sensor.name: gas_sensor, temperature_sensor.name: temperature_sensor} def add_gas_measurements(db, source: Source, sensor: Sensor): event_starts = [ - pd.Timestamp("2021-08-02T00:00:00+02:00") + timedelta(minutes=minutes) + pd.Timestamp("2021-05-02T00:00:00+02:00") + timedelta(minutes=minutes) for minutes in range(0, 30, 10) ] event_values = [91.3, 91.7, 92.1] @@ -128,7 +109,7 @@ def add_gas_measurements(db, source: Source, sensor: Sensor): def add_temperature_measurements(db, source: Source, sensor: Sensor): event_starts = [ - pd.Timestamp("2021-08-02T00:00:00+02:00") + timedelta(minutes=minutes) + pd.Timestamp("2021-05-02T00:00:00+02:00") + timedelta(minutes=minutes) for minutes in range(0, 30, 10) ] event_values = [815, 817, 818] diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index 8bf4144ac..1ae9d3bbc 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -38,7 +38,11 @@ def test_post_sensor_data( num_values=num_values, unit=unit ) sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none() - beliefs_before = TimedBelief.query.filter(TimedBelief.sensor_id == sensor.id).all() + filters = ( + TimedBelief.sensor_id == sensor.id, + TimedBelief.event_start >= post_data["start"], + ) + beliefs_before = TimedBelief.query.filter(*filters).all() print(f"BELIEFS BEFORE: {beliefs_before}") assert len(beliefs_before) == 0 @@ -50,7 +54,7 @@ def test_post_sensor_data( ) print(response.json) assert response.status_code == 200 - beliefs = TimedBelief.query.filter(TimedBelief.sensor_id == sensor.id).all() + beliefs = TimedBelief.query.filter(*filters).all() print(f"BELIEFS AFTER: {beliefs}") assert len(beliefs) == expected_num_values # check that values are scaled to the sensor unit correctly @@ -60,7 +64,6 @@ def test_post_sensor_data( def test_get_sensor_data( client, setup_api_fresh_test_data: dict[str, Sensor], - setup_api_fresh_gas_measurements, setup_roles_users_fresh_db: dict[str, User], ): """Check the /sensors/data endpoint for fetching 1 hour of data of a 10-minute resolution sensor.""" @@ -69,7 +72,7 @@ def test_get_sensor_data( assert sensor.event_resolution == timedelta(minutes=10) message = { "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", - "start": "2021-08-02T00:00:00+02:00", + "start": "2021-05-02T00:00:00+02:00", "duration": "PT1H20M", "horizon": "PT0H", "unit": "m³/h", @@ -93,7 +96,6 @@ def test_get_sensor_data( def test_get_instantaneous_sensor_data( client, setup_api_fresh_test_data: dict[str, Sensor], - setup_api_fresh_temperature_measurements, setup_roles_users_fresh_db: dict[str, User], ): """Check the /sensors/data endpoint for fetching 1 hour of data of an instantaneous sensor.""" @@ -102,7 +104,7 @@ def test_get_instantaneous_sensor_data( assert sensor.event_resolution == timedelta(minutes=0) message = { "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", - "start": "2021-08-02T00:00:00+02:00", + "start": "2021-05-02T00:00:00+02:00", "duration": "PT1H20M", "horizon": "PT0H", "unit": "°C", From 61866f165e1826eed564c6eda7633e1a2595b452 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 2 Dec 2022 12:03:22 +0100 Subject: [PATCH 36/38] Changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 46dcac8b1..ffe2657a7 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,6 +17,7 @@ New features * The CLI command ``flexmeasures show beliefs`` supports showing beliefs data in a custom resolution and/or timezone, and also saving the shown beliefs data to a CSV file [see `PR #519 `_] * Improved import of time series data from CSV file: 1) drop duplicate records with warning, 2) allow configuring which column contains explicit recording times for each data point (use case: import forecasts) [see `PR #501 `_], 3) localize timezone naive data, 4) support reading in datetime and timedelta values, 5) remove rows with NaN values, and 6) filter by values in specific columns [see `PR #521 `_] * Filter data by source in the API endpoint `/sensors/data` (GET) [see `PR #543 `_] +* New resampling functionality for instantaneous sensor data: 1) ``flexmeasures show beliefs`` can now handle showing (and saving) instantaneous sensor data and non-instaneous sensor data together, and 2) the API endpoint `/sensors/data` (GET) now allows fetching instantaneous sensor data in a custom frequency, by using the "resolution" field [see `PR #542 `_] Bugfixes ----------- From 16e056ea92a304f198d977da068c8510bf4b44d6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 2 Dec 2022 12:05:01 +0100 Subject: [PATCH 37/38] Typo Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index ffe2657a7..ef70dad40 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,7 +17,7 @@ New features * The CLI command ``flexmeasures show beliefs`` supports showing beliefs data in a custom resolution and/or timezone, and also saving the shown beliefs data to a CSV file [see `PR #519 `_] * Improved import of time series data from CSV file: 1) drop duplicate records with warning, 2) allow configuring which column contains explicit recording times for each data point (use case: import forecasts) [see `PR #501 `_], 3) localize timezone naive data, 4) support reading in datetime and timedelta values, 5) remove rows with NaN values, and 6) filter by values in specific columns [see `PR #521 `_] * Filter data by source in the API endpoint `/sensors/data` (GET) [see `PR #543 `_] -* New resampling functionality for instantaneous sensor data: 1) ``flexmeasures show beliefs`` can now handle showing (and saving) instantaneous sensor data and non-instaneous sensor data together, and 2) the API endpoint `/sensors/data` (GET) now allows fetching instantaneous sensor data in a custom frequency, by using the "resolution" field [see `PR #542 `_] +* New resampling functionality for instantaneous sensor data: 1) ``flexmeasures show beliefs`` can now handle showing (and saving) instantaneous sensor data and non-instantaneous sensor data together, and 2) the API endpoint `/sensors/data` (GET) now allows fetching instantaneous sensor data in a custom frequency, by using the "resolution" field [see `PR #542 `_] Bugfixes ----------- From 9c5c6f100429345f9821395b7102d8f0fbfd9d7b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 12 Dec 2022 09:56:39 +0100 Subject: [PATCH 38/38] Correct sentence Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index 1ae9d3bbc..c5d26833b 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -121,5 +121,5 @@ def test_get_instantaneous_sensor_data( assert response.status_code == 200 values = response.json["values"] # We expect two data point (from conftest) followed by 2 null values (which are converted to None by .json) - # The first data point is 815 is the first of [815, 817], and the second data point is the first of [818, None]. + # The first data point is the first of [815, 817], and the second data point is the first of [818, None]. assert all(a == b for a, b in zip(values, [815, 818, None, None]))