From 88c6a429a4c4ffa15007968f01fa2763a49a9463 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 11:46:57 +0200 Subject: [PATCH 01/45] remove tests already present in v3 Signed-off-by: F.N. Claessen --- .../api/v2_0/tests/test_api_v2_0_assets.py | 205 ------------------ 1 file changed, 205 deletions(-) diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py index d9e2922f2..444b62c47 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py @@ -9,72 +9,6 @@ from flexmeasures.api.v2_0.tests.utils import get_asset_post_data -@pytest.mark.parametrize("use_auth", [False, True]) -def test_get_assets_badauth(client, use_auth): - """ - Attempt to get assets with wrong or missing auth. - """ - # the case without auth: authentication will fail - headers = {"content-type": "application/json"} - query = {} - if use_auth: - # in this case, we successfully authenticate, but fail authorization - headers["Authorization"] = get_auth_token( - client, "test_prosumer_user_2@seita.nl", "testtest" - ) - test_prosumer_id = find_user_by_email("test_prosumer_user@seita.nl").id - query = {"owner_id": test_prosumer_id} - - get_assets_response = client.get( - url_for("flexmeasures_api_v2_0.get_assets"), query_string=query, headers=headers - ) - print("Server responded with:\n%s" % get_assets_response.json) - if use_auth: - assert get_assets_response.status_code == 403 - else: - assert get_assets_response.status_code == 401 - - -def test_get_asset_nonadmin_access(client): - """Without being an admin, test correct responses when accessing one asset.""" - with UserContext("test_prosumer_user@seita.nl") as prosumer1: - prosumer1_assets = prosumer1.assets - with UserContext("test_prosumer_user_2@seita.nl") as prosumer2: - prosumer2_assets = prosumer2.assets - headers = { - "content-type": "application/json", - "Authorization": get_auth_token( - client, "test_prosumer_user_2@seita.nl", "testtest" - ), - } - - # okay to look at own asset - asset_response = client.get( - url_for("flexmeasures_api_v2_0.get_asset", id=prosumer2_assets[0].id), - headers=headers, - follow_redirects=True, - ) - check_deprecation(asset_response) - assert asset_response.status_code == 200 - # not okay to see assets owned by others - asset_response = client.get( - url_for("flexmeasures_api_v2_0.get_asset", id=prosumer1_assets[0].id), - headers=headers, - follow_redirects=True, - ) - check_deprecation(asset_response) - assert asset_response.status_code == 403 - # proper 404 for non-existing asset - asset_response = client.get( - url_for("flexmeasures_api_v2_0.get_asset", id=8171766575), - headers=headers, - follow_redirects=True, - ) - check_deprecation(asset_response) - assert asset_response.status_code == 404 - assert "not found" in asset_response.json["message"] - - @pytest.mark.parametrize("use_owner_id, num_assets", [(False, 7), (True, 1)]) def test_get_assets(client, add_charging_station_assets, use_owner_id, num_assets): """ @@ -110,101 +44,6 @@ def test_get_assets(client, add_charging_station_assets, use_owner_id, num_asset assert battery["capacity_in_mw"] == 2 -def test_alter_an_asset_wrongauth(client): - # without admin and owner rights, no asset can be created ... - with UserContext("test_prosumer_user@seita.nl") as prosumer1: - prosumer1_asset = prosumer1.assets[0] - with UserContext("test_prosumer_user_2@seita.nl") as prosumer2: - auth_token = prosumer2.get_auth_token() - prosumer2_asset = prosumer2.assets[0] - asset_creation_response = client.post( - url_for("flexmeasures_api_v2_0.post_assets"), - headers={"content-type": "application/json", "Authorization": auth_token}, - json={}, - ) - print(f"Response: {asset_creation_response.json}") - check_deprecation(asset_creation_response) - assert asset_creation_response.status_code == 403 - # ... or edited ... - asset_edit_response = client.patch( - url_for("flexmeasures_api_v2_0.patch_asset", id=prosumer1_asset.id), - headers={"content-type": "application/json", "Authorization": auth_token}, - json={}, - ) - check_deprecation(asset_edit_response) - assert asset_edit_response.status_code == 403 - # ... or deleted ... - asset_delete_response = client.delete( - url_for("flexmeasures_api_v2_0.delete_asset", id=prosumer1_asset.id), - headers={"content-type": "application/json", "Authorization": auth_token}, - json={}, - ) - check_deprecation(asset_delete_response) - assert asset_delete_response.status_code == 403 - # ... which is impossible even if you're the owner - asset_delete_response = client.delete( - url_for("flexmeasures_api_v2_0.delete_asset", id=prosumer2_asset.id), - headers={"content-type": "application/json", "Authorization": auth_token}, - json={}, - ) - check_deprecation(asset_delete_response) - assert asset_delete_response.status_code == 403 - - -def test_post_an_asset_with_existing_name(client): - """Catch DB error (Unique key violated) correctly""" - with UserContext("test_admin_user@seita.nl") as prosumer: - auth_token = prosumer.get_auth_token() - with UserContext("test_prosumer_user@seita.nl") as prosumer: - test_prosumer_id = prosumer.id - existing_asset = prosumer.assets[0] - post_data = get_asset_post_data() - post_data["name"] = existing_asset.name - post_data["owner_id"] = test_prosumer_id - asset_creation = client.post( - url_for("flexmeasures_api_v2_0.post_assets"), - json=post_data, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - check_deprecation(asset_creation) - assert asset_creation.status_code == 422 - assert "already exists" in asset_creation.json["message"]["json"]["name"][0] - - -def test_post_an_asset_with_nonexisting_field(client): - """Posting a field that is unexpected leads to a 422""" - with UserContext("test_admin_user@seita.nl") as prosumer: - auth_token = prosumer.get_auth_token() - post_data = get_asset_post_data() - post_data["nnname"] = "This field does not exist" - asset_creation = client.post( - url_for("flexmeasures_api_v2_0.post_assets"), - json=post_data, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - check_deprecation(asset_creation) - assert asset_creation.status_code == 422 - assert asset_creation.json["message"]["json"]["nnname"][0] == "Unknown field." - - -def test_posting_multiple_assets(client): - """We can only send one at a time""" - with UserContext("test_admin_user@seita.nl") as prosumer: - auth_token = prosumer.get_auth_token() - post_data1 = get_asset_post_data() - post_data2 = get_asset_post_data() - post_data2["name"] = "Test battery 3" - asset_creation = client.post( - url_for("flexmeasures_api_v2_0.post_assets"), - json=[post_data1, post_data2], - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print(f"Response: {asset_creation.json}") - check_deprecation(asset_creation) - assert asset_creation.status_code == 422 - assert asset_creation.json["message"]["json"]["_schema"][0] == "Invalid input type." - - def test_post_an_asset(client): """ Post one extra asset, as an admin user. @@ -229,50 +68,6 @@ def test_post_an_asset(client): assert asset.capacity_in_mw == 3 -def test_post_an_asset_with_invalid_data(client, db): - """ - Add an asset with some fields having invalid data and one field missing. - The right error messages should be in the response and the number of assets has not increased. - """ - with UserContext("test_admin_user@seita.nl") as prosumer: - num_assets_before = len(prosumer.assets) - - auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") - - post_data = get_asset_post_data() - post_data["latitude"] = 70.4 - post_data["longitude"] = 300.9 - post_data["capacity_in_mw"] = -100 - post_data["min_soc_in_mwh"] = 10 - post_data["max_soc_in_mwh"] = 5 - del post_data["unit"] - - post_asset_response = client.post( - url_for("flexmeasures_api_v2_0.post_assets"), - json=post_data, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_asset_response.json) - check_deprecation(post_asset_response) - assert post_asset_response.status_code == 422 - - assert ( - "Must be greater than or equal to 0" - in post_asset_response.json["message"]["json"]["capacity_in_mw"][0] - ) - assert ( - "Longitude 300.9 exceeds the maximum longitude of 180 degrees." - in post_asset_response.json["message"]["json"]["longitude"][0] - ) - assert "required field" in post_asset_response.json["message"]["json"]["unit"][0] - assert ( - "must be equal or higher than the minimum soc" - in post_asset_response.json["message"]["json"]["max_soc_in_mwh"] - ) - - assert Asset.query.filter_by(owner_id=prosumer.id).count() == num_assets_before - - def test_edit_an_asset(client, db): with UserContext("test_prosumer_user@seita.nl") as prosumer: existing_asset = prosumer.assets[1] From f633cdb62af3af03e5a4b133766215139ff23477 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 11:58:19 +0200 Subject: [PATCH 02/45] move asset deletion test to v3 Signed-off-by: F.N. Claessen --- .../api/v2_0/tests/test_api_v2_0_assets.py | 35 ------------------- .../api/v3_0/tests/test_assets_api.py | 16 ++++++++- 2 files changed, 15 insertions(+), 36 deletions(-) diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py index 444b62c47..192cc8ab4 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py @@ -66,38 +66,3 @@ def test_post_an_asset(client): asset: Asset = Asset.query.filter(Asset.name == "Test battery 2").one_or_none() assert asset is not None assert asset.capacity_in_mw == 3 - - -def test_edit_an_asset(client, db): - with UserContext("test_prosumer_user@seita.nl") as prosumer: - existing_asset = prosumer.assets[1] - - post_data = dict(latitude=10, id=999) # id will be ignored - auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") - edit_asset_response = client.patch( - url_for("flexmeasures_api_v2_0.patch_asset", id=existing_asset.id), - json=post_data, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - check_deprecation(edit_asset_response) - assert edit_asset_response.status_code == 200 - updated_asset = Asset.query.filter_by(id=existing_asset.id).one_or_none() - assert updated_asset.latitude == 10 # changed value - assert updated_asset.longitude == existing_asset.longitude - assert updated_asset.capacity_in_mw == existing_asset.capacity_in_mw - assert updated_asset.name == existing_asset.name - - -def test_delete_an_asset(client, db): - with UserContext("test_prosumer_user@seita.nl") as prosumer: - existing_asset_id = prosumer.assets[0].id - - auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") - delete_asset_response = client.delete( - url_for("flexmeasures_api_v2_0.delete_asset", id=existing_asset_id), - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - check_deprecation(delete_asset_response) - assert delete_asset_response.status_code == 204 - deleted_asset = Asset.query.filter_by(id=existing_asset_id).one_or_none() - assert deleted_asset is None diff --git a/flexmeasures/api/v3_0/tests/test_assets_api.py b/flexmeasures/api/v3_0/tests/test_assets_api.py index 0a7aca430..22ee52d16 100644 --- a/flexmeasures/api/v3_0/tests/test_assets_api.py +++ b/flexmeasures/api/v3_0/tests/test_assets_api.py @@ -139,7 +139,7 @@ def test_alter_an_asset(client, setup_api_test_data, setup_accounts): url_for("AssetAPI:patch", id=prosumer_asset.id), headers={"content-type": "application/json", "Authorization": auth_token}, json={ - "latitude": prosumer_asset.latitude + "latitude": prosumer_asset.latitude, }, # we're not changing values to keep other tests clean here ) print(f"Editing Response: {asset_edit_response.json}") @@ -315,3 +315,17 @@ def test_post_an_asset_with_invalid_data(client, setup_api_test_data): GenericAsset.query.filter_by(account_id=prosumer.id).count() == num_assets_before ) + + +def test_delete_an_asset(client, setup_api_test_data): + + existing_asset_id = setup_api_test_data["some gas sensor"].generic_asset.id + + auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") + delete_asset_response = client.delete( + url_for("AssetAPI:delete", id=existing_asset_id), + headers={"content-type": "application/json", "Authorization": auth_token}, + ) + assert delete_asset_response.status_code == 204 + deleted_asset = GenericAsset.query.filter_by(id=existing_asset_id).one_or_none() + assert deleted_asset is None From 3bcd8f1c3c47738847d0dc3c7059c0e82f541a33 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 12:05:19 +0200 Subject: [PATCH 03/45] move asset creation test to v3 Signed-off-by: F.N. Claessen --- .../api/v2_0/tests/test_api_v2_0_assets.py | 24 ------------------- .../api/v3_0/tests/test_assets_api.py | 23 ++++++++++++++++++ 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py index 192cc8ab4..d88d61c26 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py @@ -42,27 +42,3 @@ def test_get_assets(client, add_charging_station_assets, use_owner_id, num_asset ) assert battery["owner_id"] == test_prosumer2_id assert battery["capacity_in_mw"] == 2 - - -def test_post_an_asset(client): - """ - Post one extra asset, as an admin user. - TODO: Soon we'll allow creating assets on an account-basis, i.e. for users - who have the user role "account-admin" or sthg similar. Then we'll - test that here. - """ - auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") - post_data = get_asset_post_data() - post_assets_response = client.post( - url_for("flexmeasures_api_v2_0.post_assets"), - json=post_data, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_assets_response.json) - check_deprecation(post_assets_response) - assert post_assets_response.status_code == 201 - assert post_assets_response.json["latitude"] == 30.1 - - asset: Asset = Asset.query.filter(Asset.name == "Test battery 2").one_or_none() - assert asset is not None - assert asset.capacity_in_mw == 3 diff --git a/flexmeasures/api/v3_0/tests/test_assets_api.py b/flexmeasures/api/v3_0/tests/test_assets_api.py index 22ee52d16..6c243b400 100644 --- a/flexmeasures/api/v3_0/tests/test_assets_api.py +++ b/flexmeasures/api/v3_0/tests/test_assets_api.py @@ -317,6 +317,29 @@ def test_post_an_asset_with_invalid_data(client, setup_api_test_data): ) +def test_post_an_asset(client, setup_api_test_data): + """ + Post one extra asset, as an admin user. + TODO: Soon we'll allow creating assets on an account-basis, i.e. for users + who have the user role "account-admin" or something similar. Then we'll + test that here. + """ + auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") + post_data = get_asset_post_data() + post_assets_response = client.post( + url_for("AssetAPI:post"), + json=post_data, + headers={"content-type": "application/json", "Authorization": auth_token}, + ) + print("Server responded with:\n%s" % post_assets_response.json) + assert post_assets_response.status_code == 201 + assert post_assets_response.json["latitude"] == 30.1 + + asset: GenericAsset = GenericAsset.query.filter_by(name="Test battery 2").one_or_none() + assert asset is not None + assert asset.latitude == 30.1 + + def test_delete_an_asset(client, setup_api_test_data): existing_asset_id = setup_api_test_data["some gas sensor"].generic_asset.id From d7b0c39a6b356aae3b4e86e645ba562d3e97ecb3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 12:08:36 +0200 Subject: [PATCH 04/45] remove last remaining asset test from v2 Signed-off-by: F.N. Claessen --- .../api/v2_0/tests/test_api_v2_0_assets.py | 44 ------------------- 1 file changed, 44 deletions(-) delete mode 100644 flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py deleted file mode 100644 index d88d61c26..000000000 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py +++ /dev/null @@ -1,44 +0,0 @@ -from flask import url_for -import pytest - -import pandas as pd - -from flexmeasures.data.models.assets import Asset -from flexmeasures.data.services.users import find_user_by_email -from flexmeasures.api.tests.utils import check_deprecation, get_auth_token, UserContext -from flexmeasures.api.v2_0.tests.utils import get_asset_post_data - - -@pytest.mark.parametrize("use_owner_id, num_assets", [(False, 7), (True, 1)]) -def test_get_assets(client, add_charging_station_assets, use_owner_id, num_assets): - """ - Get assets, either for all users (our user here is admin, so is allowed to see all 7 assets) or for - a unique one (prosumer user 2 has one asset ― "Test battery"). - """ - auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") - test_prosumer2_id = find_user_by_email("test_prosumer_user_2@seita.nl").id - - query = {} - if use_owner_id: - query["owner_id"] = test_prosumer2_id - - get_assets_response = client.get( - url_for("flexmeasures_api_v2_0.get_assets"), - query_string=query, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_assets_response.json) - check_deprecation(get_assets_response) - assert get_assets_response.status_code == 200 - assert len(get_assets_response.json) == num_assets - - battery = {} - for asset in get_assets_response.json: - if asset["name"] == "Test battery": - battery = asset - assert battery - assert pd.Timestamp(battery["soc_datetime"]) == pd.Timestamp( - "2015-01-01T00:00:00+01:00" - ) - assert battery["owner_id"] == test_prosumer2_id - assert battery["capacity_in_mw"] == 2 From c4e34c4d3fc08644b3e4654b540c33fcfc8d96f7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 12:15:13 +0200 Subject: [PATCH 05/45] stop testing post_price_data and post_prognosis on v2 Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/tests/conftest.py | 13 -- .../api/v2_0/tests/test_api_v2_0_sensors.py | 37 ----- .../tests/test_api_v2_0_sensors_fresh_db.py | 63 --------- flexmeasures/api/v2_0/tests/utils.py | 128 ------------------ 4 files changed, 241 deletions(-) delete mode 100644 flexmeasures/api/v2_0/tests/conftest.py delete mode 100644 flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py delete mode 100644 flexmeasures/api/v2_0/tests/test_api_v2_0_sensors_fresh_db.py delete mode 100644 flexmeasures/api/v2_0/tests/utils.py diff --git a/flexmeasures/api/v2_0/tests/conftest.py b/flexmeasures/api/v2_0/tests/conftest.py deleted file mode 100644 index 305a36400..000000000 --- a/flexmeasures/api/v2_0/tests/conftest.py +++ /dev/null @@ -1,13 +0,0 @@ -import pytest - - -@pytest.fixture(scope="module", autouse=True) -def setup_api_test_data(db, setup_roles_users, add_market_prices, add_battery_assets): - """ - Set up data for API v2.0 tests. - """ - print("Setting up data for API v2.0 tests on %s" % db.engine) - - # Add battery asset - battery = add_battery_assets["Test battery"] - battery.owner = setup_roles_users["Test Prosumer User 2"] diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py deleted file mode 100644 index 92056cecf..000000000 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py +++ /dev/null @@ -1,37 +0,0 @@ -from flask import url_for -import pytest - -from flexmeasures.api.tests.utils import check_deprecation, get_auth_token -from flexmeasures.api.v2_0.tests.utils import ( - message_for_post_prognosis, - verify_sensor_data_in_db, -) - - -@pytest.mark.parametrize( - "post_message, fm_scheme", - [ - (message_for_post_prognosis(), "fm1"), - ], -) -def test_post_prognosis_2_0(db, app, post_message, fm_scheme): - with app.test_client() as client: - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - post_prognosis_response = client.post( - url_for("flexmeasures_api_v2_0.post_prognosis"), - json=post_message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_prognosis_response.json) - check_deprecation(post_prognosis_response) - assert post_prognosis_response.status_code == 200 - assert post_prognosis_response.json["type"] == "PostPrognosisResponse" - - verify_sensor_data_in_db( - post_message, - post_message["values"], - db, - entity_type="connection", - fm_scheme=fm_scheme, - swapped_sign=True, - ) diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors_fresh_db.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors_fresh_db.py deleted file mode 100644 index b782a9e4c..000000000 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors_fresh_db.py +++ /dev/null @@ -1,63 +0,0 @@ -from datetime import timedelta - -import pytest -from flask import url_for -from iso8601 import parse_date - -from flexmeasures.api.common.schemas.sensors import SensorField -from flexmeasures.api.tests.utils import check_deprecation, get_auth_token -from flexmeasures.api.v2_0.tests.utils import ( - message_for_post_price_data, - verify_sensor_data_in_db, -) - - -@pytest.mark.parametrize( - "post_message", - [ - message_for_post_price_data(market_id=7), - message_for_post_price_data(market_id=1, prior_instead_of_horizon=True), - ], -) -def test_post_price_data_2_0( - fresh_db, - setup_roles_users_fresh_db, - setup_markets_fresh_db, - clean_redis, - app, - post_message, -): - """ - Try to post price data as a logged-in test user, which should succeed. - """ - db = fresh_db - # call with client whose context ends, so that we can test for, - # after-effects in the database after teardown committed. - with app.test_client() as client: - # post price data - auth_token = get_auth_token(client, "test_prosumer_user_2@seita.nl", "testtest") - post_price_data_response = client.post( - url_for("flexmeasures_api_v2_0.post_price_data"), - json=post_message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_price_data_response.json) - check_deprecation(post_price_data_response) - assert post_price_data_response.status_code == 200 - assert post_price_data_response.json["type"] == "PostPriceDataResponse" - - verify_sensor_data_in_db( - post_message, post_message["values"], db, entity_type="market", fm_scheme="fm1" - ) - - # look for Forecasting jobs in queue - assert ( - len(app.queues["forecasting"]) == 2 - ) # only one market is affected, but two horizons - horizons = [timedelta(hours=24), timedelta(hours=48)] - jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"]) - market = SensorField("market", fm_scheme="fm1").deserialize(post_message["market"]) - for job, horizon in zip(jobs, horizons): - assert job.kwargs["horizon"] == horizon - assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon - assert job.kwargs["sensor_id"] == market.id diff --git a/flexmeasures/api/v2_0/tests/utils.py b/flexmeasures/api/v2_0/tests/utils.py deleted file mode 100644 index 31e237b2e..000000000 --- a/flexmeasures/api/v2_0/tests/utils.py +++ /dev/null @@ -1,128 +0,0 @@ -from typing import Optional -from datetime import timedelta -from isodate import parse_datetime, parse_duration - -import pandas as pd -import timely_beliefs as tb - -from flexmeasures.api.common.schemas.sensors import SensorField -from flexmeasures.data.models.time_series import Sensor, TimedBelief -from flexmeasures.data.services.users import find_user_by_email -from flexmeasures.api.v1_1.tests.utils import ( - message_for_post_price_data as v1_1_message_for_post_price_data, -) -from flexmeasures.utils.time_utils import duration_isoformat - - -def get_asset_post_data() -> dict: - post_data = { - "name": "Test battery 2", - "unit": "MW", - "capacity_in_mw": 3, - "event_resolution": timedelta(minutes=10).seconds / 60, - "latitude": 30.1, - "longitude": 100.42, - "asset_type_name": "battery", - "owner_id": find_user_by_email("test_prosumer_user@seita.nl").id, - "market_id": Sensor.query.filter_by(name="epex_da").one_or_none().id, - } - return post_data - - -def message_for_post_price_data( - market_id: int, - tile_n: int = 1, - compress_n: int = 1, - duration: Optional[timedelta] = None, - invalid_unit: bool = False, - no_horizon: bool = False, - prior_instead_of_horizon: bool = False, -) -> dict: - """ - The default message has 24 hourly values. - - :param tile_n: Tile the price profile back to back to obtain price data for n days (default = 1). - :param compress_n: Compress the price profile to obtain price data with a coarser resolution (default = 1), - e.g. compress=4 leads to a resolution of 4 hours. - :param duration: Set a duration explicitly to obtain price data with a coarser or finer resolution - (the default is equal to 24 hours * tile_n), - e.g. (assuming tile_n=1) duration=timedelta(hours=6) leads to a resolution of 15 minutes, - and duration=timedelta(hours=48) leads to a resolution of 2 hours. - :param invalid_unit: Choose an invalid unit for the test market (epex_da). - :param no_horizon: Remove the horizon parameter. - :param prior_instead_of_horizon: Remove the horizon parameter and replace it with a prior parameter. - """ - message = v1_1_message_for_post_price_data( - tile_n=tile_n, - compress_n=compress_n, - duration=duration, - invalid_unit=invalid_unit, - ) - message["market"] = f"ea1.2018-06.localhost:fm1.{market_id}" - message["horizon"] = duration_isoformat(timedelta(hours=0)) - if no_horizon or prior_instead_of_horizon: - message.pop("horizon", None) - if prior_instead_of_horizon: - message["prior"] = "2021-01-05T12:00:00+01:00" - return message - - -def verify_sensor_data_in_db( - post_message, - values, - db, - entity_type: str, - fm_scheme: str, - swapped_sign: bool = False, -): - """util method to verify that sensor data ended up in the database""" - start = parse_datetime(post_message["start"]) - end = start + parse_duration(post_message["duration"]) - sensor: Sensor = SensorField(entity_type, fm_scheme).deserialize( - post_message[entity_type] - ) - resolution = sensor.event_resolution - query = ( - db.session.query( - TimedBelief.event_start, - TimedBelief.event_value, - TimedBelief.belief_horizon, - ) - .filter( - (TimedBelief.event_start > start - resolution) - & (TimedBelief.event_start < end) - ) - # .filter(TimedBelief.belief_horizon == (TimedBelief.event_start + resolution) - prior) # only for sensors with 0-hour ex_post knowledge horizon function - .join(Sensor) - .filter(Sensor.name == sensor.name) - ) - if "horizon" in post_message: - horizon = parse_duration(post_message["horizon"]) - query = query.filter(TimedBelief.belief_horizon == horizon) - # todo: after basing sensor data on TimedBelief, we should be able to get a BeliefsDataFrame from the query directly - df = pd.DataFrame( - query.all(), columns=[col["name"] for col in query.column_descriptions] - ) - bdf = tb.BeliefsDataFrame(df, sensor=sensor, source="Some source") - if "prior" in post_message: - prior = parse_datetime(post_message["prior"]) - bdf = bdf.fixed_viewpoint(prior) - if swapped_sign: - bdf["event_value"] = -bdf["event_value"] - assert bdf["event_value"].tolist() == values - - -def message_for_post_prognosis(fm_scheme: str = "fm1"): - """ - Posting prognosis for a wind turbine's production. - """ - message = { - "type": "PostPrognosisRequest", - "connection": f"ea1.2018-06.localhost:{fm_scheme}.2", - "values": [-300, -300, -300, 0, 0, -300], - "start": "2021-01-01T00:00:00Z", - "duration": "PT1H30M", - "prior": "2020-12-31T18:00:00Z", - "unit": "MW", - } - return message From c197cdfbee9f414b874029908ef6f5daefaf77b7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 12:31:15 +0200 Subject: [PATCH 06/45] move test scheduling with unknown prices from v1.3 to v3 Signed-off-by: F.N. Claessen --- .../api/v1_3/tests/test_api_v1_3_fresh_db.py | 82 ------------------- .../api/v3_0/tests/test_sensor_schedules.py | 68 +++++++++++++++ 2 files changed, 68 insertions(+), 82 deletions(-) delete mode 100644 flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py diff --git a/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py b/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py deleted file mode 100644 index ea53f0b47..000000000 --- a/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py +++ /dev/null @@ -1,82 +0,0 @@ -import pytest -from flask import url_for -from isodate import parse_datetime -from rq.job import Job - -from flexmeasures.api.common.responses import unknown_schedule -from flexmeasures.api.tests.utils import check_deprecation, get_auth_token -from flexmeasures.api.v1_3.tests.utils import ( - message_for_post_udi_event, - message_for_get_device_message, -) -from flexmeasures.data.models.data_sources import DataSource -from flexmeasures.data.models.time_series import Sensor -from flexmeasures.data.services.scheduling import handle_scheduling_exception -from flexmeasures.data.tests.utils import work_on_rq - - -@pytest.mark.parametrize("message", [message_for_post_udi_event(unknown_prices=True)]) -def test_post_udi_event_and_get_device_message_with_unknown_prices( - setup_fresh_api_test_data, clean_redis, app, message -): - auth_token = None - with app.test_client() as client: - sensor = Sensor.query.filter(Sensor.name == "Test battery").one_or_none() - message["event"] = message["event"] % sensor.id - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - post_udi_event_response = client.post( - url_for("flexmeasures_api_v1_3.post_udi_event"), - json=message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_udi_event_response.json) - check_deprecation(post_udi_event_response) - assert post_udi_event_response.status_code == 200 - assert post_udi_event_response.json["type"] == "PostUdiEventResponse" - - # look for scheduling jobs in queue - assert ( - len(app.queues["scheduling"]) == 1 - ) # only 1 schedule should be made for 1 asset - job = app.queues["scheduling"].jobs[0] - assert job.kwargs["sensor_id"] == sensor.id - assert job.kwargs["start"] == parse_datetime(message["datetime"]) - assert job.id == message["event"] - assert ( - Job.fetch(message["event"], connection=app.queues["scheduling"].connection) - == job - ) - - # process the scheduling queue - work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) - processed_job = Job.fetch( - message["event"], connection=app.queues["scheduling"].connection - ) - assert processed_job.is_failed is True - - # check results are not in the database - scheduler_source = DataSource.query.filter_by( - name="Seita", type="scheduler" - ).one_or_none() - assert ( - scheduler_source is None - ) # Make sure the scheduler data source is still not there - - # try to retrieve the schedule through the getDeviceMessage api endpoint - message = message_for_get_device_message() - message["event"] = message["event"] % sensor.id - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - get_device_message_response = client.get( - url_for("flexmeasures_api_v1_3.get_device_message"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_device_message_response.json) - check_deprecation(get_device_message_response) - assert get_device_message_response.status_code == 400 - assert get_device_message_response.json["type"] == "GetDeviceMessageResponse" - assert ( - get_device_message_response.json["status"] - == unknown_schedule()[0]["status"] - ) - assert "prices unknown" in get_device_message_response.json["message"].lower() diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index fbf541419..9b6c03908 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -5,9 +5,11 @@ import pandas as pd from rq.job import Job +from flexmeasures.api.common.responses import unknown_schedule from flexmeasures.api.tests.utils import check_deprecation, get_auth_token from flexmeasures.api.v1_3.tests.utils import message_for_get_device_message from flexmeasures.api.v3_0.tests.utils import message_for_trigger_schedule +from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.tests.utils import work_on_rq from flexmeasures.data.services.scheduling import ( @@ -73,6 +75,72 @@ def test_trigger_schedule_with_invalid_flexmodel( ) +@pytest.mark.parametrize("message", [message_for_trigger_schedule(unknown_prices=True)]) +def test_trigger_and_get_schedule_with_unknown_prices( + app, + add_market_prices, + add_battery_assets, + battery_soc_sensor, + add_charging_station_assets, + keep_scheduling_queue_empty, + message, +): + auth_token = None + with app.test_client() as client: + sensor = Sensor.query.filter(Sensor.name == "Test battery").one_or_none() + + # trigger a schedule through the /sensors//schedules/trigger [POST] api endpoint + auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") + trigger_schedule_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + headers={"Authorization": auth_token}, + ) + print("Server responded with:\n%s" % trigger_schedule_response.json) + check_deprecation(trigger_schedule_response, deprecation=None, sunset=None) + assert trigger_schedule_response.status_code == 200 + job_id = trigger_schedule_response.json["schedule"] + + # look for scheduling jobs in queue + assert ( + len(app.queues["scheduling"]) == 1 + ) # only 1 schedule should be made for 1 asset + job = app.queues["scheduling"].jobs[0] + assert job.kwargs["sensor_id"] == sensor.id + assert job.kwargs["start"] == parse_datetime(message["start"]) + assert job.id == job_id + + # process the scheduling queue + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + assert ( + Job.fetch(job_id, connection=app.queues["scheduling"].connection).is_failed + is True + ) + + # check results are not in the database + scheduler_source = DataSource.query.filter_by( + name="Seita", type="scheduler" + ).one_or_none() + assert ( + scheduler_source is None + ) # Make sure the scheduler data source is still not there + + # try to retrieve the schedule through the /sensors//schedules/ [GET] api endpoint + message = message_for_get_device_message() + message["event"] = message["event"] % sensor.id + auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") + get_schedule_response = client.get( + url_for("SensorAPI:get_schedule", id=sensor.id, uuid=job_id), + query_string=message, + headers={"content-type": "application/json", "Authorization": auth_token}, + ) + print("Server responded with:\n%s" % get_schedule_response.json) + check_deprecation(get_schedule_response, deprecation=None, sunset=None) + assert get_schedule_response.status_code == 400 + assert get_schedule_response.json["status"] == unknown_schedule()[0]["status"] + assert "prices unknown" in get_schedule_response.json["message"].lower() + + @pytest.mark.parametrize( "message, asset_name", [ From 80bba2ce2b1ce4d036b4ef44a444cf86e24b58a9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 12:37:53 +0200 Subject: [PATCH 07/45] remove v1.3 test already present in v3 Signed-off-by: F.N. Claessen --- flexmeasures/api/v1_3/tests/test_api_v1_3.py | 186 +------------------ 1 file changed, 2 insertions(+), 184 deletions(-) diff --git a/flexmeasures/api/v1_3/tests/test_api_v1_3.py b/flexmeasures/api/v1_3/tests/test_api_v1_3.py index 3ac080ac0..ae4ca5396 100644 --- a/flexmeasures/api/v1_3/tests/test_api_v1_3.py +++ b/flexmeasures/api/v1_3/tests/test_api_v1_3.py @@ -1,22 +1,10 @@ from flask import url_for import pytest -from datetime import timedelta -from isodate import parse_datetime - -import pandas as pd -from rq.job import Job from flexmeasures.api.common.responses import unrecognized_event from flexmeasures.api.tests.utils import check_deprecation, get_auth_token -from flexmeasures.api.v1_3.tests.utils import ( - message_for_get_device_message, - message_for_post_udi_event, -) -from flexmeasures.data.models.data_sources import DataSource -from flexmeasures.data.models.time_series import Sensor, TimedBelief -from flexmeasures.data.tests.utils import work_on_rq -from flexmeasures.data.services.scheduling import handle_scheduling_exception -from flexmeasures.utils.calculations import integrate_time_series +from flexmeasures.api.v1_3.tests.utils import message_for_get_device_message +from flexmeasures.data.models.time_series import Sensor @pytest.mark.parametrize("message", [message_for_get_device_message(wrong_id=True)]) @@ -37,173 +25,3 @@ def test_get_device_message_wrong_event_id(client, message): get_device_message_response.json["status"] == unrecognized_event(9999, "soc")[0]["status"] ) - - -@pytest.mark.parametrize( - "message, asset_name", - [ - (message_for_post_udi_event(), "Test battery"), - (message_for_post_udi_event(targets=True), "Test charging station"), - ], -) -def test_post_udi_event_and_get_device_message( - app, add_charging_station_assets, message, asset_name -): - auth_token = None - with app.test_client() as client: - sensor = Sensor.query.filter(Sensor.name == asset_name).one_or_none() - message["event"] = message["event"] % sensor.id - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - post_udi_event_response = client.post( - url_for("flexmeasures_api_v1_3.post_udi_event"), - json=message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_udi_event_response.json) - check_deprecation(post_udi_event_response) - assert post_udi_event_response.status_code == 200 - assert post_udi_event_response.json["type"] == "PostUdiEventResponse" - - # test database state - msg_dt = message["datetime"] - sensor = Sensor.query.filter(Sensor.name == asset_name).one_or_none() - assert sensor.generic_asset.get_attribute("soc_datetime") == msg_dt - assert sensor.generic_asset.get_attribute("soc_in_mwh") == message["value"] / 1000 - assert sensor.generic_asset.get_attribute("soc_udi_event_id") == 204 - - # look for scheduling jobs in queue - assert ( - len(app.queues["scheduling"]) == 1 - ) # only 1 schedule should be made for 1 asset - job = app.queues["scheduling"].jobs[0] - assert job.kwargs["sensor_id"] == sensor.id - assert job.kwargs["start"] == parse_datetime(message["datetime"]) - assert job.id == message["event"] - - # process the scheduling queue - work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) - assert ( - Job.fetch( - message["event"], connection=app.queues["scheduling"].connection - ).is_finished - is True - ) - - # check results are in the database - job.refresh() # catch meta info that was added on this very instance - data_source_info = job.meta.get("data_source_info") - scheduler_source = DataSource.query.filter_by( - type="scheduler", **data_source_info - ).one_or_none() - assert ( - scheduler_source is not None - ) # Make sure the scheduler data source is now there - power_values = ( - TimedBelief.query.filter(TimedBelief.sensor_id == sensor.id) - .filter(TimedBelief.source_id == scheduler_source.id) - .all() - ) - resolution = timedelta(minutes=15) - consumption_schedule = pd.Series( - [-v.event_value for v in power_values], - index=pd.DatetimeIndex([v.event_start for v in power_values], freq=resolution), - ) # For consumption schedules, positive values denote consumption. For the db, consumption is negative - assert ( - len(consumption_schedule) - == app.config.get("FLEXMEASURES_PLANNING_HORIZON") / resolution - ) - - # check targets, if applicable - if "targets" in message: - start_soc = message["value"] / 1000 # in MWh - soc_schedule = integrate_time_series( - consumption_schedule, - start_soc, - decimal_precision=6, - ) - print(consumption_schedule) - print(soc_schedule) - for target in message["targets"]: - assert soc_schedule[target["datetime"]] == target["value"] / 1000 - - # try to retrieve the schedule through the getDeviceMessage api endpoint - get_device_message = message_for_get_device_message() - get_device_message["event"] = get_device_message["event"] % sensor.id - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - get_device_message_response = client.get( - url_for("flexmeasures_api_v1_3.get_device_message"), - query_string=get_device_message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_device_message_response.json) - check_deprecation(get_device_message_response) - assert get_device_message_response.status_code == 200 - assert get_device_message_response.json["type"] == "GetDeviceMessageResponse" - assert len(get_device_message_response.json["values"]) == 192 - - # Test that a shorter planning horizon yields the same result for the shorter planning horizon - get_device_message["duration"] = "PT6H" - get_device_message_response_short = client.get( - url_for("flexmeasures_api_v1_3.get_device_message"), - query_string=get_device_message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - assert ( - get_device_message_response_short.json["values"] - == get_device_message_response.json["values"][0:24] - ) - - # Test that a much longer planning horizon yields the same result (when there are only 2 days of prices) - get_device_message["duration"] = "PT1000H" - get_device_message_response_long = client.get( - url_for("flexmeasures_api_v1_3.get_device_message"), - query_string=get_device_message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - assert ( - get_device_message_response_long.json["values"][0:192] - == get_device_message_response.json["values"] - ) - - # sending again results in an error, unless we increase the event ID - with app.test_client() as client: - next_msg_dt = parse_datetime(msg_dt) + timedelta(minutes=5) - message["datetime"] = next_msg_dt.strftime("%Y-%m-%dT%H:%M:%S.%f%z") - post_udi_event_response = client.post( - url_for("flexmeasures_api_v1_3.post_udi_event"), - json=message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_udi_event_response.json) - check_deprecation(post_udi_event_response) - assert post_udi_event_response.status_code == 400 - assert post_udi_event_response.json["type"] == "PostUdiEventResponse" - assert post_udi_event_response.json["status"] == "OUTDATED_UDI_EVENT" - - message["event"] = message["event"].replace("204", "205") - post_udi_event_response = client.post( - url_for("flexmeasures_api_v1_3.post_udi_event"), - json=message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_udi_event_response.json) - assert post_udi_event_response.status_code == 200 - assert post_udi_event_response.json["type"] == "PostUdiEventResponse" - - # test database state - sensor = Sensor.query.filter(Sensor.name == asset_name).one_or_none() - assert parse_datetime( - sensor.generic_asset.get_attribute("soc_datetime") - ) == parse_datetime(message["datetime"]) - assert sensor.generic_asset.get_attribute("soc_in_mwh") == message["value"] / 1000 - assert sensor.generic_asset.get_attribute("soc_udi_event_id") == 205 - - # process the scheduling queue - work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) - # the job still fails due to missing prices for the last time slot, but we did test that the api and worker now processed the UDI event and attempted to create a schedule - assert ( - Job.fetch( - message["event"], connection=app.queues["scheduling"].connection - ).is_failed - is True - ) From 66b374b1cb3ea4be45c84120dfbea802882a2e86 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 12:58:47 +0200 Subject: [PATCH 08/45] move test for wrong job id from v1.3 to v3 Signed-off-by: F.N. Claessen --- flexmeasures/api/v1_3/tests/conftest.py | 17 -------- flexmeasures/api/v1_3/tests/test_api_v1_3.py | 27 ------------ flexmeasures/api/v1_3/tests/utils.py | 43 ------------------- .../api/v3_0/tests/test_sensor_schedules.py | 40 +++++++++++------ 4 files changed, 26 insertions(+), 101 deletions(-) delete mode 100644 flexmeasures/api/v1_3/tests/conftest.py delete mode 100644 flexmeasures/api/v1_3/tests/test_api_v1_3.py delete mode 100644 flexmeasures/api/v1_3/tests/utils.py diff --git a/flexmeasures/api/v1_3/tests/conftest.py b/flexmeasures/api/v1_3/tests/conftest.py deleted file mode 100644 index c19263788..000000000 --- a/flexmeasures/api/v1_3/tests/conftest.py +++ /dev/null @@ -1,17 +0,0 @@ -import pytest - - -@pytest.fixture(scope="module", autouse=True) -def setup_api_test_data(db, add_market_prices, add_battery_assets): - """ - Set up data for API v1.3 tests. - """ - print("Setting up data for API v1.3 tests on %s" % db.engine) - - -@pytest.fixture(scope="function") -def setup_fresh_api_test_data(fresh_db, add_battery_assets_fresh_db): - """ - Set up data for API v1.3 tests. - """ - pass diff --git a/flexmeasures/api/v1_3/tests/test_api_v1_3.py b/flexmeasures/api/v1_3/tests/test_api_v1_3.py deleted file mode 100644 index ae4ca5396..000000000 --- a/flexmeasures/api/v1_3/tests/test_api_v1_3.py +++ /dev/null @@ -1,27 +0,0 @@ -from flask import url_for -import pytest - -from flexmeasures.api.common.responses import unrecognized_event -from flexmeasures.api.tests.utils import check_deprecation, get_auth_token -from flexmeasures.api.v1_3.tests.utils import message_for_get_device_message -from flexmeasures.data.models.time_series import Sensor - - -@pytest.mark.parametrize("message", [message_for_get_device_message(wrong_id=True)]) -def test_get_device_message_wrong_event_id(client, message): - sensor = Sensor.query.filter(Sensor.name == "Test battery").one_or_none() - message["event"] = message["event"] % sensor.id - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - get_device_message_response = client.get( - url_for("flexmeasures_api_v1_3.get_device_message"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_device_message_response.json) - check_deprecation(get_device_message_response) - assert get_device_message_response.status_code == 400 - assert get_device_message_response.json["type"] == "GetDeviceMessageResponse" - assert ( - get_device_message_response.json["status"] - == unrecognized_event(9999, "soc")[0]["status"] - ) diff --git a/flexmeasures/api/v1_3/tests/utils.py b/flexmeasures/api/v1_3/tests/utils.py deleted file mode 100644 index 2e64dc13f..000000000 --- a/flexmeasures/api/v1_3/tests/utils.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Useful test messages""" - - -def message_for_get_device_message( - wrong_id: bool = False, - unknown_prices: bool = False, - targets: bool = False, -) -> dict: - message = { - "type": "GetDeviceMessageRequest", - "duration": "PT48H", - "event": "ea1.2018-06.localhost:%s:204:soc", - } - if wrong_id: - message["event"] = "ea1.2018-06.localhost:%s:9999:soc" - if targets: - message["event"] = message["event"] + "-with-targets" - if unknown_prices: - message[ - "duration" - ] = "PT1000H" # We have no beliefs in our test database about prices so far into the future - return message - - -def message_for_post_udi_event( - unknown_prices: bool = False, - targets: bool = False, -) -> dict: - message = { - "type": "PostUdiEventRequest", - "event": "ea1.2018-06.localhost:%s:204:soc", - "datetime": "2015-01-01T00:00:00+01:00", - "value": 12.1, - "unit": "kWh", - } - if targets: - message["event"] = message["event"] + "-with-targets" - message["targets"] = [{"value": 25, "datetime": "2015-01-02T23:00:00+01:00"}] - if unknown_prices: - message[ - "datetime" - ] = "2040-01-01T00:00:00+01:00" # We have no beliefs in our test database about 2040 prices - return message diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 9b6c03908..53891c677 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -5,9 +5,8 @@ import pandas as pd from rq.job import Job -from flexmeasures.api.common.responses import unknown_schedule +from flexmeasures.api.common.responses import unknown_schedule, unrecognized_event from flexmeasures.api.tests.utils import check_deprecation, get_auth_token -from flexmeasures.api.v1_3.tests.utils import message_for_get_device_message from flexmeasures.api.v3_0.tests.utils import message_for_trigger_schedule from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.time_series import Sensor, TimedBelief @@ -19,6 +18,28 @@ from flexmeasures.utils.calculations import integrate_time_series +def test_get_schedule_wrong_job_id( + app, + add_market_prices, + add_battery_assets, + battery_soc_sensor, + add_charging_station_assets, + keep_scheduling_queue_empty, +): + wrong_job_id = 9999 + sensor = Sensor.query.filter(Sensor.name == "Test battery").one_or_none() + with app.test_client() as client: + auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") + get_schedule_response = client.get( + url_for("SensorAPI:get_schedule", id=sensor.id, uuid=wrong_job_id), + headers={"content-type": "application/json", "Authorization": auth_token}, + ) + print("Server responded with:\n%s" % get_schedule_response.json) + check_deprecation(get_schedule_response, deprecation=None, sunset=None) + assert get_schedule_response.status_code == 400 + assert get_schedule_response.json == unrecognized_event(wrong_job_id, "job")[0] + + @pytest.mark.parametrize( "message, field, sent_value, err_msg", [ @@ -126,12 +147,9 @@ def test_trigger_and_get_schedule_with_unknown_prices( ) # Make sure the scheduler data source is still not there # try to retrieve the schedule through the /sensors//schedules/ [GET] api endpoint - message = message_for_get_device_message() - message["event"] = message["event"] % sensor.id auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") get_schedule_response = client.get( url_for("SensorAPI:get_schedule", id=sensor.id, uuid=job_id), - query_string=message, headers={"content-type": "application/json", "Authorization": auth_token}, ) print("Server responded with:\n%s" % get_schedule_response.json) @@ -261,14 +279,10 @@ def test_trigger_and_get_schedule( assert soc_schedule[target["datetime"]] == target["value"] / 1000 # try to retrieve the schedule through the /sensors//schedules/ [GET] api endpoint - get_schedule_message = message_for_get_device_message( - targets="soc-targets" in message - ) - del get_schedule_message["type"] auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") get_schedule_response = client.get( url_for("SensorAPI:get_schedule", id=sensor.id, uuid=job_id), - query_string=get_schedule_message, + query_string={"duration": "PT48H"}, headers={"content-type": "application/json", "Authorization": auth_token}, ) print("Server responded with:\n%s" % get_schedule_response.json) @@ -277,10 +291,9 @@ def test_trigger_and_get_schedule( assert len(get_schedule_response.json["values"]) == expected_length_of_schedule # Test that a shorter planning horizon yields the same result for the shorter planning horizon - get_schedule_message["duration"] = "PT6H" get_schedule_response_short = client.get( url_for("SensorAPI:get_schedule", id=sensor.id, uuid=job_id), - query_string=get_schedule_message, + query_string={"duration": "PT6H"}, headers={"content-type": "application/json", "Authorization": auth_token}, ) assert ( @@ -289,10 +302,9 @@ def test_trigger_and_get_schedule( ) # Test that a much longer planning horizon yields the same result (when there are only 2 days of prices) - get_schedule_message["duration"] = "PT1000H" get_schedule_response_long = client.get( url_for("SensorAPI:get_schedule", id=sensor.id, uuid=job_id), - query_string=get_schedule_message, + query_string={"duration": "PT1000H"}, headers={"content-type": "application/json", "Authorization": auth_token}, ) assert ( From 5c21e2b59de5d863a03661ea2c06a746a11d4ffd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 13:02:17 +0200 Subject: [PATCH 09/45] no tests worth moving in v1.2 Signed-off-by: F.N. Claessen --- flexmeasures/api/v1_2/tests/conftest.py | 9 - flexmeasures/api/v1_2/tests/test_api_v1_2.py | 172 ------------------- flexmeasures/api/v1_2/tests/utils.py | 29 ---- 3 files changed, 210 deletions(-) delete mode 100644 flexmeasures/api/v1_2/tests/conftest.py delete mode 100644 flexmeasures/api/v1_2/tests/test_api_v1_2.py delete mode 100644 flexmeasures/api/v1_2/tests/utils.py diff --git a/flexmeasures/api/v1_2/tests/conftest.py b/flexmeasures/api/v1_2/tests/conftest.py deleted file mode 100644 index 947a82df5..000000000 --- a/flexmeasures/api/v1_2/tests/conftest.py +++ /dev/null @@ -1,9 +0,0 @@ -import pytest - - -@pytest.fixture(scope="module", autouse=True) -def setup_api_test_data(db, add_market_prices, add_battery_assets): - """ - Set up data for API v1.2 tests. - """ - print("Setting up data for API v1.2 tests on %s" % db.engine) diff --git a/flexmeasures/api/v1_2/tests/test_api_v1_2.py b/flexmeasures/api/v1_2/tests/test_api_v1_2.py deleted file mode 100644 index c1c5b7d73..000000000 --- a/flexmeasures/api/v1_2/tests/test_api_v1_2.py +++ /dev/null @@ -1,172 +0,0 @@ -from flask import url_for -import pytest -from datetime import timedelta -from isodate import parse_datetime - -from flexmeasures.api.common.responses import unrecognized_event, unknown_prices -from flexmeasures.api.tests.utils import check_deprecation, get_auth_token -from flexmeasures.api.v1_2.tests.utils import ( - message_for_get_device_message, - message_for_post_udi_event, -) -from flexmeasures.data.models.time_series import Sensor - - -@pytest.mark.parametrize("message", [message_for_get_device_message()]) -def test_get_device_message(client, message): - sensor = Sensor.query.filter(Sensor.name == "Test battery").one_or_none() - message["event"] = message["event"] % sensor.id - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - get_device_message_response = client.get( - url_for("flexmeasures_api_v1_2.get_device_message"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_device_message_response.json) - check_deprecation(get_device_message_response) - assert get_device_message_response.status_code == 200 - assert get_device_message_response.json["type"] == "GetDeviceMessageResponse" - assert len(get_device_message_response.json["values"]) == 192 - - # Test that a shorter planning horizon yields a shorter result - # Note that the scheduler might give a different result, because it doesn't look as far ahead - message["duration"] = "PT6H" - get_device_message_response_short = client.get( - url_for("flexmeasures_api_v1_2.get_device_message"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_device_message_response_short.json) - assert get_device_message_response_short.status_code == 200 - assert len(get_device_message_response_short.json["values"]) == 24 - - # Test that a much longer planning horizon yields the same result (when there are only 2 days of prices) - message["duration"] = "PT1000H" - get_device_message_response_long = client.get( - url_for("flexmeasures_api_v1_2.get_device_message"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - assert ( - get_device_message_response_long.json["values"][0:192] - == get_device_message_response.json["values"] - ) - - -def test_get_device_message_mistyped_duration(client): - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - message = message_for_get_device_message() - sensor = Sensor.query.filter(Sensor.name == "Test battery").one_or_none() - message["event"] = message["event"] % sensor.id - message["duration"] = "PTT6H" - get_device_message_response = client.get( - url_for("flexmeasures_api_v1_2.get_device_message"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_device_message_response.json) - check_deprecation(get_device_message_response) - assert get_device_message_response.status_code == 422 - assert ( - "Cannot parse PTT6H as ISO8601 duration" - in get_device_message_response.json["message"]["args_and_json"]["duration"][0] - ) - - -@pytest.mark.parametrize("message", [message_for_get_device_message(wrong_id=True)]) -def test_get_device_message_wrong_event_id(client, message): - sensor = Sensor.query.filter(Sensor.name == "Test battery").one_or_none() - message["event"] = message["event"] % sensor.id - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - get_device_message_response = client.get( - url_for("flexmeasures_api_v1_2.get_device_message"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_device_message_response.json) - check_deprecation(get_device_message_response) - assert get_device_message_response.status_code == 400 - assert get_device_message_response.json["type"] == "GetDeviceMessageResponse" - assert ( - get_device_message_response.json["status"] - == unrecognized_event(9999, "soc")[0]["status"] - ) - - -@pytest.mark.parametrize( - "message", [message_for_get_device_message(unknown_prices=True)] -) -def test_get_device_message_unknown_prices(client, message): - sensor = Sensor.query.filter( - Sensor.name == "Test battery with no known prices" - ).one_or_none() - message["event"] = message["event"] % sensor.id - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - get_device_message_response = client.get( - url_for("flexmeasures_api_v1_2.get_device_message"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_device_message_response.json) - assert get_device_message_response.status_code == 400 - assert get_device_message_response.json["type"] == "GetDeviceMessageResponse" - assert get_device_message_response.json["status"] == unknown_prices()[0]["status"] - - -@pytest.mark.parametrize("message", [message_for_post_udi_event()]) -def test_post_udi_event(app, message): - auth_token = None - with app.test_client() as client: - sensor = Sensor.query.filter(Sensor.name == "Test battery").one_or_none() - message["event"] = message["event"] % sensor.id - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - post_udi_event_response = client.post( - url_for("flexmeasures_api_v1_2.post_udi_event"), - json=message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_udi_event_response.json) - check_deprecation(post_udi_event_response) - assert post_udi_event_response.status_code == 200 - assert post_udi_event_response.json["type"] == "PostUdiEventResponse" - - msg_dt = message["datetime"] - - # test database state - sensor = Sensor.query.filter(Sensor.name == "Test battery").one_or_none() - assert sensor.generic_asset.get_attribute("soc_datetime") == msg_dt - assert sensor.generic_asset.get_attribute("soc_in_mwh") == message["value"] / 1000 - assert sensor.generic_asset.get_attribute("soc_udi_event_id") == 204 - - # sending again results in an error, unless we increase the event ID - with app.test_client() as client: - next_msg_dt = parse_datetime(msg_dt) + timedelta(minutes=5) - message["datetime"] = next_msg_dt.strftime("%Y-%m-%dT%H:%M:%S.%f%z") - post_udi_event_response = client.post( - url_for("flexmeasures_api_v1_2.post_udi_event"), - json=message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_udi_event_response.json) - check_deprecation(post_udi_event_response) - assert post_udi_event_response.status_code == 400 - assert post_udi_event_response.json["type"] == "PostUdiEventResponse" - assert post_udi_event_response.json["status"] == "OUTDATED_UDI_EVENT" - - message["event"] = message["event"].replace("204", "205") - post_udi_event_response = client.post( - url_for("flexmeasures_api_v1_2.post_udi_event"), - json=message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_udi_event_response.json) - assert post_udi_event_response.status_code == 200 - assert post_udi_event_response.json["type"] == "PostUdiEventResponse" - - # test database state - sensor = Sensor.query.filter(Sensor.name == "Test battery").one_or_none() - assert parse_datetime( - sensor.generic_asset.get_attribute("soc_datetime") - ) == parse_datetime(message["datetime"]) - assert sensor.generic_asset.get_attribute("soc_in_mwh") == message["value"] / 1000 - assert sensor.generic_asset.get_attribute("soc_udi_event_id") == 205 diff --git a/flexmeasures/api/v1_2/tests/utils.py b/flexmeasures/api/v1_2/tests/utils.py deleted file mode 100644 index 34ec71a8c..000000000 --- a/flexmeasures/api/v1_2/tests/utils.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Useful test messages""" - - -def message_for_get_device_message( - wrong_id: bool = False, unknown_prices: bool = False -) -> dict: - message = { - "type": "GetDeviceMessageRequest", - "duration": "PT48H", - "event": "ea1.2018-06.localhost:%s:203:soc", - } - if wrong_id: - message["event"] = "ea1.2018-06.localhost:%s:9999:soc" - if unknown_prices: - message[ - "duration" - ] = "PT1000H" # We have no beliefs in our test database about prices so far into the future - return message - - -def message_for_post_udi_event() -> dict: - message = { - "type": "PostUdiEventRequest", - "event": "ea1.2018-06.io.flexmeasures.company:%s:204:soc", - "datetime": "2018-09-27T10:00:00+00:00", - "value": 12.1, - "unit": "kWh", - } - return message From fea604ac1e4f7bb892ab4c1589ed1243ece81e0e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 13:18:23 +0200 Subject: [PATCH 10/45] remove test for posting weather data from v1.1 Signed-off-by: F.N. Claessen --- .../api/v1_1/tests/test_api_v1_1_fresh_db.py | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py b/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py index 0848afe91..498c959dd 100644 --- a/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py +++ b/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py @@ -53,39 +53,3 @@ def test_post_price_data_unexpected_resolution( post_message, [v for v in post_message["values"] for i in range(4)], db ) - -@pytest.mark.parametrize( - "post_message", - [message_for_post_weather_data(as_forecasts=False)], -) -def test_post_weather_data( - setup_fresh_api_v1_1_test_data, - add_weather_sensors_fresh_db, - app, - client, - post_message, -): - """ - Try to post wind speed data as a logged-in test user, which should lead to forecasting jobs. - """ - auth_token = get_auth_token(client, "test_prosumer_user_2@seita.nl", "testtest") - post_weather_data_response = client.post( - url_for("flexmeasures_api_v1_1.post_weather_data"), - json=post_message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_weather_data_response.json) - check_deprecation(post_weather_data_response) - assert post_weather_data_response.status_code == 200 - assert post_weather_data_response.json["type"] == "PostWeatherDataResponse" - - forecast_horizons = forecast_horizons_for(timedelta(minutes=5)) - jobs = get_forecasting_jobs(last_n=len(forecast_horizons)) - for job, horizon in zip( - sorted(jobs, key=lambda x: x.kwargs["horizon"]), forecast_horizons - ): - # check if jobs have expected horizons - assert job.kwargs["horizon"] == horizon - # check if jobs' start time (the time to be forecasted) - # is the weather observation plus the horizon - assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon From 0f3cee595e3acae3bb3c5fa91eb7963c1a828782 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 13:37:57 +0200 Subject: [PATCH 11/45] move test for posting data with incompatible resolution from v1.1 to v3 Signed-off-by: F.N. Claessen --- .../api/v1_1/tests/test_api_v1_1_fresh_db.py | 55 ------------------- .../v3_0/tests/test_sensor_data_fresh_db.py | 32 ++++++++--- 2 files changed, 25 insertions(+), 62 deletions(-) delete mode 100644 flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py diff --git a/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py b/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py deleted file mode 100644 index 498c959dd..000000000 --- a/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py +++ /dev/null @@ -1,55 +0,0 @@ -from datetime import timedelta -from iso8601 import parse_date - -import pytest -from flask import url_for -from isodate import duration_isoformat - -from flexmeasures.utils.time_utils import forecast_horizons_for -from flexmeasures.api.common.responses import unapplicable_resolution -from flexmeasures.api.tests.utils import check_deprecation, get_auth_token -from flexmeasures.api.v1_1.tests.utils import ( - message_for_post_price_data, - message_for_post_weather_data, - verify_prices_in_db, - get_forecasting_jobs, -) - - -@pytest.mark.parametrize( - "post_message, status, msg", - [ - ( - message_for_post_price_data( - duration=duration_isoformat(timedelta(minutes=2)) - ), - 400, - unapplicable_resolution()[0]["message"], - ), - (message_for_post_price_data(compress_n=4), 200, "Request has been processed."), - ], -) -def test_post_price_data_unexpected_resolution( - setup_fresh_api_v1_1_test_data, app, client, post_message, status, msg -): - """ - Try to post price data with an unexpected resolution, - which might be fixed with upsampling or otherwise fail. - """ - db = setup_fresh_api_v1_1_test_data - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - post_price_data_response = client.post( - url_for("flexmeasures_api_v1_1.post_price_data"), - json=post_message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_price_data_response.json) - check_deprecation(post_price_data_response) - assert post_price_data_response.json["type"] == "PostPriceDataResponse" - assert post_price_data_response.status_code == status - assert msg in post_price_data_response.json["message"] - if "processed" in msg: - verify_prices_in_db( - post_message, [v for v in post_message["values"] for i in range(4)], db - ) - diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index fe924f776..b585910d9 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -13,20 +13,37 @@ @pytest.mark.parametrize( - "num_values, expected_num_values, unit, include_a_null, expected_value", + "num_values, expected_num_values, unit, include_a_null, expected_value, expected_status", [ - (6, 6, "m³/h", False, -11.28), - (6, 5, "m³/h", True, -11.28), # NaN value does not enter database - (6, 6, "m³", False, 6 * -11.28), # 6 * 10-min intervals per hour - (6, 6, "l/h", False, -11.28 / 1000), # 1 m³ = 1000 l - (3, 6, "m³/h", False, -11.28), # upsample from 20-min intervals + (6, 6, "m³/h", False, -11.28, 200), + (6, 5, "m³/h", True, -11.28, 200), # NaN value does not enter database + (6, 6, "m³", False, 6 * -11.28, 200), # 6 * 10-min intervals per hour + (6, 6, "l/h", False, -11.28 / 1000, 200), # 1 m³ = 1000 l + (3, 6, "m³/h", False, -11.28, 200), # upsample from 20-min intervals ( 1, 6, "m³/h", False, -11.28, + 200, ), # upsample from single value for 1-hour interval, sent as float rather than list of floats + ( + 4, + 0, + "m³/h", + False, + None, + 422, + ), # failed to resample from 15-min intervals to 10-min intervals + ( + 10, + 0, + "m³/h", + False, + None, + 422, + ), # failed to resample from 6-min intervals to 10-min intervals ], ) def test_post_sensor_data( @@ -37,6 +54,7 @@ def test_post_sensor_data( unit, include_a_null, expected_value, + expected_status, ): post_data = make_sensor_data_request_for_gas_sensor( num_values=num_values, unit=unit, include_a_null=include_a_null @@ -57,7 +75,7 @@ def test_post_sensor_data( headers={"Authorization": auth_token}, ) print(response.json) - assert response.status_code == 200 + assert response.status_code == expected_status beliefs = TimedBelief.query.filter(*filters).all() print(f"BELIEFS AFTER: {beliefs}") assert len(beliefs) == expected_num_values From e29d20a1172126be1646aa5df2fc5e32466e2bb6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 17:01:52 +0200 Subject: [PATCH 12/45] move test for posting data with a user that is not properly set up as a data source, from v1.1 to v3; also test posting is not allowed for inactive admins Signed-off-by: F.N. Claessen --- flexmeasures/__init__.py | 7 +++- .../api/common/schemas/sensor_data.py | 11 +---- flexmeasures/api/v1_1/tests/test_api_v1_1.py | 28 ------------- flexmeasures/api/v3_0/tests/conftest.py | 40 ++++++++++++++++--- .../api/v3_0/tests/test_api_v3_0_users.py | 2 +- .../tests/test_api_v3_0_users_fresh_db.py | 17 ++++---- .../v3_0/tests/test_sensor_data_fresh_db.py | 33 +++++++++++++++ 7 files changed, 85 insertions(+), 53 deletions(-) diff --git a/flexmeasures/__init__.py b/flexmeasures/__init__.py index 703c36398..fd677cbb6 100644 --- a/flexmeasures/__init__.py +++ b/flexmeasures/__init__.py @@ -1,7 +1,12 @@ from importlib_metadata import version, PackageNotFoundError from flexmeasures.data.models.annotations import Annotation # noqa F401 -from flexmeasures.data.models.user import Account, AccountRole, User # noqa F401 +from flexmeasures.data.models.user import ( # noqa F401 + Account, + AccountRole, + User, + Role as UserRole, +) from flexmeasures.data.models.data_sources import DataSource as Source # noqa F401 from flexmeasures.data.models.generic_assets import ( # noqa F401 GenericAsset as Asset, diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 1502d8286..29a8b6358 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -10,12 +10,12 @@ import pandas as pd from flexmeasures.data import ma -from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.time_series import Sensor from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.api.common.utils.api_utils import upsample_values from flexmeasures.data.models.planning.utils import initialize_index from flexmeasures.data.schemas import AwareDateTimeField, DurationField, SourceIdField +from flexmeasures.data.services.data_sources import get_or_create_source from flexmeasures.data.services.time_series import simplify_index from flexmeasures.utils.time_utils import ( decide_resolution, @@ -350,14 +350,7 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame: """ Turn the de-serialized and validated data into a BeliefsDataFrame. """ - source = DataSource.query.filter( - DataSource.user_id == current_user.id - ).one_or_none() - if not source: - raise ValidationError( - f"User {current_user.id} is not an accepted data source." - ) - + source = get_or_create_source(current_user) num_values = len(sensor_data["values"]) event_resolution = sensor_data["duration"] / num_values dt_index = pd.date_range( diff --git a/flexmeasures/api/v1_1/tests/test_api_v1_1.py b/flexmeasures/api/v1_1/tests/test_api_v1_1.py index 9d08bb5dc..d30a9c972 100644 --- a/flexmeasures/api/v1_1/tests/test_api_v1_1.py +++ b/flexmeasures/api/v1_1/tests/test_api_v1_1.py @@ -245,31 +245,3 @@ def test_post_weather_forecasts_invalid_unit(setup_api_test_data, client, post_m invalid_unit("wind speed", ["m/s"])[0]["message"] in post_weather_data_response.json["message"] ) - - -@pytest.mark.parametrize("post_message", [message_for_post_price_data()]) -def test_auto_fix_missing_registration_of_user_as_data_source( - setup_api_test_data, client, post_message -): - """Try to post price data as a user that has not been properly registered as a data source. - The API call should succeed and the user should be automatically registered as a data source. - """ - - # post price data - auth_token = get_auth_token(client, "test_improper_user@seita.nl", "testtest") - post_price_data_response = client.post( - url_for("flexmeasures_api_v1_1.post_price_data"), - json=post_message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_price_data_response.json) - check_deprecation(post_price_data_response) - assert post_price_data_response.status_code == 200 - - formerly_improper_user = User.query.filter( - User.email == "test_improper_user@seita.nl" - ).one_or_none() - data_source = DataSource.query.filter( - DataSource.user == formerly_improper_user - ).one_or_none() - assert data_source is not None diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index 95447a7e1..a1b13a246 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -4,7 +4,7 @@ import pytest from flask_security import SQLAlchemySessionUserDatastore, hash_password -from flexmeasures import Sensor, Source +from flexmeasures import Sensor, Source, User, UserRole from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset from flexmeasures.data.models.time_series import TimedBelief @@ -40,18 +40,46 @@ def setup_api_fresh_test_data( @pytest.fixture(scope="module") def setup_inactive_user(db, setup_accounts, setup_roles_users): """ - Set up one inactive user. + Set up one inactive user and one inactive admin. """ - from flexmeasures.data.models.user import User, Role - - user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) + user_datastore = SQLAlchemySessionUserDatastore(db.session, User, UserRole) user_datastore.create_user( username="inactive test user", - email="inactive@seita.nl", + email="inactive_user@seita.nl", password=hash_password("testtest"), account_id=setup_accounts["Prosumer"].id, active=False, ) + admin = user_datastore.create_user( + username="inactive test admin", + email="inactive_admin@seita.nl", + password=hash_password("testtest"), + account_id=setup_accounts["Prosumer"].id, + active=False, + ) + role = user_datastore.find_role("admin") + user_datastore.add_role_to_user(admin, role) + + +@pytest.fixture(scope="function") +def setup_user_without_data_source( + fresh_db, setup_accounts_fresh_db, setup_roles_users_fresh_db +) -> User: + """ + Set up one user directly without setting up a corresponding data source. + """ + + user_datastore = SQLAlchemySessionUserDatastore(fresh_db.session, User, UserRole) + user = user_datastore.create_user( + username="test admin with improper registration as a data source", + email="improper_user@seita.nl", + password=hash_password("testtest"), + account_id=setup_accounts_fresh_db["Prosumer"].id, + active=True, + ) + role = user_datastore.find_role("admin") + user_datastore.add_role_to_user(user, role) + return user @pytest.fixture(scope="function") diff --git a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py index 603999ba7..35093cd0d 100644 --- a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py +++ b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py @@ -51,7 +51,7 @@ def test_get_users_inactive(client, setup_inactive_user, include_inactive): if include_inactive is False: assert len(get_users_response.json) == 2 else: - assert len(get_users_response.json) == 3 + assert len(get_users_response.json) == 4 @pytest.mark.parametrize( diff --git a/flexmeasures/api/v3_0/tests/test_api_v3_0_users_fresh_db.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users_fresh_db.py index 1603b8102..78bc8b2d2 100644 --- a/flexmeasures/api/v3_0/tests/test_api_v3_0_users_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_api_v3_0_users_fresh_db.py @@ -7,13 +7,14 @@ @pytest.mark.parametrize( "sender", - ( - (""), - ("test_prosumer_user@seita.nl"), - ("test_prosumer_user_2@seita.nl"), - ("test_admin_user@seita.nl"), - ("inactive@seita.nl"), - ), + [ + "", + "test_prosumer_user@seita.nl", + "test_prosumer_user_2@seita.nl", + "test_admin_user@seita.nl", + "inactive_user@seita.nl", + "inactive_admin@seita.nl", + ], ) def test_user_reset_password(app, client, setup_inactive_user, sender): """ @@ -34,7 +35,7 @@ def test_user_reset_password(app, client, setup_inactive_user, sender): ) print("Server responded with:\n%s" % pwd_reset_response.json) - if sender in ("", "inactive@seita.nl"): + if sender in ("", "inactive_user@seita.nl", "inactive_admin@seita.nl"): assert pwd_reset_response.status_code == 401 return if sender == "test_prosumer_user@seita.nl": diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index b585910d9..ecb42b8fc 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -85,6 +85,39 @@ def test_post_sensor_data( ) +def test_auto_fix_missing_registration_of_user_as_data_source( + client, + setup_api_fresh_test_data, + setup_user_without_data_source, +): + """Try to post sensor data as a user that has not been properly registered as a data source. + The API call should succeed and the user should be automatically registered as a data source. + """ + + # Make sure the user is not yet registered as a data source + data_source = Source.query.filter_by( + user=setup_user_without_data_source + ).one_or_none() + assert data_source is None + + post_data = make_sensor_data_request_for_gas_sensor( + num_values=6, unit="m³/h", include_a_null=False + ) + auth_token = get_auth_token(client, "improper_user@seita.nl", "testtest") + response = client.post( + url_for("SensorAPI:post_data"), + json=post_data, + headers={"Authorization": auth_token}, + ) + assert response.status_code == 200 + + # Make sure the user is now registered as a data source + data_source = Source.query.filter_by( + user=setup_user_without_data_source + ).one_or_none() + assert data_source is not None + + def test_get_sensor_data( client, setup_api_fresh_test_data: dict[str, Sensor], From a5236a9c34e26b62f1e6fe31f26d26835f0a10bc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 17:19:56 +0200 Subject: [PATCH 13/45] move test for getting data for an empty period from v1.1 to v3 Signed-off-by: F.N. Claessen --- flexmeasures/api/v1_1/tests/test_api_v1_1.py | 247 ------------------ .../api/v3_0/tests/test_sensor_data.py | 30 +++ 2 files changed, 30 insertions(+), 247 deletions(-) delete mode 100644 flexmeasures/api/v1_1/tests/test_api_v1_1.py diff --git a/flexmeasures/api/v1_1/tests/test_api_v1_1.py b/flexmeasures/api/v1_1/tests/test_api_v1_1.py deleted file mode 100644 index d30a9c972..000000000 --- a/flexmeasures/api/v1_1/tests/test_api_v1_1.py +++ /dev/null @@ -1,247 +0,0 @@ -from flask import url_for -import pytest -from datetime import timedelta -from iso8601 import parse_date - -from flexmeasures.api.common.schemas.sensors import SensorField -from flexmeasures.utils.entity_address_utils import parse_entity_address -from flexmeasures.api.common.responses import ( - request_processed, - invalid_horizon, - invalid_unit, -) -from flexmeasures.api.tests.utils import check_deprecation, get_auth_token -from flexmeasures.api.common.utils.api_utils import ( - message_replace_name_with_ea, -) -from flexmeasures.api.v1_1.tests.utils import ( - message_for_get_prognosis, - message_for_post_price_data, - message_for_post_weather_data, - verify_prices_in_db, - get_forecasting_jobs, -) -from flexmeasures.auth.error_handling import UNAUTH_ERROR_STATUS -from flexmeasures.data.models.data_sources import DataSource -from flexmeasures.data.models.user import User -from flexmeasures.data.models.time_series import Sensor - - -@pytest.mark.parametrize("query", [{}, {"access": "Prosumer"}]) -def test_get_service(client, query): - get_service_response = client.get( - url_for("flexmeasures_api_v1_1.get_service"), - query_string=query, - headers={"content-type": "application/json"}, - ) - print("Server responded with:\n%s" % get_service_response.json) - check_deprecation(get_service_response) - assert get_service_response.status_code == 200 - assert get_service_response.json["type"] == "GetServiceResponse" - assert get_service_response.json["status"] == request_processed()[0]["status"] - if "access" in query: - for service in get_service_response.json["services"]: - assert "Prosumer" in service["access"] - - -def test_unauthorized_prognosis_request(client): - get_prognosis_response = client.get( - url_for("flexmeasures_api_v1_1.get_prognosis"), - query_string=message_for_get_prognosis(), - headers={"content-type": "application/json"}, - ) - print("Server responded with:\n%s" % get_prognosis_response.json) - check_deprecation(get_prognosis_response) - assert get_prognosis_response.status_code == 401 - assert get_prognosis_response.json["type"] == "GetPrognosisResponse" - assert get_prognosis_response.json["status"] == UNAUTH_ERROR_STATUS - - -@pytest.mark.parametrize( - "message", - [ - message_for_get_prognosis(invalid_horizon=True), - ], -) -def test_invalid_horizon(setup_api_test_data, client, message): - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - get_prognosis_response = client.get( - url_for("flexmeasures_api_v1_1.get_prognosis"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_prognosis_response.json) - check_deprecation(get_prognosis_response) - assert get_prognosis_response.status_code == 400 - assert get_prognosis_response.json["type"] == "GetPrognosisResponse" - assert get_prognosis_response.json["status"] == invalid_horizon()[0]["status"] - - -def test_no_data(setup_api_test_data, client): - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - get_prognosis_response = client.get( - url_for("flexmeasures_api_v1_1.get_prognosis"), - query_string=message_replace_name_with_ea( - message_for_get_prognosis(no_data=True) - ), - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_prognosis_response.json) - check_deprecation(get_prognosis_response) - assert get_prognosis_response.status_code == 200 - assert get_prognosis_response.json["type"] == "GetPrognosisResponse" - assert get_prognosis_response.json["values"] == [] - - -@pytest.mark.parametrize( - "message", - [ - message_for_get_prognosis(), - message_for_get_prognosis(single_connection=False), - message_for_get_prognosis(single_connection=True), - message_for_get_prognosis(no_resolution=True), - message_for_get_prognosis(rolling_horizon=True), - message_for_get_prognosis(with_prior=True), - message_for_get_prognosis(rolling_horizon=True, timezone_alternative=True), - ], -) -def test_get_prognosis(setup_api_test_data, client, message): - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - get_prognosis_response = client.get( - url_for("flexmeasures_api_v1_1.get_prognosis"), - query_string=message_replace_name_with_ea(message), - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_prognosis_response.json) - check_deprecation(get_prognosis_response) - assert get_prognosis_response.status_code == 200 - if "groups" in get_prognosis_response.json: - assert get_prognosis_response.json["groups"][0]["values"] == [ - 300, - 301, - 302, - 303, - 304, - 305, - ] - else: - assert get_prognosis_response.json["values"] == [300, 301, 302, 303, 304, 305] - - -@pytest.mark.parametrize("post_message", [message_for_post_price_data()]) -def test_post_price_data(setup_api_test_data, db, app, clean_redis, post_message): - """ - Try to post price data as a logged-in test user with the Prosumer role, which should succeed. - """ - # call with client whose context ends, so that we can test for, - # after-effects in the database after teardown committed. - with app.test_client() as client: - # post price data - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - post_price_data_response = client.post( - url_for("flexmeasures_api_v1_1.post_price_data"), - json=post_message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_price_data_response.json) - check_deprecation(post_price_data_response) - assert post_price_data_response.status_code == 200 - assert post_price_data_response.json["type"] == "PostPriceDataResponse" - - verify_prices_in_db(post_message, post_message["values"], db) - - # look for Forecasting jobs in queue - assert ( - len(app.queues["forecasting"]) == 2 - ) # only one market is affected, but two horizons - horizons = [timedelta(hours=24), timedelta(hours=48)] - jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"]) - market = SensorField("market", "fm0").deserialize(post_message["market"]) - for job, horizon in zip(jobs, horizons): - assert job.kwargs["horizon"] == horizon - assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon - assert job.kwargs["sensor_id"] == market.id - - -@pytest.mark.parametrize( - "post_message", [message_for_post_price_data(invalid_unit=True)] -) -def test_post_price_data_invalid_unit(setup_api_test_data, client, post_message): - """ - Try to post price data with the wrong unit, which should fail. - """ - - # post price data - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - post_price_data_response = client.post( - url_for("flexmeasures_api_v1_1.post_price_data"), - json=post_message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_price_data_response.json) - check_deprecation(post_price_data_response) - assert post_price_data_response.status_code == 400 - assert post_price_data_response.json["type"] == "PostPriceDataResponse" - ea = parse_entity_address(post_message["market"], "market", fm_scheme="fm0") - market_name = ea["market_name"] - sensor = Sensor.query.filter_by(name=market_name).one_or_none() - assert ( - invalid_unit("%s prices" % sensor.name, ["EUR/MWh"])[0]["message"] - in post_price_data_response.json["message"] - ) - - -@pytest.mark.parametrize( - "post_message", - [message_for_post_weather_data(), message_for_post_weather_data(temperature=True)], -) -def test_post_weather_forecasts( - setup_api_test_data, add_weather_sensors, app, client, post_message -): - """ - Try to post wind speed and temperature forecasts as a logged-in test user with the Prosumer role, which should succeed. - As only forecasts are sent, no additional forecasting jobs are expected. - """ - num_jobs_before = len(get_forecasting_jobs()) - - # post weather data - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - post_weather_data_response = client.post( - url_for("flexmeasures_api_v1_1.post_weather_data"), - json=post_message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_weather_data_response.json) - check_deprecation(post_weather_data_response) - assert post_weather_data_response.status_code == 200 - assert post_weather_data_response.json["type"] == "PostWeatherDataResponse" - - num_jobs_after = len(get_forecasting_jobs()) - assert num_jobs_after == num_jobs_before - - -@pytest.mark.parametrize( - "post_message", [message_for_post_weather_data(invalid_unit=True)] -) -def test_post_weather_forecasts_invalid_unit(setup_api_test_data, client, post_message): - """ - Try to post wind speed data as a logged-in test user with the Prosumer role, but with a wrong unit for wind speed, - which should fail. - """ - - # post weather data - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - post_weather_data_response = client.post( - url_for("flexmeasures_api_v1_1.post_weather_data"), - json=post_message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_weather_data_response.json) - check_deprecation(post_weather_data_response) - assert post_weather_data_response.status_code == 400 - assert post_weather_data_response.json["type"] == "PostWeatherDataResponse" - # also checks that any underscore in the physical or economic quantity should be replaced with a space - assert ( - invalid_unit("wind speed", ["m/s"])[0]["message"] - in post_weather_data_response.json["message"] - ) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 81180dba7..c7c686075 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -1,10 +1,40 @@ +from __future__ import annotations + from flask import url_for import pytest +from flexmeasures import Sensor from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor +def test_get_no_sensor_data( + client, + setup_api_test_data: dict[str, Sensor], +): + """Check the /sensors/data endpoint for fetching data for a period without any data.""" + sensor = setup_api_test_data["some gas sensor"] + message = { + "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", + "start": "1921-05-02T00:00:00+02:00", # we have loaded no test data for this year + "duration": "PT1H20M", + "horizon": "PT0H", + "unit": "m³/h", + "resolution": "PT20M", + } + auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") + response = client.get( + url_for("SensorAPI:get_data"), + query_string=message, + headers={"content-type": "application/json", "Authorization": auth_token}, + ) + print("Server responded with:\n%s" % response.json) + assert response.status_code == 200 + values = response.json["values"] + # We expect only null values (which are converted to None by .json) + assert all(a == b for a, b in zip(values, [None, None, None, None])) + + @pytest.mark.parametrize("use_auth", [False, True]) def test_post_sensor_data_bad_auth(client, setup_api_test_data, use_auth): """ From a95864bd7c740b8d6f46510dd6c7020e674a9555 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 17:28:36 +0200 Subject: [PATCH 14/45] move v3 tests that just GET data to test module that uses db fixture rather than fresh_db fixture Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_sensor_data.py | 68 ++++++++++++++++++- .../v3_0/tests/test_sensor_data_fresh_db.py | 68 +------------------ 2 files changed, 68 insertions(+), 68 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index c7c686075..5af1ec722 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -1,9 +1,11 @@ from __future__ import annotations +from datetime import timedelta + from flask import url_for import pytest -from flexmeasures import Sensor +from flexmeasures import Sensor, Source, User from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor @@ -35,6 +37,70 @@ def test_get_no_sensor_data( assert all(a == b for a, b in zip(values, [None, None, None, None])) +def test_get_sensor_data( + client, + setup_api_test_data: dict[str, Sensor], + setup_roles_users: dict[str, User], +): + """Check the /sensors/data endpoint for fetching 1 hour of data of a 10-minute resolution sensor.""" + sensor = setup_api_test_data["some gas sensor"] + source: Source = setup_roles_users["Test Supplier User"].data_source[0] + assert sensor.event_resolution == timedelta(minutes=10) + message = { + "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", + "start": "2021-05-02T00:00:00+02:00", + "duration": "PT1H20M", + "horizon": "PT0H", + "unit": "m³/h", + "source": source.id, + "resolution": "PT20M", + } + auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") + response = client.get( + url_for("SensorAPI:get_data"), + query_string=message, + headers={"content-type": "application/json", "Authorization": auth_token}, + ) + print("Server responded with:\n%s" % response.json) + assert response.status_code == 200 + values = response.json["values"] + # We expect two data points (from conftest) followed by 2 null values (which are converted to None by .json) + # The first data point averages [91.3, 91.7], and the second data point averages [92.1, None]. + assert all(a == b for a, b in zip(values, [91.5, 92.1, None, None])) + + +def test_get_instantaneous_sensor_data( + client, + setup_api_test_data: dict[str, Sensor], + setup_roles_users: dict[str, User], +): + """Check the /sensors/data endpoint for fetching 1 hour of data of an instantaneous sensor.""" + sensor = setup_api_test_data["some temperature sensor"] + source: Source = setup_roles_users["Test Supplier User"].data_source[0] + assert sensor.event_resolution == timedelta(minutes=0) + message = { + "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", + "start": "2021-05-02T00:00:00+02:00", + "duration": "PT1H20M", + "horizon": "PT0H", + "unit": "°C", + "source": source.id, + "resolution": "PT20M", + } + auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") + response = client.get( + url_for("SensorAPI:get_data"), + query_string=message, + headers={"content-type": "application/json", "Authorization": auth_token}, + ) + print("Server responded with:\n%s" % response.json) + assert response.status_code == 200 + values = response.json["values"] + # We expect two data point (from conftest) followed by 2 null values (which are converted to None by .json) + # The first data point is the first of [815, 817], and the second data point is the first of [818, None]. + assert all(a == b for a, b in zip(values, [815, 818, None, None])) + + @pytest.mark.parametrize("use_auth", [False, True]) def test_post_sensor_data_bad_auth(client, setup_api_test_data, use_auth): """ diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index ecb42b8fc..e082275f0 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -1,12 +1,10 @@ from __future__ import annotations -from datetime import timedelta - import pytest from flask import url_for from timely_beliefs.tests.utils import equal_lists -from flexmeasures import Sensor, Source, User +from flexmeasures import Sensor, Source from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor from flexmeasures.data.models.time_series import TimedBelief @@ -116,67 +114,3 @@ def test_auto_fix_missing_registration_of_user_as_data_source( user=setup_user_without_data_source ).one_or_none() assert data_source is not None - - -def test_get_sensor_data( - client, - setup_api_fresh_test_data: dict[str, Sensor], - setup_roles_users_fresh_db: dict[str, User], -): - """Check the /sensors/data endpoint for fetching 1 hour of data of a 10-minute resolution sensor.""" - sensor = setup_api_fresh_test_data["some gas sensor"] - source: Source = setup_roles_users_fresh_db["Test Supplier User"].data_source[0] - assert sensor.event_resolution == timedelta(minutes=10) - message = { - "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", - "start": "2021-05-02T00:00:00+02:00", - "duration": "PT1H20M", - "horizon": "PT0H", - "unit": "m³/h", - "source": source.id, - "resolution": "PT20M", - } - auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") - response = client.get( - url_for("SensorAPI:get_data"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % response.json) - assert response.status_code == 200 - values = response.json["values"] - # We expect two data points (from conftest) followed by 2 null values (which are converted to None by .json) - # The first data point averages [91.3, 91.7], and the second data point averages [92.1, None]. - assert all(a == b for a, b in zip(values, [91.5, 92.1, None, None])) - - -def test_get_instantaneous_sensor_data( - client, - setup_api_fresh_test_data: dict[str, Sensor], - setup_roles_users_fresh_db: dict[str, User], -): - """Check the /sensors/data endpoint for fetching 1 hour of data of an instantaneous sensor.""" - sensor = setup_api_fresh_test_data["some temperature sensor"] - source: Source = setup_roles_users_fresh_db["Test Supplier User"].data_source[0] - assert sensor.event_resolution == timedelta(minutes=0) - message = { - "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", - "start": "2021-05-02T00:00:00+02:00", - "duration": "PT1H20M", - "horizon": "PT0H", - "unit": "°C", - "source": source.id, - "resolution": "PT20M", - } - auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") - response = client.get( - url_for("SensorAPI:get_data"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % response.json) - assert response.status_code == 200 - values = response.json["values"] - # We expect two data point (from conftest) followed by 2 null values (which are converted to None by .json) - # The first data point is the first of [815, 817], and the second data point is the first of [818, None]. - assert all(a == b for a, b in zip(values, [815, 818, None, None])) From 3f37be16dcd5da05a462cfab7350f138dac755d2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 17:33:41 +0200 Subject: [PATCH 15/45] remove v1.1 test utils including old simulation script that uses one of them Signed-off-by: F.N. Claessen --- flexmeasures/api/v1_1/tests/__init__.py | 0 flexmeasures/api/v1_1/tests/conftest.py | 126 ------------ flexmeasures/api/v1_1/tests/utils.py | 185 ------------------ flexmeasures/data/scripts/_test_simulation.py | 75 ------- flexmeasures/data/scripts/simulation_utils.py | 129 ------------ 5 files changed, 515 deletions(-) delete mode 100644 flexmeasures/api/v1_1/tests/__init__.py delete mode 100644 flexmeasures/api/v1_1/tests/conftest.py delete mode 100644 flexmeasures/api/v1_1/tests/utils.py delete mode 100644 flexmeasures/data/scripts/_test_simulation.py delete mode 100644 flexmeasures/data/scripts/simulation_utils.py diff --git a/flexmeasures/api/v1_1/tests/__init__.py b/flexmeasures/api/v1_1/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/flexmeasures/api/v1_1/tests/conftest.py b/flexmeasures/api/v1_1/tests/conftest.py deleted file mode 100644 index 55d7063aa..000000000 --- a/flexmeasures/api/v1_1/tests/conftest.py +++ /dev/null @@ -1,126 +0,0 @@ -from typing import List -import pytest -from datetime import timedelta - -import isodate -from flask_security import SQLAlchemySessionUserDatastore -from flask_security.utils import hash_password - -from flexmeasures.data.models.data_sources import DataSource -from flexmeasures.data.models.time_series import TimedBelief -from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorType - - -@pytest.fixture(scope="module") -def setup_api_test_data(db, setup_accounts, setup_roles_users, add_market_prices): - """ - Set up data for API v1.1 tests. - """ - print("Setting up data for API v1.1 tests on %s" % db.engine) - - from flexmeasures.data.models.user import User, Role - from flexmeasures.data.models.assets import Asset, AssetType - - user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) - - # Create a user without proper registration as a data source - user_datastore.create_user( - username="test user with improper registration", - email="test_improper_user@seita.nl", - password=hash_password("testtest"), - account_id=setup_accounts["Prosumer"].id, - ) - - # Create 3 test assets for the test_user - test_user = setup_roles_users["Test Prosumer User"] - test_asset_type = AssetType(name="test-type") - db.session.add(test_asset_type) - asset_names = ["CS 1", "CS 2", "CS 3"] - assets: List[Asset] = [] - for asset_name in asset_names: - asset = Asset( - name=asset_name, - owner_id=test_user.id, - asset_type_name="test-type", - event_resolution=timedelta(minutes=15), - capacity_in_mw=1, - latitude=100, - longitude=100, - unit="MW", - ) - assets.append(asset) - db.session.add(asset) - - # Add power forecasts to the assets - cs_1 = Asset.query.filter(Asset.name == "CS 1").one_or_none() - cs_2 = Asset.query.filter(Asset.name == "CS 2").one_or_none() - cs_3 = Asset.query.filter(Asset.name == "CS 3").one_or_none() - data_source = DataSource.query.filter(DataSource.user == test_user).one_or_none() - cs1_beliefs = [ - TimedBelief( - event_start=isodate.parse_datetime("2015-01-01T00:00:00Z") - + timedelta(minutes=15 * i), - belief_horizon=timedelta(hours=6), - event_value=(300 + i) * -1, - sensor=cs_1.corresponding_sensor, - source=data_source, - ) - for i in range(6) - ] - cs2_beliefs = [ - TimedBelief( - event_start=isodate.parse_datetime("2015-01-01T00:00:00Z") - + timedelta(minutes=15 * i), - belief_horizon=timedelta(hours=6), - event_value=(300 - i) * -1, - sensor=cs_2.corresponding_sensor, - source=data_source, - ) - for i in range(6) - ] - cs3_beliefs = [ - TimedBelief( - event_start=isodate.parse_datetime("2015-01-01T00:00:00Z") - + timedelta(minutes=15 * i), - belief_horizon=timedelta(hours=6), - event_value=(0 + i) * -1, - sensor=cs_3.corresponding_sensor, - source=data_source, - ) - for i in range(6) - ] - db.session.add_all(cs1_beliefs + cs2_beliefs + cs3_beliefs) - - add_legacy_weather_sensors(db) - print("Done setting up data for API v1.1 tests") - - -@pytest.fixture(scope="function") -def setup_fresh_api_v1_1_test_data( - fresh_db, setup_roles_users_fresh_db, setup_markets_fresh_db -): - add_legacy_weather_sensors(fresh_db) - return fresh_db - - -def add_legacy_weather_sensors(db): - test_sensor_type = WeatherSensorType(name="wind speed") - db.session.add(test_sensor_type) - wind_sensor = WeatherSensor( - name="wind_speed_sensor", - weather_sensor_type_name="wind speed", - event_resolution=timedelta(minutes=5), - latitude=33.4843866, - longitude=126, - ) - db.session.add(wind_sensor) - test_sensor_type2 = WeatherSensorType(name="temperature") - db.session.add(test_sensor_type2) - temperature_sensor = WeatherSensor( - name="temperature_sensor", - weather_sensor_type_name="temperature", - event_resolution=timedelta(minutes=5), - latitude=33.4843866, - longitude=126, - ) - db.session.add(temperature_sensor) diff --git a/flexmeasures/api/v1_1/tests/utils.py b/flexmeasures/api/v1_1/tests/utils.py deleted file mode 100644 index bb21cb982..000000000 --- a/flexmeasures/api/v1_1/tests/utils.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Useful test messages""" -from typing import Optional, Dict, Any, List, Union -from datetime import timedelta -from isodate import parse_datetime, parse_duration - -import pandas as pd -from numpy import tile -from rq.job import Job -from flask import current_app - -from flexmeasures.api.common.schemas.sensors import SensorField -from flexmeasures.data.models.time_series import Sensor, TimedBelief -from flexmeasures.utils.time_utils import duration_isoformat - - -def message_for_get_prognosis( - invalid_horizon=False, - rolling_horizon=False, - with_prior=False, - no_data=False, - no_resolution=False, - single_connection=False, - timezone_alternative=False, -) -> dict: - message = { - "type": "GetPrognosisRequest", - "start": "2015-01-01T00:00:00Z", - "duration": "PT1H30M", - "horizon": "PT6H", - "resolution": "PT15M", - "connections": ["CS 1", "CS 2", "CS 3"], - "unit": "MW", - } - if invalid_horizon: - message["horizon"] = "T6H" - elif rolling_horizon: - message[ - "horizon" - ] = "R/PT6H" # with or without R/ shouldn't matter: both are interpreted as rolling horizons - if with_prior: - message["prior"] = ("2015-03-01T00:00:00Z",) - if no_data: - message["start"] = ("2010-01-01T00:00:00Z",) - if no_resolution: - message.pop("resolution", None) - if single_connection: - message["connection"] = message["connections"][0] - message.pop("connections", None) - if timezone_alternative: - message["start"] = ("2015-01-01T00:00:00+00:00",) - return message - - -def message_for_post_price_data( - tile_n: int = 1, - compress_n: int = 1, - duration: Optional[Union[timedelta, str]] = None, - invalid_unit: bool = False, -) -> dict: - """ - The default message has 24 hourly values. - - :param tile_n: Tile the price profile back to back to obtain price data for n days (default = 1). - :param compress_n: Compress the price profile to obtain price data with a coarser resolution (default = 1), - e.g. compress=4 leads to a resolution of 4 hours. - :param duration: timedelta or iso8601 string - Set a duration explicitly to obtain price data with a coarser or finer resolution - (the default is equal to 24 hours * tile_n), - e.g. (assuming tile_n=1) duration=timedelta(hours=6) leads to a resolution of 15 minutes, - and duration=timedelta(hours=48) leads to a resolution of 2 hours. - :param invalid_unit: Choose an invalid unit for the test market (epex_da). - """ - message = { - "type": "PostPriceDataRequest", - "market": "ea1.2018-06.localhost:epex_da", - "values": tile( - [ - 52.37, - 51.14, - 49.09, - 48.35, - 48.47, - 49.98, - 58.7, - 67.76, - 69.21, - 70.26, - 70.46, - 70, - 70.7, - 70.41, - 70, - 64.53, - 65.92, - 69.72, - 70.51, - 75.49, - 70.35, - 70.01, - 66.98, - 58.61, - ], - tile_n, - ).tolist(), - "start": "2021-01-06T00:00:00+01:00", - "duration": duration_isoformat(timedelta(hours=24 * tile_n)), - "horizon": duration_isoformat(timedelta(hours=11 + 24 * tile_n)), - "unit": "EUR/MWh", - } - if duration is not None: - message["duration"] = ( - duration_isoformat(duration) - if isinstance(duration, timedelta) - else duration - ) - if compress_n > 1: - message["values"] = message["values"][::compress_n] - if invalid_unit: - message["unit"] = "KRW/kWh" # That is, an invalid unit for EPEX SPOT. - return message - - -def message_for_post_weather_data( - invalid_unit: bool = False, temperature: bool = False, as_forecasts: bool = True -) -> dict: - message: Dict[str, Any] = { - "type": "PostWeatherDataRequest", - "groups": [ - { - "sensor": "ea1.2018-06.localhost:wind speed:33.4843866:126", - "values": [20.04, 20.23, 20.41, 20.51, 20.55, 20.57], - } - ], - "start": "2015-01-01T15:00:00+09:00", - "duration": "PT30M", - "horizon": "PT3H", - "unit": "m/s", - } - if temperature: - message["groups"][0][ - "sensor" - ] = "ea1.2018-06.localhost:temperature:33.4843866:126" - if not invalid_unit: - message["unit"] = "°C" # Right unit for temperature - elif invalid_unit: - message["unit"] = "°C" # Wrong unit for wind speed - if not as_forecasts: - message["horizon"] = "PT0H" # weather measurements - return message - - -def verify_prices_in_db(post_message, values, db, swapped_sign: bool = False): - """util method to verify that price data ended up in the database""" - start = parse_datetime(post_message["start"]) - end = start + parse_duration(post_message["duration"]) - horizon = parse_duration(post_message["horizon"]) - sensor = SensorField("market", "fm0").deserialize(post_message["market"]) - resolution = sensor.event_resolution - query = ( - db.session.query(TimedBelief.event_value, TimedBelief.belief_horizon) - .filter( - (TimedBelief.event_start > start - resolution) - & (TimedBelief.event_start < end) - ) - .filter( - TimedBelief.belief_horizon - == horizon - (end - (TimedBelief.event_start + resolution)) - ) - .join(Sensor) - .filter(TimedBelief.sensor_id == Sensor.id) - .filter(Sensor.name == sensor.name) - ) - df = pd.DataFrame( - query.all(), columns=[col["name"] for col in query.column_descriptions] - ) - if swapped_sign: - df["event_value"] = -df["event_value"] - assert df["event_value"].tolist() == values - - -def get_forecasting_jobs(last_n: Optional[int] = None) -> List[Job]: - """Get all or last n forecasting jobs.""" - if last_n: - return current_app.queues["forecasting"].jobs[-last_n:] - return current_app.queues["forecasting"].jobs diff --git a/flexmeasures/data/scripts/_test_simulation.py b/flexmeasures/data/scripts/_test_simulation.py deleted file mode 100644 index ac445b222..000000000 --- a/flexmeasures/data/scripts/_test_simulation.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Tests a small simulation against FlexMeasures running on a server.""" -import sys -from datetime import timedelta - -from isodate import parse_datetime - -from flexmeasures.data.scripts.simulation_utils import ( - check_version, - check_services, - get_auth_token, - get_connections, - post_meter_data, - post_price_forecasts, - post_weather_data, -) - - -# Setup host - pass hostname to overwrite -host = "http://localhost:5000" -if len(sys.argv) > 1: - host = sys.argv[1] - -latest_version = check_version(host) -services = check_services(host, latest_version) -auth_token = get_auth_token(host) -connections = get_connections(host, latest_version, auth_token) - -# Initialisation -num_days = 50 -sim_start = "2018-01-01T00:00:00+09:00" -post_price_forecasts( - host, latest_version, auth_token, start=parse_datetime(sim_start), num_days=num_days -) -post_weather_data( - host, latest_version, auth_token, start=parse_datetime(sim_start), num_days=num_days -) -post_meter_data( - host, - latest_version, - auth_token, - start=parse_datetime(sim_start), - num_days=num_days, - connection=connections[1], -) - -# Run forecasting jobs -input( - "Run all forecasting jobs, then press Enter to continue ...\n" - "You can run this in another flexmeasures-venv shell:\n\n" - "flask run_forecasting_worker\n" -) - -# Steps -num_steps = 10 -for i in range(num_steps): - print("Running step %s out of %s" % (i, num_steps)) - - post_weather_data( - host, - latest_version, - auth_token, - start=parse_datetime(sim_start) + timedelta(days=num_days + i), - num_days=1, - ) - post_meter_data( - host, - latest_version, - auth_token, - start=parse_datetime(sim_start) + timedelta(days=num_days + i), - num_days=1, - connection=connections[1], - ) - - # Run forecasting jobs - input("Run all forecasting jobs again and press enter to continue..\n") diff --git a/flexmeasures/data/scripts/simulation_utils.py b/flexmeasures/data/scripts/simulation_utils.py deleted file mode 100644 index 6601eb1cf..000000000 --- a/flexmeasures/data/scripts/simulation_utils.py +++ /dev/null @@ -1,129 +0,0 @@ -from typing import List, Optional -import requests -from random import random -from datetime import datetime, timedelta - -from numpy import sin, tile -from isodate import duration_isoformat, datetime_isoformat - -from flexmeasures.utils.entity_address_utils import build_ea_scheme_and_naming_authority -from flexmeasures.api.v1.tests.utils import message_for_post_meter_data -from flexmeasures.api.v1_1.tests.utils import message_for_post_price_data - - -def check_version(host: str) -> str: - response = requests.get("%s/api/" % host) - latest_version = response.json()["versions"][-1] - print("Latest API version on host %s is %s." % (host, latest_version)) - return latest_version - - -def check_services(host: str, latest_version: str) -> List[str]: - response = requests.get("%s/api/%s/getService" % (host, latest_version)) - services = [service["name"] for service in response.json()["services"]] - for service in ( - "getConnection", - "postWeatherData", - "postPriceData", - "postMeterData", - "getPrognosis", - "postUdiEvent", - "getDeviceMessage", - ): - assert service in services - return services - - -def get_auth_token(host: str) -> str: - response = requests.post( - "%s/api/requestAuthToken" % host, - json={"email": "solar@seita.nl", "password": "solar"}, - ) - return response.json()["auth_token"] - - -def get_connections(host: str, latest_version: str, auth_token: str) -> List[str]: - response = requests.get( - "%s/api/%s/getConnection" % (host, latest_version), - headers={"Authorization": auth_token}, - ) - return response.json()["connections"] - - -def post_meter_data( - host: str, - latest_version: str, - auth_token: str, - start: datetime, - num_days: int, - connection: str, -): - message = message_for_post_meter_data( - tile_n=num_days * 16, production=True - ) # Original message is just 1.5 hours - message["start"] = datetime_isoformat(start) - message["connection"] = connection - response = requests.post( - "%s/api/%s/postMeterData" % (host, latest_version), - headers={"Authorization": auth_token}, - json=message, - ) - assert response.status_code == 200 - - -def post_price_forecasts( - host: str, - latest_version: str, - auth_token: str, - start: datetime, - num_days: int, - host_auth_start_month: Optional[str] = None, -): - market_ea = "%s:%s" % ( - build_ea_scheme_and_naming_authority(host, host_auth_start_month), - "kpx_da", - ) - message = message_for_post_price_data(tile_n=num_days) - message["start"] = datetime_isoformat(start) - message["market"] = market_ea - message["unit"] = "KRW/kWh" - response = requests.post( - "%s/api/%s/postPriceData" % (host, latest_version), - headers={"Authorization": auth_token}, - json=message, - ) - assert response.status_code == 200 - - -def post_weather_data( - host: str, - latest_version: str, - auth_token: str, - start: datetime, - num_days: int, - host_auth_start_month: Optional[str] = None, -): - lat = 33.4843866 - lng = 126 - values = [random() * 600 * (1 + sin(x / 15)) for x in range(96 * num_days)] - message = { - "type": "PostWeatherDataRequest", - "sensor": "%s:%s:%s:%s" - % ( - build_ea_scheme_and_naming_authority(host, host_auth_start_month), - "irradiance", - lat, - lng, - ), - "values": tile(values, 1).tolist(), - "start": datetime_isoformat(start), - "duration": duration_isoformat(timedelta(hours=24 * num_days)), - "horizon": "R/PT0H", - "unit": "kW/m²", - } - response = requests.post( - "%s/api/%s/postWeatherData" % (host, latest_version), - headers={"Authorization": auth_token}, - json=message, - ) - assert response.status_code == 200 From 900d9e2a588fec0ba5b51ee77acb976f76cbdcf3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 17:46:37 +0200 Subject: [PATCH 16/45] salvage only 1 test from v1, which logs out a user Signed-off-by: F.N. Claessen --- flexmeasures/api/v1/tests/test_api_v1.py | 277 ------------------ .../api/v1/tests/test_api_v1_fresh_db.py | 105 ------- .../api/v3_0/tests/test_api_v3_0_users.py | 13 + 3 files changed, 13 insertions(+), 382 deletions(-) delete mode 100644 flexmeasures/api/v1/tests/test_api_v1.py delete mode 100644 flexmeasures/api/v1/tests/test_api_v1_fresh_db.py diff --git a/flexmeasures/api/v1/tests/test_api_v1.py b/flexmeasures/api/v1/tests/test_api_v1.py deleted file mode 100644 index 3989d1d2c..000000000 --- a/flexmeasures/api/v1/tests/test_api_v1.py +++ /dev/null @@ -1,277 +0,0 @@ -from datetime import timedelta - -from flask import url_for -import isodate -import pandas as pd -import pytest - -from flexmeasures.api.common.responses import ( - invalid_domain, - invalid_sender, - invalid_unit, - request_processed, - unrecognized_connection_group, -) -from flexmeasures.api.tests.utils import check_deprecation, get_auth_token -from flexmeasures.api.common.utils.api_utils import message_replace_name_with_ea -from flexmeasures.api.common.utils.validators import validate_user_sources -from flexmeasures.api.v1.tests.utils import ( - message_for_get_meter_data, - message_for_post_meter_data, - verify_power_in_db, -) -from flexmeasures.auth.error_handling import UNAUTH_ERROR_STATUS -from flexmeasures.data.models.time_series import Sensor - - -@pytest.mark.parametrize("query", [{}, {"access": "Prosumer"}]) -def test_get_service(client, query): - get_service_response = client.get( - url_for("flexmeasures_api_v1.get_service"), - query_string=query, - headers={"content-type": "application/json"}, - ) - print("Server responded with:\n%s" % get_service_response.json) - check_deprecation(get_service_response) - assert get_service_response.status_code == 200 - assert get_service_response.json["type"] == "GetServiceResponse" - assert get_service_response.json["status"] == request_processed()[0]["status"] - if "access" in query: - for service in get_service_response.json["services"]: - assert "Prosumer" in service["access"] - - -def test_unauthorized_request(client): - get_meter_data_response = client.get( - url_for("flexmeasures_api_v1.get_meter_data"), - query_string=message_for_get_meter_data(no_connection=True), - headers={"content-type": "application/json"}, - ) - print("Server responded with:\n%s" % get_meter_data_response.json) - check_deprecation(get_meter_data_response) - assert get_meter_data_response.status_code == 401 - assert get_meter_data_response.json["type"] == "GetMeterDataResponse" - assert get_meter_data_response.json["status"] == UNAUTH_ERROR_STATUS - - -def test_no_connection_in_get_request(client): - get_meter_data_response = client.get( - url_for("flexmeasures_api_v1.get_meter_data"), - query_string=message_for_get_meter_data(no_connection=True), - headers={ - "Authorization": get_auth_token( - client, "test_prosumer_user@seita.nl", "testtest" - ) - }, - ) - print("Server responded with:\n%s" % get_meter_data_response.json) - check_deprecation(get_meter_data_response) - assert get_meter_data_response.status_code == 400 - assert get_meter_data_response.json["type"] == "GetMeterDataResponse" - assert ( - get_meter_data_response.json["status"] - == unrecognized_connection_group()[0]["status"] - ) - - -def test_invalid_connection_in_get_request(client): - get_meter_data_response = client.get( - url_for("flexmeasures_api_v1.get_meter_data"), - query_string=message_for_get_meter_data(invalid_connection=True), - headers={ - "Authorization": get_auth_token( - client, "test_prosumer_user@seita.nl", "testtest" - ) - }, - ) - print("Server responded with:\n%s" % get_meter_data_response.json) - check_deprecation(get_meter_data_response) - assert get_meter_data_response.status_code == 400 - assert get_meter_data_response.json["type"] == "GetMeterDataResponse" - assert get_meter_data_response.json["status"] == invalid_domain()[0]["status"] - - -@pytest.mark.parametrize("method", ["GET", "POST"]) -@pytest.mark.parametrize( - "message", - [ - message_for_get_meter_data(no_unit=True), - message_for_get_meter_data(invalid_unit=True), - ], -) -def test_invalid_or_no_unit(client, method, message): - if method == "GET": - get_meter_data_response = client.get( - url_for("flexmeasures_api_v1.get_meter_data"), - query_string=message, - headers={ - "Authorization": get_auth_token( - client, "test_prosumer_user@seita.nl", "testtest" - ) - }, - ) - elif method == "POST": - get_meter_data_response = client.post( - url_for("flexmeasures_api_v1.get_meter_data"), - json=message, - headers={ - "Authorization": get_auth_token( - client, "test_prosumer_user@seita.nl", "testtest" - ) - }, - ) - check_deprecation(get_meter_data_response) - else: - get_meter_data_response = [] - assert get_meter_data_response.status_code == 400 - assert get_meter_data_response.json["type"] == "GetMeterDataResponse" - assert ( - get_meter_data_response.json["status"] - == invalid_unit("power", ["MW"])[0]["status"] - ) - - -@pytest.mark.parametrize( - "user_email, get_message", - [ - ["test_dummy_user_3@seita.nl", message_for_get_meter_data()], - ["demo@seita.nl", message_for_get_meter_data(demo_connection=True)], - ], -) -def test_invalid_sender_and_logout(client, user_email, get_message): - """ - Tries to get meter data as a logged-in test user without any suitable - account role, which should fail. - Then tries to log out, which should succeed as a url direction. - """ - - # get meter data - auth_token = get_auth_token(client, user_email, "testtest") - get_meter_data_response = client.get( - url_for("flexmeasures_api_v1.get_meter_data"), - query_string=get_message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_meter_data_response.json) - check_deprecation(get_meter_data_response) - assert get_meter_data_response.status_code == 403 - assert get_meter_data_response.json["status"] == invalid_sender()[0]["status"] - - # log out - logout_response = client.get( - url_for("security.logout"), - headers={"Authorization ": auth_token, "content-type": "application/json"}, - ) - assert logout_response.status_code == 302 - - -def test_invalid_resolution_str(client): - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - query_string = message_for_get_meter_data() - query_string["resolution"] = "15M" # invalid - get_meter_data_response = client.get( - url_for("flexmeasures_api_v1.get_meter_data"), - query_string=query_string, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_meter_data_response.json) - check_deprecation(get_meter_data_response) - assert get_meter_data_response.status_code == 400 - assert get_meter_data_response.json["type"] == "GetMeterDataResponse" - assert get_meter_data_response.json["status"] == "INVALID_RESOLUTION" - - -@pytest.mark.parametrize( - "message", - [ - message_for_get_meter_data(single_connection=True), - message_for_get_meter_data( - single_connection=True, source="Prosumer" - ), # sourced by a Prosumer - message_for_get_meter_data( - single_connection=True, source=["Prosumer", 109304] - ), # sourced by a Prosumer or user 109304 - ], -) -def test_get_meter_data(db, app, client, message): - """Checks Charging Station 5, which has multi-sourced data for the same time interval: - 6 values from a Prosumer, and 6 values from a Supplier. - - All data should be in the database, and currently only the Prosumer data is returned. - """ - message["connection"] = "CS 5" - - # set up frame with expected values, and filter by source if needed - expected_values = pd.concat( - [ - pd.DataFrame.from_dict( - dict( - event_value=[(100.0 + i) for i in range(6)], - event_start=[ - isodate.parse_datetime("2015-01-01T00:00:00Z") - + timedelta(minutes=15 * i) - for i in range(6) - ], - source_id=1, - ) - ), - pd.DataFrame.from_dict( - dict( - event_value=[(1000.0 - 10 * i) for i in range(6)], - event_start=[ - isodate.parse_datetime("2015-01-01T00:00:00Z") - + timedelta(minutes=15 * i) - for i in range(6) - ], - source_id=2, - ) - ), - ] - ) - if "source" in message: - source_ids = validate_user_sources(message["source"]) - expected_values = expected_values[expected_values["source_id"].isin(source_ids)] - expected_values = expected_values.set_index( - ["event_start", "source_id"] - ).sort_index() - - # check whether conftest.py did its job setting up the database with expected values - cs_5 = Sensor.query.filter(Sensor.name == "CS 5").one_or_none() - verify_power_in_db(message, cs_5, expected_values, db, swapped_sign=True) - - # check whether the API returns the expected values (currently only the Prosumer data is returned) - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - get_meter_data_response = client.get( - url_for("flexmeasures_api_v1.get_meter_data"), - query_string=message_replace_name_with_ea(message), - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_meter_data_response.json) - check_deprecation(get_meter_data_response) - assert get_meter_data_response.status_code == 200 - assert get_meter_data_response.json["values"] == [(100.0 + i) for i in range(6)] - - -def test_post_meter_data_to_different_resolutions(app, client): - """ - Tries to post meter data to assets with different event_resolutions, which is not accepted. - """ - - post_message = message_for_post_meter_data(different_target_resolutions=True) - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - post_meter_data_response = client.post( - url_for("flexmeasures_api_v1.post_meter_data"), - json=message_replace_name_with_ea(post_message), - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_meter_data_response.json) - check_deprecation(post_meter_data_response) - assert post_meter_data_response.json["type"] == "PostMeterDataResponse" - assert post_meter_data_response.status_code == 400 - assert ( - "assets do not have matching resolutions" - in post_meter_data_response.json["message"] - ) - assert "CS 2" in post_meter_data_response.json["message"] - assert "CS 4" in post_meter_data_response.json["message"] - assert post_meter_data_response.json["status"] == "INVALID_RESOLUTION" diff --git a/flexmeasures/api/v1/tests/test_api_v1_fresh_db.py b/flexmeasures/api/v1/tests/test_api_v1_fresh_db.py deleted file mode 100644 index 68c62a8b6..000000000 --- a/flexmeasures/api/v1/tests/test_api_v1_fresh_db.py +++ /dev/null @@ -1,105 +0,0 @@ -from datetime import timedelta - -import pytest -from flask import url_for -from iso8601 import parse_date -from numpy import repeat - -from flexmeasures.api.common.utils.api_utils import message_replace_name_with_ea -from flexmeasures.api.tests.utils import check_deprecation, get_auth_token -from flexmeasures.api.v1.tests.utils import ( - message_for_post_meter_data, - message_for_get_meter_data, - count_connections_in_post_message, -) -from flexmeasures.data.models.time_series import Sensor - - -@pytest.mark.parametrize( - "post_message", - [ - message_for_post_meter_data(), - message_for_post_meter_data(single_connection=True), - message_for_post_meter_data(single_connection_group=True), - ], -) -@pytest.mark.parametrize( - "get_message", - [ - message_for_get_meter_data(), - message_for_get_meter_data(single_connection=False), - message_for_get_meter_data(resolution="PT30M"), - ], -) -def test_post_and_get_meter_data( - setup_fresh_api_test_data, app, clean_redis, client, post_message, get_message -): - """ - Tries to post meter data as a logged-in test user with the MDC role, which should succeed. - There should be some ForecastingJobs waiting now. - Then tries to get meter data, which should succeed, and should return the same meter data as was posted, - or a downsampled version, if that was requested. - """ - - # post meter data - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") - post_meter_data_response = client.post( - url_for("flexmeasures_api_v1.post_meter_data"), - json=message_replace_name_with_ea(post_message), - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_meter_data_response.json) - check_deprecation(post_meter_data_response) - assert post_meter_data_response.status_code == 200 - assert post_meter_data_response.json["type"] == "PostMeterDataResponse" - - # look for Forecasting jobs - expected_connections = count_connections_in_post_message(post_message) - assert ( - len(app.queues["forecasting"]) == 4 * expected_connections - ) # four horizons times the number of assets - horizons = repeat( - [ - timedelta(hours=1), - timedelta(hours=6), - timedelta(hours=24), - timedelta(hours=48), - ], - expected_connections, - ) - jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"]) - for job, horizon in zip(jobs, horizons): - assert job.kwargs["horizon"] == horizon - assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon - for asset_name in ("CS 1", "CS 2", "CS 3"): - if asset_name in str(post_message): - sensor = Sensor.query.filter_by(name=asset_name).one_or_none() - assert sensor.id in [job.kwargs["sensor_id"] for job in jobs] - - # get meter data - get_meter_data_response = client.get( - url_for("flexmeasures_api_v1.get_meter_data"), - query_string=message_replace_name_with_ea(get_message), - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_meter_data_response.json) - assert get_meter_data_response.status_code == 200 - assert get_meter_data_response.json["type"] == "GetMeterDataResponse" - if "groups" in post_message: - posted_values = post_message["groups"][0]["values"] - else: - posted_values = post_message["values"] - if "groups" in get_meter_data_response.json: - gotten_values = get_meter_data_response.json["groups"][0]["values"] - else: - gotten_values = get_meter_data_response.json["values"] - - if "resolution" not in get_message or get_message["resolution"] == "": - assert gotten_values == posted_values - else: - # We used a target resolution of 30 minutes, so double of 15 minutes. - # Six values went in, three come out. - if posted_values[1] > 0: # see utils.py:message_for_post_meter_data - assert gotten_values == [306.66, -0.0, 306.66] - else: - assert gotten_values == [153.33, 0, 306.66] diff --git a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py index 35093cd0d..c5eea4220 100644 --- a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py +++ b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py @@ -153,3 +153,16 @@ def test_edit_user_with_unexpected_fields(client, unexpected_fields: dict): ) print("Server responded with:\n%s" % user_edit_response.json) assert user_edit_response.status_code == 422 + + +def test_logout(client, setup_api_test_data): + """Tries to log out, which should succeed as a url direction.""" + + # log out + with UserContext("test_admin_user@seita.nl") as admin: + auth_token = admin.get_auth_token() + logout_response = client.get( + url_for("security.logout"), + headers={"Authorization ": auth_token, "content-type": "application/json"}, + ) + assert logout_response.status_code == 302 From 73cfe29e85482ec76768bde8443a715c1261a73d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 17:48:48 +0200 Subject: [PATCH 17/45] add missing fixtures to let tests succeed on their own, too Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_api_v3_0_users.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py index c5eea4220..a70072eb4 100644 --- a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py +++ b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py @@ -32,7 +32,9 @@ def test_get_users_bad_auth(client, setup_api_test_data, use_auth): @pytest.mark.parametrize("include_inactive", [False, True]) -def test_get_users_inactive(client, setup_inactive_user, include_inactive): +def test_get_users_inactive( + client, setup_api_test_data, setup_inactive_user, include_inactive +): headers = { "content-type": "application/json", "Authorization": get_auth_token( @@ -64,7 +66,7 @@ def test_get_users_inactive(client, setup_inactive_user, include_inactive): ("test_admin_user@seita.nl", 200), # admin can do this from another account ], ) -def test_get_one_user(client, requesting_user, status_code): +def test_get_one_user(client, setup_api_test_data, requesting_user, status_code): test_user2_id = find_user_by_email("test_prosumer_user_2@seita.nl").id headers = {"content-type": "application/json"} if requesting_user: @@ -80,7 +82,7 @@ def test_get_one_user(client, requesting_user, status_code): assert get_user_response.json["username"] == "Test Prosumer User 2" -def test_edit_user(client): +def test_edit_user(client, setup_api_test_data): with UserContext("test_prosumer_user_2@seita.nl") as user2: user2_auth_token = user2.get_auth_token() # user2 is no admin user2_id = user2.id @@ -137,7 +139,9 @@ def test_edit_user(client): dict(account_id=10), # account_id is a dump_only field ], ) -def test_edit_user_with_unexpected_fields(client, unexpected_fields: dict): +def test_edit_user_with_unexpected_fields( + client, setup_api_test_data, unexpected_fields: dict +): """Sending unexpected fields (not in Schema) is an Unprocessable Entity error.""" with UserContext("test_prosumer_user_2@seita.nl") as user2: user2_id = user2.id From 0a7fd46a9dbcd756ef85bc4a901e79c4b072cfc8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 17:55:11 +0200 Subject: [PATCH 18/45] remove v1.0 test utils Signed-off-by: F.N. Claessen --- flexmeasures/api/v1/tests/__init__.py | 0 flexmeasures/api/v1/tests/conftest.py | 133 ------------------------ flexmeasures/api/v1/tests/utils.py | 143 -------------------------- 3 files changed, 276 deletions(-) delete mode 100644 flexmeasures/api/v1/tests/__init__.py delete mode 100644 flexmeasures/api/v1/tests/conftest.py delete mode 100644 flexmeasures/api/v1/tests/utils.py diff --git a/flexmeasures/api/v1/tests/__init__.py b/flexmeasures/api/v1/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/flexmeasures/api/v1/tests/conftest.py b/flexmeasures/api/v1/tests/conftest.py deleted file mode 100644 index 212ef9406..000000000 --- a/flexmeasures/api/v1/tests/conftest.py +++ /dev/null @@ -1,133 +0,0 @@ -from typing import List -from datetime import timedelta - -import isodate -import pytest - -from flexmeasures.data.services.users import create_user -from flexmeasures.data.models.time_series import TimedBelief - - -@pytest.fixture(scope="module", autouse=True) -def setup_api_test_data(db, setup_accounts, setup_roles_users, add_market_prices): - """ - Set up data for API v1 tests. - """ - print("Setting up data for API v1 tests on %s" % db.engine) - - from flexmeasures.data.models.assets import Asset, AssetType - from flexmeasures.data.models.data_sources import DataSource - - # Create an anonymous user TODO: used for demo purposes, maybe "demo-user" would be a better name - test_anonymous_user = create_user( - username="anonymous user", - email="demo@seita.nl", - password="testtest", - account_name=setup_accounts["Dummy"].name, - user_roles=[ - dict(name="anonymous", description="Anonymous test user"), - ], - ) - - # Create 1 test asset for the anonymous user - test_asset_type = AssetType(name="test-type") - db.session.add(test_asset_type) - asset_names = ["CS 0"] - assets: List[Asset] = [] - for asset_name in asset_names: - asset = Asset( - name=asset_name, - owner_id=test_anonymous_user.id, - asset_type_name="test-type", - event_resolution=timedelta(minutes=15), - capacity_in_mw=1, - latitude=100, - longitude=100, - unit="MW", - ) - assets.append(asset) - db.session.add(asset) - - # Create 5 test assets for the test user - test_user = setup_roles_users["Test Prosumer User"] - asset_names = ["CS 1", "CS 2", "CS 3", "CS 4", "CS 5"] - assets: List[Asset] = [] - for asset_name in asset_names: - asset = Asset( - name=asset_name, - owner_id=test_user.id, - asset_type_name="test-type", - event_resolution=timedelta(minutes=15) - if not asset_name == "CS 4" - else timedelta(hours=1), - capacity_in_mw=1, - latitude=100, - longitude=100, - unit="MW", - ) - assets.append(asset) - db.session.add(asset) - - # Add power forecasts to one of the assets, for two sources - cs_5 = Asset.query.filter(Asset.name == "CS 5").one_or_none() - user1_data_source = DataSource.query.filter( - DataSource.user == test_user - ).one_or_none() - test_user_2 = setup_roles_users["Test Prosumer User 2"] - user2_data_source = DataSource.query.filter( - DataSource.user == test_user_2 - ).one_or_none() - user1_beliefs = [ - TimedBelief( - event_start=isodate.parse_datetime("2015-01-01T00:00:00Z") - + timedelta(minutes=15 * i), - belief_horizon=timedelta(0), - event_value=(100.0 + i) * -1, - sensor=cs_5.corresponding_sensor, - source=user1_data_source, - ) - for i in range(6) - ] - user2_beliefs = [ - TimedBelief( - event_start=isodate.parse_datetime("2015-01-01T00:00:00Z") - + timedelta(minutes=15 * i), - belief_horizon=timedelta(hours=0), - event_value=(1000.0 - 10 * i) * -1, - sensor=cs_5.corresponding_sensor, - source=user2_data_source, - ) - for i in range(6) - ] - db.session.add_all(user1_beliefs + user2_beliefs) - - print("Done setting up data for API v1 tests") - - -@pytest.fixture(scope="function") -def setup_fresh_api_test_data(fresh_db, setup_roles_users_fresh_db): - db = fresh_db - setup_roles_users = setup_roles_users_fresh_db - from flexmeasures.data.models.assets import Asset, AssetType - - # Create 5 test assets for the test_prosumer user - test_user = setup_roles_users["Test Prosumer User"] - test_asset_type = AssetType(name="test-type") - db.session.add(test_asset_type) - asset_names = ["CS 1", "CS 2", "CS 3", "CS 4", "CS 5"] - assets: List[Asset] = [] - for asset_name in asset_names: - asset = Asset( - name=asset_name, - owner_id=test_user.id, - asset_type_name="test-type", - event_resolution=timedelta(minutes=15) - if not asset_name == "CS 4" - else timedelta(hours=1), - capacity_in_mw=1, - latitude=100, - longitude=100, - unit="MW", - ) - assets.append(asset) - db.session.add(asset) diff --git a/flexmeasures/api/v1/tests/utils.py b/flexmeasures/api/v1/tests/utils.py deleted file mode 100644 index a9ef31e91..000000000 --- a/flexmeasures/api/v1/tests/utils.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Useful test messages""" -from datetime import timedelta -from typing import List, Optional, Union - -from isodate import duration_isoformat, parse_datetime, parse_duration -from numpy import tile -import pandas as pd - -from flexmeasures.api.common.utils.validators import validate_user_sources -from flexmeasures.data.models.time_series import Sensor, TimedBelief - - -def message_for_get_meter_data( - no_connection: bool = False, - invalid_connection: bool = False, - single_connection: bool = False, - demo_connection: bool = False, - invalid_unit: bool = False, - no_unit: bool = False, - resolution: str = "", - source: Optional[Union[str, List[str]]] = None, -) -> dict: - message = { - "type": "GetMeterDataRequest", - "start": "2015-01-01T00:00:00Z", - "duration": "PT1H30M", - "connections": ["CS 1", "CS 2", "CS 3"], - "unit": "MW", - } - if no_connection: - message.pop("connections", None) - elif invalid_connection: - message["connections"] = ["Non-existing asset 1", "Non-existing asset 2"] - elif single_connection: - message["connection"] = message["connections"][0] - message.pop("connections", None) - elif demo_connection: - message["connection"] = "CS 0" - message.pop("connections", None) - if no_unit: - message.pop("unit", None) - elif invalid_unit: - message["unit"] = "MW/h" - if resolution: - message["resolution"] = resolution - if source: - message["source"] = source - return message - - -def message_for_post_meter_data( - no_connection: bool = False, - single_connection: bool = False, - single_connection_group: bool = False, - production: bool = False, - different_target_resolutions: bool = False, - tile_n=1, -) -> dict: - sign = 1 if production is False else -1 - message = { - "type": "PostMeterDataRequest", - "groups": [ - { - "connections": ["CS 1", "CS 2"], - "values": ( - tile([306.66, 306.66, 0, 0, 306.66, 306.66], tile_n) * sign - ).tolist(), - }, - { - "connection": ["CS 4" if different_target_resolutions else "CS 3"], - "values": ( - tile([306.66, 0, 0, 0, 306.66, 306.66], tile_n) * sign - ).tolist(), - }, - ], - "start": "2015-01-01T00:00:00Z", - "duration": duration_isoformat(timedelta(hours=1.5 * tile_n)), - "horizon": "PT0H", - "unit": "MW", - } - if no_connection: - message.pop("groups", None) - elif single_connection: - message["connection"] = message["groups"][0]["connections"][0] - message["values"] = message["groups"][1]["values"] - message.pop("groups", None) - elif single_connection_group: - message["connections"] = message["groups"][0]["connections"] - message["values"] = message["groups"][0]["values"] - message.pop("groups", None) - - return message - - -def count_connections_in_post_message(message: dict) -> int: - connections = 0 - if "groups" in message: - message = dict( - connections=message["groups"][0]["connections"], - connection=message["groups"][1]["connection"], - ) - if "connection" in message: - connections += 1 - if "connections" in message: - connections += len(message["connections"]) - return connections - - -def verify_power_in_db( - message, sensor: Sensor, expected_df: pd.DataFrame, db, swapped_sign: bool = False -): - """util method to verify that power data ended up in the database""" - # todo: combine with verify_prices_in_db (in v1_1 utils) into a single function (NB different horizon filters) - start = parse_datetime(message["start"]) - end = start + parse_duration(message["duration"]) - horizon = ( - parse_duration(message["horizon"]) if "horizon" in message else timedelta(0) - ) - resolution = sensor.event_resolution - query = ( - db.session.query( - TimedBelief.event_start, TimedBelief.event_value, TimedBelief.source_id - ) - .filter( - (TimedBelief.event_start > start - resolution) - & (TimedBelief.event_start < end) - ) - .filter(TimedBelief.belief_horizon == horizon) - .join(Sensor) - .filter(TimedBelief.sensor_id == Sensor.id) - .filter(Sensor.name == sensor.name) - ) - if "source" in message: - source_ids = validate_user_sources(message["source"]) - query = query.filter(TimedBelief.source_id.in_(source_ids)) - df = pd.DataFrame( - query.all(), columns=[col["name"] for col in query.column_descriptions] - ) - df = df.set_index(["event_start", "source_id"]).sort_index() - if swapped_sign: - df["event_value"] = -df["event_value"] - - assert df["event_value"].to_list() == expected_df["event_value"].to_list() From 316ed0b618cb1d8a0e8d475f97e26b4c3135b72d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 17:56:59 +0200 Subject: [PATCH 19/45] add changelog warning Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 2cd811206..6f0a810ee 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -6,6 +6,9 @@ FlexMeasures Changelog v0.13.0 | April XX, 2023 ============================ +.. warning:: Sunset notice for API versions 1.0, 1.1, 1.2, 1.3 and 2.0: after upgrading to ``flexmeasures==0.13``, users of these API versions will receive ``HTTP status 404`` responses. + The relevant endpoints have been deprecated since ``flexmeasures==0.12``. + .. warning:: The API endpoint (`[POST] /sensors/(id)/schedules/trigger `_) to make new schedules sunsets the deprecated (since v0.12) storage flexibility parameters (they move to the ``flex-model`` parameter group), as well as the parameters describing other sensors (they move to ``flex-context``). .. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``). From 62b0b34d76cf24fc93109a336cf5af9305624268 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 20:11:14 +0200 Subject: [PATCH 20/45] update API documentation Signed-off-by: F.N. Claessen --- documentation/api/v1.rst | 17 +---------------- documentation/api/v1_1.rst | 17 +---------------- documentation/api/v1_2.rst | 17 +---------------- documentation/api/v1_3.rst | 17 +---------------- documentation/api/v2_0.rst | 18 +----------------- 5 files changed, 5 insertions(+), 81 deletions(-) diff --git a/documentation/api/v1.rst b/documentation/api/v1.rst index b5ba0a216..3deaba5a3 100644 --- a/documentation/api/v1.rst +++ b/documentation/api/v1.rst @@ -3,20 +3,5 @@ Version 1.0 =========== -.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. -Summary -------- - -.. qrefflask:: flexmeasures.app:create(env="documentation") - :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1 - :order: path - :include-empty-docstring: - -API Details ------------ - -.. autoflask:: flexmeasures.app:create(env="documentation") - :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1 - :order: path - :include-empty-docstring: +.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 1.0 will stay available `here on readthedocs.org `_. diff --git a/documentation/api/v1_1.rst b/documentation/api/v1_1.rst index 5ef6c0272..3c67b5069 100644 --- a/documentation/api/v1_1.rst +++ b/documentation/api/v1_1.rst @@ -3,20 +3,5 @@ Version 1.1 =========== -.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. -Summary -------- - -.. qrefflask:: flexmeasures.app:create(env="documentation") - :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_1 - :order: path - :include-empty-docstring: - -API Details ------------ - -.. autoflask:: flexmeasures.app:create(env="documentation") - :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_1 - :order: path - :include-empty-docstring: +.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 1.1 will stay available `here on readthedocs.org `_. diff --git a/documentation/api/v1_2.rst b/documentation/api/v1_2.rst index bab646eda..a333c8cbd 100644 --- a/documentation/api/v1_2.rst +++ b/documentation/api/v1_2.rst @@ -3,20 +3,5 @@ Version 1.2 =========== -.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. -Summary -------- - -.. qrefflask:: flexmeasures.app:create(env="documentation") - :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_2 - :order: path - :include-empty-docstring: - -API Details ------------ - -.. autoflask:: flexmeasures.app:create(env="documentation") - :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_2 - :order: path - :include-empty-docstring: +.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 1.2 will stay available `here on readthedocs.org `_. diff --git a/documentation/api/v1_3.rst b/documentation/api/v1_3.rst index c186461f6..01b8a4127 100644 --- a/documentation/api/v1_3.rst +++ b/documentation/api/v1_3.rst @@ -3,20 +3,5 @@ Version 1.3 =========== -.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. -Summary -------- - -.. qrefflask:: flexmeasures.app:create(env="documentation") - :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_3 - :order: path - :include-empty-docstring: - -API Details ------------ - -.. autoflask:: flexmeasures.app:create(env="documentation") - :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_3 - :order: path - :include-empty-docstring: +.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 1.3 will stay available `here on readthedocs.org `_. diff --git a/documentation/api/v2_0.rst b/documentation/api/v2_0.rst index 8b1279f0d..103a171fe 100644 --- a/documentation/api/v2_0.rst +++ b/documentation/api/v2_0.rst @@ -3,20 +3,4 @@ Version 2.0 =========== -.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. - -Summary -------- - -.. qrefflask:: flexmeasures.app:create(env="documentation") - :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v2_0 - :order: path - :include-empty-docstring: - -API Details ------------ - -.. autoflask:: flexmeasures.app:create(env="documentation") - :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v2_0 - :order: path - :include-empty-docstring: +.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 2.0 will stay available `here on readthedocs.org `_. From 51e665db55f3258861eb8ecb7bef455b5d8b8f3f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 20:36:38 +0200 Subject: [PATCH 21/45] Document public endpoints in v3 Signed-off-by: F.N. Claessen --- documentation/api/v3_0.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/api/v3_0.rst b/documentation/api/v3_0.rst index 654725e03..86fd0fb6c 100644 --- a/documentation/api/v3_0.rst +++ b/documentation/api/v3_0.rst @@ -7,7 +7,7 @@ Summary ------- .. qrefflask:: flexmeasures.app:create(env="documentation") - :modules: flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health + :modules: flexmeasures.api, flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health :order: path :include-empty-docstring: @@ -15,6 +15,6 @@ API Details ----------- .. autoflask:: flexmeasures.app:create(env="documentation") - :modules: flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health + :modules: flexmeasures.api, flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health :order: path :include-empty-docstring: From 62c5ef50a93eaf3e39708889704123d06f08474e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 25 Apr 2023 20:37:50 +0200 Subject: [PATCH 22/45] black Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_assets_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_assets_api.py b/flexmeasures/api/v3_0/tests/test_assets_api.py index 6c243b400..c6cb8fad4 100644 --- a/flexmeasures/api/v3_0/tests/test_assets_api.py +++ b/flexmeasures/api/v3_0/tests/test_assets_api.py @@ -335,7 +335,9 @@ def test_post_an_asset(client, setup_api_test_data): assert post_assets_response.status_code == 201 assert post_assets_response.json["latitude"] == 30.1 - asset: GenericAsset = GenericAsset.query.filter_by(name="Test battery 2").one_or_none() + asset: GenericAsset = GenericAsset.query.filter_by( + name="Test battery 2" + ).one_or_none() assert asset is not None assert asset.latitude == 30.1 From 928510b5b5751a322d2a19dccdedd72f3b35991e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 26 Apr 2023 15:56:55 +0200 Subject: [PATCH 23/45] Implement getService for API v3 Signed-off-by: F.N. Claessen --- flexmeasures/api/__init__.py | 4 +-- flexmeasures/api/v3_0/__init__.py | 2 ++ flexmeasures/api/v3_0/public.py | 55 +++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 flexmeasures/api/v3_0/public.py diff --git a/flexmeasures/api/__init__.py b/flexmeasures/api/__init__.py index 1ae0c26f2..0d96b48ea 100644 --- a/flexmeasures/api/__init__.py +++ b/flexmeasures/api/__init__.py @@ -73,9 +73,9 @@ def get_versions() -> dict: """ response = { "message": "For these API versions a public endpoint is available, listing its service. For example: " - "/api/v2_0/getService and /api/v3_0/getService. An authentication token can be requested at: " + "/api/v3_0/getService. An authentication token can be requested at: " "/api/requestAuthToken", - "versions": ["v1", "v1_1", "v1_2", "v1_3", "v2_0", "v3_0"], + "versions": ["v3_0"], "flexmeasures_version": flexmeasures_version, } return response diff --git a/flexmeasures/api/v3_0/__init__.py b/flexmeasures/api/v3_0/__init__.py index fb80f2bb7..5b2854fa6 100644 --- a/flexmeasures/api/v3_0/__init__.py +++ b/flexmeasures/api/v3_0/__init__.py @@ -5,6 +5,7 @@ from flexmeasures.api.v3_0.users import UserAPI from flexmeasures.api.v3_0.assets import AssetAPI from flexmeasures.api.v3_0.health import HealthAPI +from flexmeasures.api.v3_0.public import ServicesAPI def register_at(app: Flask): @@ -17,3 +18,4 @@ def register_at(app: Flask): UserAPI.register(app, route_prefix=v3_0_api_prefix) AssetAPI.register(app, route_prefix=v3_0_api_prefix) HealthAPI.register(app, route_prefix=v3_0_api_prefix) + ServicesAPI.register(app) diff --git a/flexmeasures/api/v3_0/public.py b/flexmeasures/api/v3_0/public.py new file mode 100644 index 000000000..374e8c9e5 --- /dev/null +++ b/flexmeasures/api/v3_0/public.py @@ -0,0 +1,55 @@ +from operator import itemgetter + +from flask import current_app, request +from flask_classful import FlaskView, route +from flask_json import as_json + +from flexmeasures.api.common.responses import request_processed +from flexmeasures.api.common.utils.decorators import as_response_type + + +class ServicesAPI(FlaskView): + + route_base = "/api/v3_0" + trailing_slash = False + + @route("/getService", methods=["GET"]) + @as_response_type("GetServiceResponse") + @as_json + def get_service(self): + """API endpoint to get a service listing for this version. + + .. :quickref: Public; Obtain a service listing for this version + + :resheader Content-Type: application/json + :status 200: PROCESSED + """ + services = [] + for rule in current_app.url_map.iter_rules(): + url = rule.rule + if url.startswith(self.route_base): + methods: str = "/".join( + [m for m in rule.methods if m not in ("OPTIONS", "HEAD")] + ) + stripped_url = url.lstrip(self.route_base) + full_url = ( + request.url_root.rstrip("/") + url + if url.startswith("/") + else request.url_root + url + ) + services.append( + dict( + url=full_url, + name=f"{methods} {stripped_url}", + description=current_app.view_functions[ + rule.endpoint + ].__doc__.split("\n")[0], + ) + ) + response = dict( + services=sorted(services, key=itemgetter("url")), + version="3.0", + ) + + d, s = request_processed() + return dict(**response, **d), s From 4c7780cdd0f7a18e87e483ce9868b3b626ea1e03 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 26 Apr 2023 15:58:23 +0200 Subject: [PATCH 24/45] Document service listing for v3 Signed-off-by: F.N. Claessen --- documentation/api/v3_0.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/api/v3_0.rst b/documentation/api/v3_0.rst index 86fd0fb6c..486ee9c42 100644 --- a/documentation/api/v3_0.rst +++ b/documentation/api/v3_0.rst @@ -7,7 +7,7 @@ Summary ------- .. qrefflask:: flexmeasures.app:create(env="documentation") - :modules: flexmeasures.api, flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health + :modules: flexmeasures.api, flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health, flexmeasures.api.v3_0.public :order: path :include-empty-docstring: @@ -15,6 +15,6 @@ API Details ----------- .. autoflask:: flexmeasures.app:create(env="documentation") - :modules: flexmeasures.api, flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health + :modules: flexmeasures.api, flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health, flexmeasures.api.v3_0.public :order: path :include-empty-docstring: From d94d780d17b52c58b28f019b4c56823a997ffb89 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 26 Apr 2023 16:17:20 +0200 Subject: [PATCH 25/45] Show quickref contents rather than summary line of docstring Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/public.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/public.py b/flexmeasures/api/v3_0/public.py index 374e8c9e5..7f1cc7177 100644 --- a/flexmeasures/api/v3_0/public.py +++ b/flexmeasures/api/v3_0/public.py @@ -1,4 +1,6 @@ from operator import itemgetter +import re +import six from flask import current_app, request from flask_classful import FlaskView, route @@ -37,13 +39,14 @@ def get_service(self): if url.startswith("/") else request.url_root + url ) + quickref = quickref_directive( + current_app.view_functions[rule.endpoint].__doc__ + ) services.append( dict( url=full_url, name=f"{methods} {stripped_url}", - description=current_app.view_functions[ - rule.endpoint - ].__doc__.split("\n")[0], + description=quickref, ) ) response = dict( @@ -53,3 +56,24 @@ def get_service(self): d, s = request_processed() return dict(**response, **d), s + + +def quickref_directive(content): + """Adapted from sphinxcontrib/autohttp/flask_base.py:quickref_directive.""" + rcomp = re.compile(r"^\s*.. :quickref:\s*(?P.*)$") + + if isinstance(content, six.string_types): + content = content.splitlines() + description = "" + for line in content: + qref = rcomp.match(line) + if qref: + quickref = qref.group("quick") + parts = quickref.split("; ", 1) + if len(parts) > 1: + description = parts[1] + else: + description = quickref + break + + return description From c96674628950ce5a0ec26ea63e40b09d6aa5f6a0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 26 Apr 2023 17:06:51 +0200 Subject: [PATCH 26/45] Fix removal of suffix and prefix, by not using rstrip and lstrip, respectively Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/public.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/public.py b/flexmeasures/api/v3_0/public.py index 7f1cc7177..d4f439d36 100644 --- a/flexmeasures/api/v3_0/public.py +++ b/flexmeasures/api/v3_0/public.py @@ -33,9 +33,9 @@ def get_service(self): methods: str = "/".join( [m for m in rule.methods if m not in ("OPTIONS", "HEAD")] ) - stripped_url = url.lstrip(self.route_base) + stripped_url = url.removeprefix(self.route_base) full_url = ( - request.url_root.rstrip("/") + url + request.url_root.removesuffix("/") + url if url.startswith("/") else request.url_root + url ) From 250dd547135f4d5e6634ece6658aa0ec2ea0da2a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 26 Apr 2023 19:59:18 +0200 Subject: [PATCH 27/45] API changelog entry Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 39 ++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index ff375778a..acedfa195 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -5,6 +5,12 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace. +v3.0-9 | 2023-04-26 +""""""""""""""""""" + +- Added missing documentation for the public endpoints for authentication and listing active API versions. +- Added REST endpoint for listing available services for a specific API version: `/api/v3_0` (GET). This functionality is similar to the *getService* endpoint for older API versions, but now also returns the full URL for each available service. + v3.0-8 | 2023-03-23 """"""""""""""""""" @@ -106,8 +112,13 @@ v3.0-0 | 2022-03-25 - Rewrote the sections on roles and sources into a combined section that refers to account roles rather than USEF roles. - Deprecated the section on group notation. +v2.0-6 | 2022-04-26 +""""""""""""""""""" + +*API v2.0 is sunset.* + v2.0-5 | 2022-02-13 -"""""""""""""""""""" +""""""""""""""""""" *API v2.0 is deprecated.* @@ -152,6 +163,11 @@ v2.0-0 | 2020-11-14 - Added REST endpoints for managing assets: `/assets/` (GET, POST) and `/asset/` (GET, PATCH, DELETE). +v1.3-13 | 2022-04-26 +"""""""""""""""""""" + +*API v1.3 is sunset.* + v1.3-12 | 2022-02-13 """""""""""""""""""" @@ -243,8 +259,13 @@ v1.3-0 | 2020-01-28 - The *postUdiEvent* endpoint now triggers scheduling jobs to be set up (rather than scheduling directly triggered by the *getDeviceMessage* endpoint) - The *getDeviceMessage* now queries the job queue and database for an available schedule +v1.2-5 | 2022-04-26 +""""""""""""""""""" + +*API v1.2 is sunset.* + v1.2-4 | 2022-02-13 -"""""""""""""""""""" +""""""""""""""""""" *API v1.2 is deprecated.* @@ -294,8 +315,13 @@ v1.2-0 | 2018-09-08 - Added a description of the *postUdiEvent* endpoint in the Prosumer and Simulation sections - Added a description of the *getDeviceMessage* endpoint in the Prosumer and Simulation sections +v1.1-7 | 2022-04-26 +""""""""""""""""""" + +*API v1.1 is sunset.* + v1.1-6 | 2022-02-13 -"""""""""""""""""""" +""""""""""""""""""" *API v1.1 is deprecated.* @@ -352,8 +378,13 @@ v1.1-0 | 2018-07-15 - Added a description of the *getPrognosis* endpoint in the Supplier section +v1.0-3 | 2022-04-26 +""""""""""""""""""" + +*API v1.0 is sunset.* + v1.0-2 | 2022-02-13 -"""""""""""""""""""" +""""""""""""""""""" *API v1.0 is deprecated.* From fe2242826afbb15260d776fc5b32961b4b4191ab Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 26 Apr 2023 20:06:34 +0200 Subject: [PATCH 28/45] Make getService more RESTful Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 2 +- flexmeasures/api/__init__.py | 2 +- flexmeasures/api/v3_0/public.py | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index acedfa195..330f849da 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -88,7 +88,7 @@ v3.0-0 | 2022-03-25 - *getDeviceMessage* -> use `/sensors//schedules/` (GET) instead, where is the sensor id from the "event" field and is the value of the "schedule" field returned by `/sensors//schedules/trigger` (POST) - *getMeterData* -> use `/sensors/data` (GET) instead, replacing the "connection" field with "sensor" - *getPrognosis* -> use `/sensors/data` (GET) instead, replacing the "connection" field with "sensor" - - *getService* -> consult the public API documentation instead, at https://flexmeasures.readthedocs.io + - *getService* -> use `/api/v3_0` (GET) instead (since v3.0-9), or consult the public API documentation instead, at https://flexmeasures.readthedocs.io - *postMeterData* -> use `/sensors/data` (POST) instead, replacing the "connection" field with "sensor" - *postPriceData* -> use `/sensors/data` (POST) instead, replacing the "market" field with "sensor" - *postPrognosis* -> use `/sensors/data` (POST) instead, replacing the "connection" field with "sensor" diff --git a/flexmeasures/api/__init__.py b/flexmeasures/api/__init__.py index 0d96b48ea..bf8075017 100644 --- a/flexmeasures/api/__init__.py +++ b/flexmeasures/api/__init__.py @@ -73,7 +73,7 @@ def get_versions() -> dict: """ response = { "message": "For these API versions a public endpoint is available, listing its service. For example: " - "/api/v3_0/getService. An authentication token can be requested at: " + "/api/v3_0. An authentication token can be requested at: " "/api/requestAuthToken", "versions": ["v3_0"], "flexmeasures_version": flexmeasures_version, diff --git a/flexmeasures/api/v3_0/public.py b/flexmeasures/api/v3_0/public.py index d4f439d36..8bb7ca1f1 100644 --- a/flexmeasures/api/v3_0/public.py +++ b/flexmeasures/api/v3_0/public.py @@ -7,7 +7,6 @@ from flask_json import as_json from flexmeasures.api.common.responses import request_processed -from flexmeasures.api.common.utils.decorators import as_response_type class ServicesAPI(FlaskView): @@ -15,10 +14,9 @@ class ServicesAPI(FlaskView): route_base = "/api/v3_0" trailing_slash = False - @route("/getService", methods=["GET"]) - @as_response_type("GetServiceResponse") + @route("", methods=["GET"]) @as_json - def get_service(self): + def index(self): """API endpoint to get a service listing for this version. .. :quickref: Public; Obtain a service listing for this version From ee3d176507b7fafbb48aa31de34c8068eeba66c4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 26 Apr 2023 20:22:03 +0200 Subject: [PATCH 29/45] Add warning to API developer docs Signed-off-by: F.N. Claessen --- documentation/dev/api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/dev/api.rst b/documentation/dev/api.rst index 44cb6fe58..e8c72ba27 100644 --- a/documentation/dev/api.rst +++ b/documentation/dev/api.rst @@ -7,6 +7,8 @@ The FlexMeasures API is the main way that third-parties can automate their inter This is a small guide for creating new versions of the API and its docs. +.. warning:: This guide was written for API versions below v3.0 and is currently out of date. + .. todo:: A guide for endpoint design, e.g. using Marshmallow schemas and common validators. .. contents:: Table of contents From 81b06002b6360f7b6116a4d8edf85fbdb00c1046 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 26 Apr 2023 20:22:36 +0200 Subject: [PATCH 30/45] More robust against spaces Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/public.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/public.py b/flexmeasures/api/v3_0/public.py index 8bb7ca1f1..87ba4d669 100644 --- a/flexmeasures/api/v3_0/public.py +++ b/flexmeasures/api/v3_0/public.py @@ -67,9 +67,9 @@ def quickref_directive(content): qref = rcomp.match(line) if qref: quickref = qref.group("quick") - parts = quickref.split("; ", 1) + parts = quickref.split(";", 1) if len(parts) > 1: - description = parts[1] + description = parts[1].lstrip(" ") else: description = quickref break From eb83957ec011d7656ad874114285871e32aba964 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 26 Apr 2023 21:52:54 +0200 Subject: [PATCH 31/45] Update main changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 6f0a810ee..de5d8fbb8 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -6,7 +6,7 @@ FlexMeasures Changelog v0.13.0 | April XX, 2023 ============================ -.. warning:: Sunset notice for API versions 1.0, 1.1, 1.2, 1.3 and 2.0: after upgrading to ``flexmeasures==0.13``, users of these API versions will receive ``HTTP status 404`` responses. +.. warning:: Sunset notice for API versions 1.0, 1.1, 1.2, 1.3 and 2.0: after upgrading to ``flexmeasures==0.13``, users of these API versions may receive ``HTTP status 410 (Gone)`` responses. The relevant endpoints have been deprecated since ``flexmeasures==0.12``. .. warning:: The API endpoint (`[POST] /sensors/(id)/schedules/trigger `_) to make new schedules sunsets the deprecated (since v0.12) storage flexibility parameters (they move to the ``flex-model`` parameter group), as well as the parameters describing other sensors (they move to ``flex-context``). @@ -29,6 +29,7 @@ Bugfixes Infrastructure / Support ---------------------- +* Sunset API versions 1.0, 1.1, 1.2, 1.3 and 2.0 [see `PR #650 `_] * Sunset several API fields for `/sensors//schedules/trigger` (POST) that have moved into the ``flex-model`` or ``flex-context`` fields [see `PR #580 `_] * Fix broken `make show-data-model` command [see `PR #638 `_] @@ -535,7 +536,7 @@ Infrastructure / Support * Integration with `timely beliefs `__ lib: Sensors [see `PR #13 `_] * Apache 2.0 license [see `PR #16 `_] * Load js & css from CDN [see `PR #21 `_] -* Start using marshmallow for input validation, also introducing ``HTTP status 422`` in the API [see `PR #25 `_] +* Start using marshmallow for input validation, also introducing ``HTTP status 422 (Unprocessable Entity)`` in the API [see `PR #25 `_] * Replace ``solarpy`` with ``pvlib`` (due to license conflict) [see `PR #16 `_] * Stop supporting the creation of new users on asset creation (to reduce complexity) [see `PR #36 `_] From 1b8266d99f08ef57ffdccc79d1f55738eec05345 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 28 Apr 2023 13:28:12 +0200 Subject: [PATCH 32/45] Support blackout tests Signed-off-by: F.N. Claessen --- documentation/configuration.rst | 26 +++++++ .../api/common/utils/deprecation_utils.py | 73 ++++++++++++++++--- flexmeasures/api/v1/__init__.py | 10 ++- flexmeasures/api/v1_1/__init__.py | 12 ++- flexmeasures/api/v1_2/__init__.py | 12 ++- flexmeasures/api/v1_3/__init__.py | 12 ++- flexmeasures/api/v2_0/__init__.py | 12 ++- flexmeasures/utils/config_defaults.py | 5 ++ flexmeasures/utils/error_utils.py | 4 +- 9 files changed, 146 insertions(+), 20 deletions(-) diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 27c4766fc..cac8b3f89 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -592,3 +592,29 @@ When ``FLEXMEASURES_MODE=demo``\ , this setting can be used to make the FlexMeas so that old imported data can be demoed as if it were current. Default: ``None`` + +Sunset +------ + +FLEXMEASURES_API_1_AND_2_SUNSET_ACTIVE +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Allow control over the effect of sunsetting API versions 1.0, 1.1, 1.2, 1.3 and 2.0. +Specifically, if True, the endpoints in these versions will return 410 (Gone) status codes. +If False, the endpoints will work like before, including Deprecation and Sunset headers in their response. + +Default: ``False`` + +FLEXMEASURES_API_1_AND_2_SUNSET_DATE +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Allow to override the default sunset date for your clients. + +Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"2023-05-01"`` for version 2.0) + +FLEXMEASURES_API_1_AND_2_SUNSET_LINK +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Allow to override the default sunset link for your clients. + +Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html"`` for version 2.0) diff --git a/flexmeasures/api/common/utils/deprecation_utils.py b/flexmeasures/api/common/utils/deprecation_utils.py index 84263de38..0ac098cc9 100644 --- a/flexmeasures/api/common/utils/deprecation_utils.py +++ b/flexmeasures/api/common/utils/deprecation_utils.py @@ -1,12 +1,39 @@ from __future__ import annotations -from flask import current_app, request, Blueprint, Response, after_this_request +from flask import abort, current_app, request, Blueprint, Response, after_this_request from flask_security.core import current_user import pandas as pd from flexmeasures.utils.time_utils import to_http_time +def sunset_blueprint( + blueprint, + api_version_sunset: str, + sunset_info_link: str, + api_version_upgrade_to: str = "3.0", +): + """Sunsets every route on a blueprint by returning 410 (Gone) responses. + + Such errors will be logged by utils.error_utils.error_handling_router. + """ + + def let_host_switch_to_returning_410(): + + # Override with custom info link, if set by host + _sunset_info_link = current_app.config.get( + "FLEXMEASURES_API_1_AND_2_SUNSET_LINK", sunset_info_link + ) + + if current_app.config["FLEXMEASURES_API_1_AND_2_SUNSET_ACTIVE"]: + abort( + 410, + f"API version {api_version_sunset} has been sunset. Please upgrade to API version {api_version_upgrade_to}. See {_sunset_info_link} for more information.", + ) + + blueprint.before_request(let_host_switch_to_returning_410) + + def deprecate_fields( fields: str | list[str], deprecation_date: pd.Timestamp | str | None = None, @@ -50,7 +77,8 @@ def post_item(color, length): """ if not isinstance(fields, list): fields = [fields] - deprecation, sunset = _format_deprecation_and_sunset(deprecation_date, sunset_date) + deprecation = _format_deprecation(deprecation_date) + sunset = _format_sunset(sunset_date) @after_this_request def _after_request_handler(response: Response) -> Response: @@ -63,12 +91,23 @@ def _after_request_handler(response: Response) -> Response: current_app.logger.warning( f"Endpoint {request.endpoint} called by {current_user} with deprecated fields: {deprecated_fields_used}" ) + + # Override sunset date if host used corresponding config setting + _sunset = _format_sunset( + current_app.config.get("FLEXMEASURES_API_1_AND_2_SUNSET_DATE", sunset) + ) + + # Override sunset link if host used corresponding config setting + _sunset_link = current_app.config.get( + "FLEXMEASURES_API_1_AND_2_SUNSET_LINK", sunset_link + ) + return _add_headers( response, deprecation, deprecation_link, - sunset, - sunset_link, + _sunset, + _sunset_link, ) return response @@ -109,18 +148,30 @@ def deprecate_blueprint( - Deprecation header: https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header - Sunset header: https://www.rfc-editor.org/rfc/rfc8594 """ - deprecation, sunset = _format_deprecation_and_sunset(deprecation_date, sunset_date) + deprecation = _format_deprecation(deprecation_date) + sunset = _format_sunset(sunset_date) def _after_request_handler(response: Response) -> Response: current_app.logger.warning( f"Deprecated endpoint {request.endpoint} called by {current_user}" ) + + # Override sunset date if host used corresponding config setting + _sunset = _format_sunset( + current_app.config.get("FLEXMEASURES_API_1_AND_2_SUNSET_DATE", sunset) + ) + + # Override sunset link if host used corresponding config setting + _sunset_link = current_app.config.get( + "FLEXMEASURES_API_1_AND_2_SUNSET_LINK", sunset_link + ) + return _add_headers( response, deprecation, deprecation_link, - sunset, - sunset_link, + _sunset, + _sunset_link, ) blueprint.after_request(_after_request_handler) @@ -149,13 +200,17 @@ def _add_link(response: Response, link: str, rel: str) -> Response: return response -def _format_deprecation_and_sunset(deprecation_date, sunset_date): +def _format_deprecation(deprecation_date): if deprecation_date: deprecation = to_http_time(pd.Timestamp(deprecation_date) - pd.Timedelta("1s")) else: deprecation = "true" + return deprecation + + +def _format_sunset(sunset_date): if sunset_date: sunset = to_http_time(pd.Timestamp(sunset_date) - pd.Timedelta("1s")) else: sunset = None - return deprecation, sunset + return sunset diff --git a/flexmeasures/api/v1/__init__.py b/flexmeasures/api/v1/__init__.py index 8f8632cff..7e61a25f9 100644 --- a/flexmeasures/api/v1/__init__.py +++ b/flexmeasures/api/v1/__init__.py @@ -1,6 +1,9 @@ from flask import Flask, Blueprint -from flexmeasures.api.common.utils.deprecation_utils import deprecate_blueprint +from flexmeasures.api.common.utils.deprecation_utils import ( + deprecate_blueprint, + sunset_blueprint, +) # The api blueprint. It is registered with the Flask app (see register_at) @@ -9,9 +12,12 @@ flexmeasures_api, deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/introduction.html#deprecation-and-sunset", - sunset_date="2023-02-01", + sunset_date="2023-05-01", sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v1.html", ) +sunset_blueprint( + flexmeasures_api, "1.0", "https://flexmeasures.readthedocs.io/en/latest/api/v1.html" +) def register_at(app: Flask): diff --git a/flexmeasures/api/v1_1/__init__.py b/flexmeasures/api/v1_1/__init__.py index 45fa12d61..1f153a35b 100644 --- a/flexmeasures/api/v1_1/__init__.py +++ b/flexmeasures/api/v1_1/__init__.py @@ -1,6 +1,9 @@ from flask import Flask, Blueprint -from flexmeasures.api.common.utils.deprecation_utils import deprecate_blueprint +from flexmeasures.api.common.utils.deprecation_utils import ( + deprecate_blueprint, + sunset_blueprint, +) # The api blueprint. It is registered with the Flask app (see app.py) flexmeasures_api = Blueprint("flexmeasures_api_v1_1", __name__) @@ -8,9 +11,14 @@ flexmeasures_api, deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_1.html", - sunset_date="2023-02-01", + sunset_date="2023-05-01", sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_1.html", ) +sunset_blueprint( + flexmeasures_api, + "1.1", + "https://flexmeasures.readthedocs.io/en/latest/api/v1_1.html", +) def register_at(app: Flask): diff --git a/flexmeasures/api/v1_2/__init__.py b/flexmeasures/api/v1_2/__init__.py index b34ee17a5..42e12e1d0 100644 --- a/flexmeasures/api/v1_2/__init__.py +++ b/flexmeasures/api/v1_2/__init__.py @@ -1,6 +1,9 @@ from flask import Flask, Blueprint -from flexmeasures.api.common.utils.deprecation_utils import deprecate_blueprint +from flexmeasures.api.common.utils.deprecation_utils import ( + deprecate_blueprint, + sunset_blueprint, +) # The api blueprint. It is registered with the Flask app (see app.py) flexmeasures_api = Blueprint("flexmeasures_api_v1_2", __name__) @@ -8,9 +11,14 @@ flexmeasures_api, deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_2.html", - sunset_date="2023-02-01", + sunset_date="2023-05-01", sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_2.html", ) +sunset_blueprint( + flexmeasures_api, + "1.2", + "https://flexmeasures.readthedocs.io/en/latest/api/v1_2.html", +) def register_at(app: Flask): diff --git a/flexmeasures/api/v1_3/__init__.py b/flexmeasures/api/v1_3/__init__.py index d6b14b119..e15f98bd0 100644 --- a/flexmeasures/api/v1_3/__init__.py +++ b/flexmeasures/api/v1_3/__init__.py @@ -1,6 +1,9 @@ from flask import Flask, Blueprint -from flexmeasures.api.common.utils.deprecation_utils import deprecate_blueprint +from flexmeasures.api.common.utils.deprecation_utils import ( + deprecate_blueprint, + sunset_blueprint, +) # The api blueprint. It is registered with the Flask app (see app.py) flexmeasures_api = Blueprint("flexmeasures_api_v1_3", __name__) @@ -8,9 +11,14 @@ flexmeasures_api, deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_3.html", - sunset_date="2023-02-01", + sunset_date="2023-05-01", sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_3.html", ) +sunset_blueprint( + flexmeasures_api, + "1.3", + "https://flexmeasures.readthedocs.io/en/latest/api/v1_3.html", +) def register_at(app: Flask): diff --git a/flexmeasures/api/v2_0/__init__.py b/flexmeasures/api/v2_0/__init__.py index c32eb09c6..c6bb37433 100644 --- a/flexmeasures/api/v2_0/__init__.py +++ b/flexmeasures/api/v2_0/__init__.py @@ -1,15 +1,23 @@ from flask import Flask, Blueprint -from flexmeasures.api.common.utils.deprecation_utils import deprecate_blueprint +from flexmeasures.api.common.utils.deprecation_utils import ( + deprecate_blueprint, + sunset_blueprint, +) flexmeasures_api = Blueprint("flexmeasures_api_v2_0", __name__) deprecate_blueprint( flexmeasures_api, deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html", - sunset_date="2023-02-01", + sunset_date="2023-05-01", sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html", ) +sunset_blueprint( + flexmeasures_api, + "2.0", + "https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html", +) def register_at(app: Flask): diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 967bac318..cd4e66190 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -130,6 +130,11 @@ class Config(object): # todo: expand with other js versions used in FlexMeasures ) + # Custom sunset switches + FLEXMEASURES_API_1_AND_2_SUNSET_ACTIVE: bool = False # if active, sunset endpoints will return 410 (Gone) + FLEXMEASURES_API_1_AND_2_SUNSET_DATE: str | None = None # e.g. 2023-05-01 + FLEXMEASURES_API_1_AND_2_SUNSET_LINK: str | None = None # e.g. https://flexmeasures.readthedocs.io/en/latest/api/introduction.html#deprecation-and-sunset + # names of settings which cannot be None # SECRET_KEY is also required but utils.app_utils.set_secret_key takes care of this better. diff --git a/flexmeasures/utils/error_utils.py b/flexmeasures/utils/error_utils.py index 7882581f1..852a25d54 100644 --- a/flexmeasures/utils/error_utils.py +++ b/flexmeasures/utils/error_utils.py @@ -7,6 +7,7 @@ InternalServerError, BadRequest, NotFound, + Gone, ) from sqlalchemy.orm import Query @@ -70,7 +71,7 @@ def error_handling_router(error: HTTPException): error, "description", f"Something went wrong: {error.__class__.__name__}" ) - if request.is_json: + if request.is_json or request.url_rule.rule.startswith("/api"): response = jsonify( dict( message=getattr(error, "description", str(error)), @@ -99,6 +100,7 @@ def add_basic_error_handlers(app: Flask): app.register_error_handler(BadRequest, error_handling_router) app.register_error_handler(HTTPException, error_handling_router) app.register_error_handler(NotFound, error_handling_router) + app.register_error_handler(Gone, error_handling_router) app.register_error_handler(Exception, error_handling_router) From d6b6d05694223ba21fd6cac9566ce3e9b3cb008e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 28 Apr 2023 13:32:06 +0200 Subject: [PATCH 33/45] black Signed-off-by: F.N. Claessen --- flexmeasures/utils/config_defaults.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index cd4e66190..988d92a1f 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -131,7 +131,9 @@ class Config(object): ) # Custom sunset switches - FLEXMEASURES_API_1_AND_2_SUNSET_ACTIVE: bool = False # if active, sunset endpoints will return 410 (Gone) + FLEXMEASURES_API_1_AND_2_SUNSET_ACTIVE: bool = ( + False # if active, sunset endpoints will return 410 (Gone) + ) FLEXMEASURES_API_1_AND_2_SUNSET_DATE: str | None = None # e.g. 2023-05-01 FLEXMEASURES_API_1_AND_2_SUNSET_LINK: str | None = None # e.g. https://flexmeasures.readthedocs.io/en/latest/api/introduction.html#deprecation-and-sunset From ade6713e6e08499bfa3a36fd75e4da6f0938c2aa Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 28 Apr 2023 13:43:29 +0200 Subject: [PATCH 34/45] Fix test Signed-off-by: F.N. Claessen --- flexmeasures/utils/error_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/utils/error_utils.py b/flexmeasures/utils/error_utils.py index 852a25d54..a156825ea 100644 --- a/flexmeasures/utils/error_utils.py +++ b/flexmeasures/utils/error_utils.py @@ -71,7 +71,9 @@ def error_handling_router(error: HTTPException): error, "description", f"Something went wrong: {error.__class__.__name__}" ) - if request.is_json or request.url_rule.rule.startswith("/api"): + if request.is_json or ( + hasattr(request.url_rule, "rule") and request.url_rule.rule.startswith("/api") + ): response = jsonify( dict( message=getattr(error, "description", str(error)), From 25db921f415f140732267a73be4b82c666208339 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 28 Apr 2023 13:45:12 +0200 Subject: [PATCH 35/45] More specific if statement Signed-off-by: F.N. Claessen --- flexmeasures/utils/error_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/utils/error_utils.py b/flexmeasures/utils/error_utils.py index a156825ea..faa8544de 100644 --- a/flexmeasures/utils/error_utils.py +++ b/flexmeasures/utils/error_utils.py @@ -72,7 +72,7 @@ def error_handling_router(error: HTTPException): ) if request.is_json or ( - hasattr(request.url_rule, "rule") and request.url_rule.rule.startswith("/api") + request.url_rule is not None and request.url_rule.rule.startswith("/api") ): response = jsonify( dict( From 9c56501c3fdf571a3dcfc4e2af1065b32f7dced8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 28 Apr 2023 14:04:14 +0200 Subject: [PATCH 36/45] change default to activating the sunset Signed-off-by: F.N. Claessen --- documentation/configuration.rst | 2 +- flexmeasures/utils/config_defaults.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/documentation/configuration.rst b/documentation/configuration.rst index cac8b3f89..e76557f58 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -603,7 +603,7 @@ Allow control over the effect of sunsetting API versions 1.0, 1.1, 1.2, 1.3 and Specifically, if True, the endpoints in these versions will return 410 (Gone) status codes. If False, the endpoints will work like before, including Deprecation and Sunset headers in their response. -Default: ``False`` +Default: ``True`` FLEXMEASURES_API_1_AND_2_SUNSET_DATE ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 988d92a1f..1e272af43 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -131,9 +131,7 @@ class Config(object): ) # Custom sunset switches - FLEXMEASURES_API_1_AND_2_SUNSET_ACTIVE: bool = ( - False # if active, sunset endpoints will return 410 (Gone) - ) + FLEXMEASURES_API_1_AND_2_SUNSET_ACTIVE: bool = True # if True, sunset endpoints return 410 (Gone) responses; if False, they will work as before FLEXMEASURES_API_1_AND_2_SUNSET_DATE: str | None = None # e.g. 2023-05-01 FLEXMEASURES_API_1_AND_2_SUNSET_LINK: str | None = None # e.g. https://flexmeasures.readthedocs.io/en/latest/api/introduction.html#deprecation-and-sunset From 733a24428cd8072b5670eadc98c309eb51561366 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 28 Apr 2023 15:38:55 +0200 Subject: [PATCH 37/45] Add sunset documentation for FlexMeasures hosts and make the config settings reusable for future sunsets Signed-off-by: F.N. Claessen --- documentation/api/introduction.rst | 21 +++++++++++++++++++ documentation/changelog.rst | 2 ++ documentation/configuration.rst | 16 +++++++------- .../api/common/utils/deprecation_utils.py | 12 +++++------ flexmeasures/utils/config_defaults.py | 6 +++--- 5 files changed, 40 insertions(+), 17 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 6a7b2b974..7378f781b 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -107,6 +107,11 @@ which gives a response like this if the credentials are correct: Deprecation and sunset ---------------------- +Some sunsetting options are available for FlexMeasures hosts. See :ref:`api_deprecation_hosts`. + +FlexMeasures clients +^^^^^^^^^^^^^^^^^^^^ + Professional API users should monitor API responses for the ``"Deprecation"`` and ``"Sunset"`` response headers [see `draft-ietf-httpapi-deprecation-header-02 `_ and `RFC 8594 `_, respectively], so system administrators can be warned when using API endpoints that are flagged for deprecation and/or are likely to become unresponsive in the future. The deprecation header field shows an `IMF-fixdate `_ indicating when the API endpoint was deprecated. @@ -139,3 +144,19 @@ Here is a client-side code example in Python (this merely prints out the depreca print(f"Your request to {url} returned a sunset warning. Sunset: {content}") elif header == "Link" and ('rel="deprecation";' in content or 'rel="sunset";' in content): print(f"Further info is available: {content}") + +.. _api_deprecation_hosts: + +FlexMeasures hosts +^^^^^^^^^^^^^^^^^^ + +When upgrading to a FlexMeasures version that sunsets an API version, clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints. +After upgrading to one of the next FlexMeasures versions, they will receive ``HTTP status 404 (Not Found)`` responses. + +Hosts should not expect every client to monitor response headers and proactively upgrade to newer API versions. +Please make sure that your users have upgraded before you upgrade to a FlexMeasures version that sunsets an API version. +You can do this by checking your server logs for warnings about users who are still calling deprecated endpoints. + +In case you have users that haven't upgraded yet, and would still like to upgrade FlexMeasures, you can. +Just set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = False`` and consider announcing some blackout tests to your users, during which you can set this setting to ``True`` to activate the sunset. +During such a blackout test, clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints. diff --git a/documentation/changelog.rst b/documentation/changelog.rst index de5d8fbb8..8b2eabac9 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -7,6 +7,7 @@ v0.13.0 | April XX, 2023 ============================ .. warning:: Sunset notice for API versions 1.0, 1.1, 1.2, 1.3 and 2.0: after upgrading to ``flexmeasures==0.13``, users of these API versions may receive ``HTTP status 410 (Gone)`` responses. + See the `documentation for deprecation and sunset `_. The relevant endpoints have been deprecated since ``flexmeasures==0.12``. .. warning:: The API endpoint (`[POST] /sensors/(id)/schedules/trigger `_) to make new schedules sunsets the deprecated (since v0.12) storage flexibility parameters (they move to the ``flex-model`` parameter group), as well as the parameters describing other sensors (they move to ``flex-context``). @@ -29,6 +30,7 @@ Bugfixes Infrastructure / Support ---------------------- +* Support blackout tests for sunset API versions [see `PR #651 `_] * Sunset API versions 1.0, 1.1, 1.2, 1.3 and 2.0 [see `PR #650 `_] * Sunset several API fields for `/sensors//schedules/trigger` (POST) that have moved into the ``flex-model`` or ``flex-context`` fields [see `PR #580 `_] * Fix broken `make show-data-model` command [see `PR #638 `_] diff --git a/documentation/configuration.rst b/documentation/configuration.rst index e76557f58..8b9533715 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -596,24 +596,24 @@ Default: ``None`` Sunset ------ -FLEXMEASURES_API_1_AND_2_SUNSET_ACTIVE -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FLEXMEASURES_API_SUNSET_ACTIVE +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Allow control over the effect of sunsetting API versions 1.0, 1.1, 1.2, 1.3 and 2.0. -Specifically, if True, the endpoints in these versions will return 410 (Gone) status codes. +Allow control over the effect of sunsetting API versions. +Specifically, if True, the endpoints in sunset versions will return ``HTTP status 410 (Gone)`` status codes. If False, the endpoints will work like before, including Deprecation and Sunset headers in their response. Default: ``True`` -FLEXMEASURES_API_1_AND_2_SUNSET_DATE -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FLEXMEASURES_API_SUNSET_DATE +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Allow to override the default sunset date for your clients. Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"2023-05-01"`` for version 2.0) -FLEXMEASURES_API_1_AND_2_SUNSET_LINK -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FLEXMEASURES_API_SUNSET_LINK +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Allow to override the default sunset link for your clients. diff --git a/flexmeasures/api/common/utils/deprecation_utils.py b/flexmeasures/api/common/utils/deprecation_utils.py index 0ac098cc9..f7511acb6 100644 --- a/flexmeasures/api/common/utils/deprecation_utils.py +++ b/flexmeasures/api/common/utils/deprecation_utils.py @@ -22,10 +22,10 @@ def let_host_switch_to_returning_410(): # Override with custom info link, if set by host _sunset_info_link = current_app.config.get( - "FLEXMEASURES_API_1_AND_2_SUNSET_LINK", sunset_info_link + "FLEXMEASURES_API_SUNSET_LINK", sunset_info_link ) - if current_app.config["FLEXMEASURES_API_1_AND_2_SUNSET_ACTIVE"]: + if current_app.config["FLEXMEASURES_API_SUNSET_ACTIVE"]: abort( 410, f"API version {api_version_sunset} has been sunset. Please upgrade to API version {api_version_upgrade_to}. See {_sunset_info_link} for more information.", @@ -94,12 +94,12 @@ def _after_request_handler(response: Response) -> Response: # Override sunset date if host used corresponding config setting _sunset = _format_sunset( - current_app.config.get("FLEXMEASURES_API_1_AND_2_SUNSET_DATE", sunset) + current_app.config.get("FLEXMEASURES_API_SUNSET_DATE", sunset) ) # Override sunset link if host used corresponding config setting _sunset_link = current_app.config.get( - "FLEXMEASURES_API_1_AND_2_SUNSET_LINK", sunset_link + "FLEXMEASURES_API_SUNSET_LINK", sunset_link ) return _add_headers( @@ -158,12 +158,12 @@ def _after_request_handler(response: Response) -> Response: # Override sunset date if host used corresponding config setting _sunset = _format_sunset( - current_app.config.get("FLEXMEASURES_API_1_AND_2_SUNSET_DATE", sunset) + current_app.config.get("FLEXMEASURES_API_SUNSET_DATE", sunset) ) # Override sunset link if host used corresponding config setting _sunset_link = current_app.config.get( - "FLEXMEASURES_API_1_AND_2_SUNSET_LINK", sunset_link + "FLEXMEASURES_API_SUNSET_LINK", sunset_link ) return _add_headers( diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 1e272af43..aa2fef97d 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -131,9 +131,9 @@ class Config(object): ) # Custom sunset switches - FLEXMEASURES_API_1_AND_2_SUNSET_ACTIVE: bool = True # if True, sunset endpoints return 410 (Gone) responses; if False, they will work as before - FLEXMEASURES_API_1_AND_2_SUNSET_DATE: str | None = None # e.g. 2023-05-01 - FLEXMEASURES_API_1_AND_2_SUNSET_LINK: str | None = None # e.g. https://flexmeasures.readthedocs.io/en/latest/api/introduction.html#deprecation-and-sunset + FLEXMEASURES_API_SUNSET_ACTIVE: bool = True # if True, sunset endpoints return 410 (Gone) responses; if False, they will work as before + FLEXMEASURES_API_SUNSET_DATE: str | None = None # e.g. 2023-05-01 + FLEXMEASURES_API_SUNSET_LINK: str | None = None # e.g. https://flexmeasures.readthedocs.io/en/latest/api/introduction.html#deprecation-and-sunset # names of settings which cannot be None From 7d3ab42d45102806529865a2033432c83157224d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 28 Apr 2023 15:41:45 +0200 Subject: [PATCH 38/45] shorten comment Signed-off-by: F.N. Claessen --- documentation/configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 8b9533715..1e4597eb7 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -610,11 +610,11 @@ FLEXMEASURES_API_SUNSET_DATE Allow to override the default sunset date for your clients. -Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"2023-05-01"`` for version 2.0) +Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"2023-05-01"`` for v2.0) FLEXMEASURES_API_SUNSET_LINK ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Allow to override the default sunset link for your clients. -Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html"`` for version 2.0) +Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html"`` for v2.0) From 522634f05b1f407e85e09a3bd42f99bb9caed7d5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 28 Apr 2023 16:02:59 +0200 Subject: [PATCH 39/45] fix override from config setting Signed-off-by: F.N. Claessen --- .../api/common/utils/deprecation_utils.py | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/flexmeasures/api/common/utils/deprecation_utils.py b/flexmeasures/api/common/utils/deprecation_utils.py index f7511acb6..f6242a9e1 100644 --- a/flexmeasures/api/common/utils/deprecation_utils.py +++ b/flexmeasures/api/common/utils/deprecation_utils.py @@ -10,7 +10,7 @@ def sunset_blueprint( blueprint, api_version_sunset: str, - sunset_info_link: str, + sunset_link: str, api_version_upgrade_to: str = "3.0", ): """Sunsets every route on a blueprint by returning 410 (Gone) responses. @@ -21,14 +21,16 @@ def sunset_blueprint( def let_host_switch_to_returning_410(): # Override with custom info link, if set by host - _sunset_info_link = current_app.config.get( - "FLEXMEASURES_API_SUNSET_LINK", sunset_info_link - ) + sunset_link_from_config = current_app.config["FLEXMEASURES_API_SUNSET_LINK"] + if sunset_link_from_config is not None: + _sunset_link = sunset_link_from_config + else: + _sunset_link = sunset_link if current_app.config["FLEXMEASURES_API_SUNSET_ACTIVE"]: abort( 410, - f"API version {api_version_sunset} has been sunset. Please upgrade to API version {api_version_upgrade_to}. See {_sunset_info_link} for more information.", + f"API version {api_version_sunset} has been sunset. Please upgrade to API version {api_version_upgrade_to}. See {_sunset_link} for more information.", ) blueprint.before_request(let_host_switch_to_returning_410) @@ -93,14 +95,18 @@ def _after_request_handler(response: Response) -> Response: ) # Override sunset date if host used corresponding config setting - _sunset = _format_sunset( - current_app.config.get("FLEXMEASURES_API_SUNSET_DATE", sunset) - ) + sunset_date_from_config = current_app.config["FLEXMEASURES_API_SUNSET_DATE"] + if sunset_date_from_config is not None: + _sunset = sunset_date_from_config + else: + _sunset = sunset # Override sunset link if host used corresponding config setting - _sunset_link = current_app.config.get( - "FLEXMEASURES_API_SUNSET_LINK", sunset_link - ) + sunset_link_from_config = current_app.config["FLEXMEASURES_API_SUNSET_LINK"] + if sunset_link_from_config is not None: + _sunset_link = sunset_link_from_config + else: + _sunset_link = sunset_link return _add_headers( response, @@ -157,14 +163,18 @@ def _after_request_handler(response: Response) -> Response: ) # Override sunset date if host used corresponding config setting - _sunset = _format_sunset( - current_app.config.get("FLEXMEASURES_API_SUNSET_DATE", sunset) - ) + sunset_date_from_config = current_app.config["FLEXMEASURES_API_SUNSET_DATE"] + if sunset_date_from_config is not None: + _sunset = sunset_date_from_config + else: + _sunset = sunset # Override sunset link if host used corresponding config setting - _sunset_link = current_app.config.get( - "FLEXMEASURES_API_SUNSET_LINK", sunset_link - ) + sunset_link_from_config = current_app.config["FLEXMEASURES_API_SUNSET_LINK"] + if sunset_link_from_config is not None: + _sunset_link = sunset_link_from_config + else: + _sunset_link = sunset_link return _add_headers( response, From 9126a147a3c6f0bb15bf0e4720c6b706b6debfde Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 28 Apr 2023 16:05:24 +0200 Subject: [PATCH 40/45] refactor Signed-off-by: F.N. Claessen --- .../api/common/utils/deprecation_utils.py | 44 ++++++++----------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/flexmeasures/api/common/utils/deprecation_utils.py b/flexmeasures/api/common/utils/deprecation_utils.py index f6242a9e1..f2b207ddf 100644 --- a/flexmeasures/api/common/utils/deprecation_utils.py +++ b/flexmeasures/api/common/utils/deprecation_utils.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Any + from flask import abort, current_app, request, Blueprint, Response, after_this_request from flask_security.core import current_user import pandas as pd @@ -21,11 +23,7 @@ def sunset_blueprint( def let_host_switch_to_returning_410(): # Override with custom info link, if set by host - sunset_link_from_config = current_app.config["FLEXMEASURES_API_SUNSET_LINK"] - if sunset_link_from_config is not None: - _sunset_link = sunset_link_from_config - else: - _sunset_link = sunset_link + _sunset_link = override_from_config(sunset_link, "FLEXMEASURES_API_SUNSET_LINK") if current_app.config["FLEXMEASURES_API_SUNSET_ACTIVE"]: abort( @@ -95,18 +93,12 @@ def _after_request_handler(response: Response) -> Response: ) # Override sunset date if host used corresponding config setting - sunset_date_from_config = current_app.config["FLEXMEASURES_API_SUNSET_DATE"] - if sunset_date_from_config is not None: - _sunset = sunset_date_from_config - else: - _sunset = sunset + _sunset = override_from_config(sunset, "FLEXMEASURES_API_SUNSET_DATE") # Override sunset link if host used corresponding config setting - sunset_link_from_config = current_app.config["FLEXMEASURES_API_SUNSET_LINK"] - if sunset_link_from_config is not None: - _sunset_link = sunset_link_from_config - else: - _sunset_link = sunset_link + _sunset_link = override_from_config( + sunset_link, "FLEXMEASURES_API_SUNSET_LINK" + ) return _add_headers( response, @@ -163,18 +155,10 @@ def _after_request_handler(response: Response) -> Response: ) # Override sunset date if host used corresponding config setting - sunset_date_from_config = current_app.config["FLEXMEASURES_API_SUNSET_DATE"] - if sunset_date_from_config is not None: - _sunset = sunset_date_from_config - else: - _sunset = sunset + _sunset = override_from_config(sunset, "FLEXMEASURES_API_SUNSET_DATE") # Override sunset link if host used corresponding config setting - sunset_link_from_config = current_app.config["FLEXMEASURES_API_SUNSET_LINK"] - if sunset_link_from_config is not None: - _sunset_link = sunset_link_from_config - else: - _sunset_link = sunset_link + _sunset_link = override_from_config(sunset_link, "FLEXMEASURES_API_SUNSET_LINK") return _add_headers( response, @@ -224,3 +208,13 @@ def _format_sunset(sunset_date): else: sunset = None return sunset + + +def override_from_config(setting: Any, config_setting_name: str) -> Any: + """Override setting by config setting, unless the latter is None or is missing.""" + config_setting = current_app.config.get(config_setting_name) + if config_setting is not None: + _setting = config_setting + else: + _setting = setting + return _setting From 5164c8f1235f833c056e304f2d5e419b2c06da03 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 28 Apr 2023 17:15:38 +0200 Subject: [PATCH 41/45] Cross reference to snapshot version of API documentation Signed-off-by: F.N. Claessen --- documentation/configuration.rst | 2 +- flexmeasures/api/v1/__init__.py | 4 ++-- flexmeasures/api/v1_1/__init__.py | 4 ++-- flexmeasures/api/v1_2/__init__.py | 4 ++-- flexmeasures/api/v1_3/__init__.py | 4 ++-- flexmeasures/api/v2_0/__init__.py | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 1e4597eb7..503adfb26 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -617,4 +617,4 @@ FLEXMEASURES_API_SUNSET_LINK Allow to override the default sunset link for your clients. -Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html"`` for v2.0) +Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"https://flexmeasures.readthedocs.io/en/v0.13.0/api/v2_0.html"`` for v2.0) diff --git a/flexmeasures/api/v1/__init__.py b/flexmeasures/api/v1/__init__.py index 7e61a25f9..9fa1b5de1 100644 --- a/flexmeasures/api/v1/__init__.py +++ b/flexmeasures/api/v1/__init__.py @@ -13,10 +13,10 @@ deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/introduction.html#deprecation-and-sunset", sunset_date="2023-05-01", - sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v1.html", + sunset_link="https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1.html", ) sunset_blueprint( - flexmeasures_api, "1.0", "https://flexmeasures.readthedocs.io/en/latest/api/v1.html" + flexmeasures_api, "1.0", "https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1.html" ) diff --git a/flexmeasures/api/v1_1/__init__.py b/flexmeasures/api/v1_1/__init__.py index 1f153a35b..774808627 100644 --- a/flexmeasures/api/v1_1/__init__.py +++ b/flexmeasures/api/v1_1/__init__.py @@ -12,12 +12,12 @@ deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_1.html", sunset_date="2023-05-01", - sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_1.html", + sunset_link="https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1_1.html", ) sunset_blueprint( flexmeasures_api, "1.1", - "https://flexmeasures.readthedocs.io/en/latest/api/v1_1.html", + "https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1_1.html", ) diff --git a/flexmeasures/api/v1_2/__init__.py b/flexmeasures/api/v1_2/__init__.py index 42e12e1d0..09c18def0 100644 --- a/flexmeasures/api/v1_2/__init__.py +++ b/flexmeasures/api/v1_2/__init__.py @@ -12,12 +12,12 @@ deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_2.html", sunset_date="2023-05-01", - sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_2.html", + sunset_link="https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1_2.html", ) sunset_blueprint( flexmeasures_api, "1.2", - "https://flexmeasures.readthedocs.io/en/latest/api/v1_2.html", + "https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1_2.html", ) diff --git a/flexmeasures/api/v1_3/__init__.py b/flexmeasures/api/v1_3/__init__.py index e15f98bd0..ce6308d9f 100644 --- a/flexmeasures/api/v1_3/__init__.py +++ b/flexmeasures/api/v1_3/__init__.py @@ -12,12 +12,12 @@ deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_3.html", sunset_date="2023-05-01", - sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_3.html", + sunset_link="https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1_3.html", ) sunset_blueprint( flexmeasures_api, "1.3", - "https://flexmeasures.readthedocs.io/en/latest/api/v1_3.html", + "https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1_3.html", ) diff --git a/flexmeasures/api/v2_0/__init__.py b/flexmeasures/api/v2_0/__init__.py index c6bb37433..3ee0746e7 100644 --- a/flexmeasures/api/v2_0/__init__.py +++ b/flexmeasures/api/v2_0/__init__.py @@ -11,12 +11,12 @@ deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html", sunset_date="2023-05-01", - sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html", + sunset_link="https://flexmeasures.readthedocs.io/en/v0.13.0/api/v2_0.html", ) sunset_blueprint( flexmeasures_api, "2.0", - "https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html", + "https://flexmeasures.readthedocs.io/en/v0.13.0/api/v2_0.html", ) From 5fee35c6091105193ac992728603928584e68d5d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 28 Apr 2023 17:32:04 +0200 Subject: [PATCH 42/45] We want to support blackout tests already from the version that announces the deprecation and sunset Signed-off-by: F.N. Claessen --- documentation/api/introduction.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 7378f781b..1896a874f 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -150,13 +150,17 @@ Here is a client-side code example in Python (this merely prints out the depreca FlexMeasures hosts ^^^^^^^^^^^^^^^^^^ -When upgrading to a FlexMeasures version that sunsets an API version, clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints. -After upgrading to one of the next FlexMeasures versions, they will receive ``HTTP status 404 (Not Found)`` responses. +When upgrading to a FlexMeasures version that sunsets an API version (e.g. ``flexmeasures==0.13.0`` sunsets API version 2), clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints. +After upgrading to one of the next FlexMeasures versions (e.g. ``flexmeasures==0.14.0``), they will receive ``HTTP status 404 (Not Found)`` responses. Hosts should not expect every client to monitor response headers and proactively upgrade to newer API versions. Please make sure that your users have upgraded before you upgrade to a FlexMeasures version that sunsets an API version. You can do this by checking your server logs for warnings about users who are still calling deprecated endpoints. -In case you have users that haven't upgraded yet, and would still like to upgrade FlexMeasures, you can. -Just set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = False`` and consider announcing some blackout tests to your users, during which you can set this setting to ``True`` to activate the sunset. +In addition, we recommend announcing blackout tests to your users, during which you can set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = True`` to activate the sunset. +Preferably, you'd do this only after the original sunset date. During such a blackout test, clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints. + +In case you have users that haven't upgraded yet, and would still like to upgrade FlexMeasures (to the version that officially sunsets the API version), you can. +For a little while after sunset (usually one more minor version), we will continue to support "letting the sun unset". +To enable this, just set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = False`` and consider announcing some more blackout tests to your users, during which you can set this setting to ``True`` to activate the sunset. From 76284b4057b9f90e16ebafdd3a55b7d7ba94006a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 28 Apr 2023 17:34:04 +0200 Subject: [PATCH 43/45] black Signed-off-by: F.N. Claessen --- flexmeasures/api/v1/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v1/__init__.py b/flexmeasures/api/v1/__init__.py index 9fa1b5de1..0fdf4d196 100644 --- a/flexmeasures/api/v1/__init__.py +++ b/flexmeasures/api/v1/__init__.py @@ -16,7 +16,9 @@ sunset_link="https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1.html", ) sunset_blueprint( - flexmeasures_api, "1.0", "https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1.html" + flexmeasures_api, + "1.0", + "https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1.html", ) From 0d8e6cc43728c0a63a6ebe6083096e25caac8181 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 1 May 2023 13:04:24 +0200 Subject: [PATCH 44/45] Add cross-reference and explanation, and add clarity Signed-off-by: F.N. Claessen --- documentation/api/introduction.rst | 13 +++++++++++-- documentation/configuration.rst | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 1896a874f..3999cb8d0 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -157,10 +157,19 @@ Hosts should not expect every client to monitor response headers and proactively Please make sure that your users have upgraded before you upgrade to a FlexMeasures version that sunsets an API version. You can do this by checking your server logs for warnings about users who are still calling deprecated endpoints. -In addition, we recommend announcing blackout tests to your users, during which you can set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = True`` to activate the sunset. -Preferably, you'd do this only after the original sunset date. +In addition, we recommend running blackout tests during the deprecation notice phase. +You (and your users) can learn which systems need attention and how to deal with them. +Be sure to announce these beforehand. +Here is an example of how to run a blackout test: +If a sunset happens in version ``0.13``, and you are hosting a version which includes the deprecation notice (e.g. ``0.12``), FlexMeasures will simulate the sunset if you set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = True`` (see :ref:`Sunset Configuration`). During such a blackout test, clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints. +.. admonition:: What is a blackout test + + A blackout test is a planned, timeboxed event when a host will turn off a certain API or some of the API capabilities. + The test is meant to help developers understand the impact the retirement will have on the applications and users. + `Source: Platform of Trust `_ + In case you have users that haven't upgraded yet, and would still like to upgrade FlexMeasures (to the version that officially sunsets the API version), you can. For a little while after sunset (usually one more minor version), we will continue to support "letting the sun unset". To enable this, just set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = False`` and consider announcing some more blackout tests to your users, during which you can set this setting to ``True`` to activate the sunset. diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 503adfb26..b3568f5f3 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -593,6 +593,8 @@ so that old imported data can be demoed as if it were current. Default: ``None`` +.. _sunset-config: + Sunset ------ From 1c2a181acda478661a5edc829497a42d389f8bbf Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 1 May 2023 13:27:31 +0200 Subject: [PATCH 45/45] Customize admonition Signed-off-by: F.N. Claessen --- documentation/_static/css/custom.css | 6 +++++- documentation/api/introduction.rst | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/documentation/_static/css/custom.css b/documentation/_static/css/custom.css index 1de969a7d..9cd328ec0 100644 --- a/documentation/_static/css/custom.css +++ b/documentation/_static/css/custom.css @@ -20,4 +20,8 @@ div .contents li { -webkit-column-break-inside: avoid; page-break-inside: avoid; break-inside: avoid-column; -} \ No newline at end of file +} + +div.admonition.info-icon > .admonition-title:before { + content: "\f05a"; /* the fa-circle-info icon */ +} diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 3999cb8d0..5f200b962 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -165,6 +165,7 @@ If a sunset happens in version ``0.13``, and you are hosting a version which inc During such a blackout test, clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints. .. admonition:: What is a blackout test + :class: info-icon A blackout test is a planned, timeboxed event when a host will turn off a certain API or some of the API capabilities. The test is meant to help developers understand the impact the retirement will have on the applications and users.