diff --git a/documentation/changelog.rst b/documentation/changelog.rst index b01fc0502..e579b7c57 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -18,6 +18,7 @@ Bugfixes Infrastructure / Support ---------------------- * Make assets use MW as their default unit and enforce that in CLI, as well (API already did) [see `PR #108 `_] +* Re-use the database between automated tests, if possible. This shaves 2/3rd off of the time it takes for the FlexMeasures test suite to run [see `PR #115 `_] v0.4.1 | May 7, 2021 diff --git a/flexmeasures/api/tests/conftest.py b/flexmeasures/api/tests/conftest.py index ace7cb10d..32c6b57d7 100644 --- a/flexmeasures/api/tests/conftest.py +++ b/flexmeasures/api/tests/conftest.py @@ -7,8 +7,8 @@ from flask_security.utils import hash_password -@pytest.fixture(scope="function", autouse=True) -def setup_api_test_data(db): +@pytest.fixture(scope="module", autouse=True) +def setup_api_test_data(db, setup_roles_users): """ Adding the task-runner """ diff --git a/flexmeasures/api/v1/tests/conftest.py b/flexmeasures/api/v1/tests/conftest.py index 6ebc96298..adb09523f 100644 --- a/flexmeasures/api/v1/tests/conftest.py +++ b/flexmeasures/api/v1/tests/conftest.py @@ -4,27 +4,23 @@ import isodate import pytest -from flask_security import SQLAlchemySessionUserDatastore from flask_security.utils import hash_password from flexmeasures.data.services.users import create_user -@pytest.fixture(scope="function", autouse=True) -def setup_api_test_data(db): +@pytest.fixture(scope="module", autouse=True) +def setup_api_test_data(db, setup_roles_users, add_market_prices): """ Set up data for API v1 tests. """ print("Setting up data for API v1 tests on %s" % db.engine) - from flexmeasures.data.models.user import User, Role from flexmeasures.data.models.assets import Asset, AssetType, Power from flexmeasures.data.models.data_sources import DataSource - user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) - # Create an anonymous user - create_user( + test_anonymous_prosumer = create_user( username="anonymous user with Prosumer role", email="demo@seita.nl", password=hash_password("testtest"), @@ -35,7 +31,6 @@ def setup_api_test_data(db): ) # Create 1 test asset for the anonymous user - test_prosumer = user_datastore.find_user(email="demo@seita.nl") test_asset_type = AssetType(name="test-type") db.session.add(test_asset_type) asset_names = ["CS 0"] @@ -50,7 +45,7 @@ def setup_api_test_data(db): longitude=100, unit="MW", ) - asset.owner = test_prosumer + asset.owner = test_anonymous_prosumer assets.append(asset) db.session.add(asset) @@ -62,7 +57,7 @@ def setup_api_test_data(db): ) # Create 5 test assets for the test_prosumer user - test_prosumer = user_datastore.find_user(email="test_prosumer@seita.nl") + test_prosumer = setup_roles_users["Test Prosumer"] asset_names = ["CS 1", "CS 2", "CS 3", "CS 4", "CS 5"] assets: List[Asset] = [] for asset_name in asset_names: @@ -83,7 +78,7 @@ def setup_api_test_data(db): # Add power forecasts to one of the assets, for two sources cs_5 = Asset.query.filter(Asset.name == "CS 5").one_or_none() - test_supplier = user_datastore.find_user(email="test_supplier@seita.nl") + test_supplier = setup_roles_users["Test Supplier"] prosumer_data_source = DataSource.query.filter( DataSource.user == test_prosumer ).one_or_none() @@ -113,3 +108,32 @@ def setup_api_test_data(db): db.session.bulk_save_objects(meter_data) print("Done setting up data for API v1 tests") + + +@pytest.fixture(scope="function") +def setup_fresh_api_test_data(fresh_db, setup_roles_users_fresh_db): + db = fresh_db + setup_roles_users = setup_roles_users_fresh_db + from flexmeasures.data.models.assets import Asset, AssetType + + # Create 5 test assets for the test_prosumer user + test_prosumer = setup_roles_users["Test Prosumer"] + test_asset_type = AssetType(name="test-type") + db.session.add(test_asset_type) + asset_names = ["CS 1", "CS 2", "CS 3", "CS 4", "CS 5"] + assets: List[Asset] = [] + for asset_name in asset_names: + asset = Asset( + name=asset_name, + asset_type_name="test-type", + event_resolution=timedelta(minutes=15), + capacity_in_mw=1, + latitude=100, + longitude=100, + unit="MW", + ) + asset.owner = test_prosumer + if asset_name == "CS 4": + asset.event_resolution = timedelta(hours=1) + assets.append(asset) + db.session.add(asset) diff --git a/flexmeasures/api/v1/tests/test_api_v1.py b/flexmeasures/api/v1/tests/test_api_v1.py index 2b95713d1..6c72178d0 100644 --- a/flexmeasures/api/v1/tests/test_api_v1.py +++ b/flexmeasures/api/v1/tests/test_api_v1.py @@ -4,9 +4,6 @@ import isodate import pandas as pd import pytest -from iso8601 import parse_date -from numpy import repeat - from flexmeasures.api.common.responses import ( invalid_domain, @@ -24,7 +21,6 @@ verify_power_in_db, ) from flexmeasures.data.auth_setup import UNAUTH_ERROR_STATUS -from flexmeasures.api.v1.tests.utils import count_connections_in_post_message from flexmeasures.data.models.assets import Asset @@ -250,94 +246,7 @@ def test_get_meter_data(db, app, client, message): assert get_meter_data_response.json["values"] == [(100.0 + i) for i in range(6)] -@pytest.mark.parametrize( - "post_message", - [ - message_for_post_meter_data(), - message_for_post_meter_data(single_connection=True), - message_for_post_meter_data(single_connection_group=True), - ], -) -@pytest.mark.parametrize( - "get_message", - [ - message_for_get_meter_data(), - message_for_get_meter_data(single_connection=False), - message_for_get_meter_data(resolution="PT30M"), - ], -) -def test_post_and_get_meter_data(db, app, client, post_message, get_message): - """ - Tries to post meter data as a logged-in test user with the MDC role, which should succeed. - There should be some ForecastingJobs waiting now. - Then tries to get meter data, which should succeed, and should return the same meter data as was posted, - or a downsampled version, if that was requested. - """ - - # post meter data - auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest") - post_meter_data_response = client.post( - url_for("flexmeasures_api_v1.post_meter_data"), - json=message_replace_name_with_ea(post_message), - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_meter_data_response.json) - assert post_meter_data_response.status_code == 200 - assert post_meter_data_response.json["type"] == "PostMeterDataResponse" - - # look for Forecasting jobs - expected_connections = count_connections_in_post_message(post_message) - assert ( - len(app.queues["forecasting"]) == 4 * expected_connections - ) # four horizons times the number of assets - horizons = repeat( - [ - timedelta(hours=1), - timedelta(hours=6), - timedelta(hours=24), - timedelta(hours=48), - ], - expected_connections, - ) - jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"]) - for job, horizon in zip(jobs, horizons): - assert job.kwargs["horizon"] == horizon - assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon - for asset_name in ("CS 1", "CS 2", "CS 3"): - if asset_name in str(post_message): - asset = Asset.query.filter_by(name=asset_name).one_or_none() - assert asset.id in [job.kwargs["asset_id"] for job in jobs] - - # get meter data - get_meter_data_response = client.get( - url_for("flexmeasures_api_v1.get_meter_data"), - query_string=message_replace_name_with_ea(get_message), - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_meter_data_response.json) - assert get_meter_data_response.status_code == 200 - assert get_meter_data_response.json["type"] == "GetMeterDataResponse" - if "groups" in post_message: - posted_values = post_message["groups"][0]["values"] - else: - posted_values = post_message["values"] - if "groups" in get_meter_data_response.json: - gotten_values = get_meter_data_response.json["groups"][0]["values"] - else: - gotten_values = get_meter_data_response.json["values"] - - if "resolution" not in get_message or get_message["resolution"] == "": - assert gotten_values == posted_values - else: - # We used a target resolution of 30 minutes, so double of 15 minutes. - # Six values went in, three come out. - if posted_values[1] > 0: # see utils.py:message_for_post_meter_data - assert gotten_values == [306.66, -0.0, 306.66] - else: - assert gotten_values == [153.33, 0, 306.66] - - -def test_post_meter_data_to_different_resolutions(db, app, client): +def test_post_meter_data_to_different_resolutions(app, client): """ Tries to post meter data to assets with different event_resolutions, which is not accepted. """ diff --git a/flexmeasures/api/v1/tests/test_api_v1_fresh_db.py b/flexmeasures/api/v1/tests/test_api_v1_fresh_db.py new file mode 100644 index 000000000..e3b1c517d --- /dev/null +++ b/flexmeasures/api/v1/tests/test_api_v1_fresh_db.py @@ -0,0 +1,104 @@ +from datetime import timedelta + +import pytest +from flask import url_for +from iso8601 import parse_date +from numpy import repeat + +from flexmeasures.api.common.utils.api_utils import message_replace_name_with_ea +from flexmeasures.api.tests.utils import get_auth_token +from flexmeasures.api.v1.tests.utils import ( + message_for_post_meter_data, + message_for_get_meter_data, + count_connections_in_post_message, +) +from flexmeasures.data.models.assets import Asset + + +@pytest.mark.parametrize( + "post_message", + [ + message_for_post_meter_data(), + message_for_post_meter_data(single_connection=True), + message_for_post_meter_data(single_connection_group=True), + ], +) +@pytest.mark.parametrize( + "get_message", + [ + message_for_get_meter_data(), + message_for_get_meter_data(single_connection=False), + message_for_get_meter_data(resolution="PT30M"), + ], +) +def test_post_and_get_meter_data( + setup_fresh_api_test_data, app, clean_redis, client, post_message, get_message +): + """ + Tries to post meter data as a logged-in test user with the MDC role, which should succeed. + There should be some ForecastingJobs waiting now. + Then tries to get meter data, which should succeed, and should return the same meter data as was posted, + or a downsampled version, if that was requested. + """ + + # post meter data + auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest") + post_meter_data_response = client.post( + url_for("flexmeasures_api_v1.post_meter_data"), + json=message_replace_name_with_ea(post_message), + headers={"Authorization": auth_token}, + ) + print("Server responded with:\n%s" % post_meter_data_response.json) + assert post_meter_data_response.status_code == 200 + assert post_meter_data_response.json["type"] == "PostMeterDataResponse" + + # look for Forecasting jobs + expected_connections = count_connections_in_post_message(post_message) + assert ( + len(app.queues["forecasting"]) == 4 * expected_connections + ) # four horizons times the number of assets + horizons = repeat( + [ + timedelta(hours=1), + timedelta(hours=6), + timedelta(hours=24), + timedelta(hours=48), + ], + expected_connections, + ) + jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"]) + for job, horizon in zip(jobs, horizons): + assert job.kwargs["horizon"] == horizon + assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon + for asset_name in ("CS 1", "CS 2", "CS 3"): + if asset_name in str(post_message): + asset = Asset.query.filter_by(name=asset_name).one_or_none() + assert asset.id in [job.kwargs["asset_id"] for job in jobs] + + # get meter data + get_meter_data_response = client.get( + url_for("flexmeasures_api_v1.get_meter_data"), + query_string=message_replace_name_with_ea(get_message), + headers={"Authorization": auth_token}, + ) + print("Server responded with:\n%s" % get_meter_data_response.json) + assert get_meter_data_response.status_code == 200 + assert get_meter_data_response.json["type"] == "GetMeterDataResponse" + if "groups" in post_message: + posted_values = post_message["groups"][0]["values"] + else: + posted_values = post_message["values"] + if "groups" in get_meter_data_response.json: + gotten_values = get_meter_data_response.json["groups"][0]["values"] + else: + gotten_values = get_meter_data_response.json["values"] + + if "resolution" not in get_message or get_message["resolution"] == "": + assert gotten_values == posted_values + else: + # We used a target resolution of 30 minutes, so double of 15 minutes. + # Six values went in, three come out. + if posted_values[1] > 0: # see utils.py:message_for_post_meter_data + assert gotten_values == [306.66, -0.0, 306.66] + else: + assert gotten_values == [153.33, 0, 306.66] diff --git a/flexmeasures/api/v1_1/tests/conftest.py b/flexmeasures/api/v1_1/tests/conftest.py index 578e357d1..1fe0ebbcd 100644 --- a/flexmeasures/api/v1_1/tests/conftest.py +++ b/flexmeasures/api/v1_1/tests/conftest.py @@ -11,8 +11,8 @@ from flexmeasures.data.services.users import create_user -@pytest.fixture(scope="function", autouse=True) -def setup_api_test_data(db): +@pytest.fixture(scope="module") +def setup_api_test_data(db, setup_roles_users, add_market_prices): """ Set up data for API v1.1 tests. """ @@ -41,7 +41,7 @@ def setup_api_test_data(db): ) # Create 3 test assets for the test_prosumer user - test_prosumer = user_datastore.find_user(email="test_prosumer@seita.nl") + test_prosumer = setup_roles_users["Test Prosumer"] test_asset_type = AssetType(name="test-type") db.session.add(test_asset_type) asset_names = ["CS 1", "CS 2", "CS 3"] @@ -124,3 +124,10 @@ def setup_api_test_data(db): db.session.add(sensor) print("Done setting up data for API v1.1 tests") + + +@pytest.fixture(scope="function") +def setup_fresh_api_v1_1_test_data( + fresh_db, setup_roles_users_fresh_db, setup_markets_fresh_db +): + return fresh_db diff --git a/flexmeasures/api/v1_1/tests/test_api_v1_1.py b/flexmeasures/api/v1_1/tests/test_api_v1_1.py index 0ce5519c7..88d540b0c 100644 --- a/flexmeasures/api/v1_1/tests/test_api_v1_1.py +++ b/flexmeasures/api/v1_1/tests/test_api_v1_1.py @@ -1,14 +1,12 @@ from flask import url_for import pytest from datetime import timedelta -from isodate import duration_isoformat from iso8601 import parse_date from flexmeasures.utils.entity_address_utils import parse_entity_address from flexmeasures.api.common.responses import ( request_processed, invalid_horizon, - unapplicable_resolution, invalid_unit, ) from flexmeasures.api.tests.utils import get_auth_token @@ -63,7 +61,7 @@ def test_unauthorized_prognosis_request(client): message_for_get_prognosis(invalid_horizon=True), ], ) -def test_invalid_horizon(client, message): +def test_invalid_horizon(setup_api_test_data, client, message): auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest") get_prognosis_response = client.get( url_for("flexmeasures_api_v1_1.get_prognosis"), @@ -76,7 +74,7 @@ def test_invalid_horizon(client, message): assert get_prognosis_response.json["status"] == invalid_horizon()[0]["status"] -def test_no_data(client): +def test_no_data(setup_api_test_data, client): auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest") get_prognosis_response = client.get( url_for("flexmeasures_api_v1_1.get_prognosis"), @@ -103,7 +101,7 @@ def test_no_data(client): message_for_get_prognosis(rolling_horizon=True, timezone_alternative=True), ], ) -def test_get_prognosis(client, message): +def test_get_prognosis(setup_api_test_data, client, message): auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest") get_prognosis_response = client.get( url_for("flexmeasures_api_v1_1.get_prognosis"), @@ -126,7 +124,7 @@ def test_get_prognosis(client, message): @pytest.mark.parametrize("post_message", [message_for_post_price_data()]) -def test_post_price_data(db, app, post_message): +def test_post_price_data(setup_api_test_data, db, app, clean_redis, post_message): """ Try to post price data as a logged-in test user with the Supplier role, which should succeed. """ @@ -163,7 +161,7 @@ def test_post_price_data(db, app, post_message): @pytest.mark.parametrize( "post_message", [message_for_post_price_data(invalid_unit=True)] ) -def test_post_price_data_invalid_unit(client, post_message): +def test_post_price_data_invalid_unit(setup_api_test_data, client, post_message): """ Try to post price data with the wrong unit, which should fail. """ @@ -187,46 +185,11 @@ def test_post_price_data_invalid_unit(client, post_message): ) -@pytest.mark.parametrize( - "post_message,status,msg", - [ - ( - message_for_post_price_data( - duration=duration_isoformat(timedelta(minutes=2)) - ), - 400, - unapplicable_resolution()[0]["message"], - ), - (message_for_post_price_data(compress_n=4), 200, "Request has been processed."), - ], -) -def test_post_price_data_unexpected_resolution(db, app, post_message, status, msg): - """ - Try to post price data with an unexpected resolution, - which might be fixed with upsampling or otherwise fail. - """ - with app.test_client() as client: - auth_token = get_auth_token(client, "test_supplier@seita.nl", "testtest") - post_price_data_response = client.post( - url_for("flexmeasures_api_v1_1.post_price_data"), - json=post_message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_price_data_response.json) - assert post_price_data_response.json["type"] == "PostPriceDataResponse" - assert post_price_data_response.status_code == status - assert msg in post_price_data_response.json["message"] - if "processed" in msg: - verify_prices_in_db( - post_message, [v for v in post_message["values"] for i in range(4)], db - ) - - @pytest.mark.parametrize( "post_message", [message_for_post_weather_data(), message_for_post_weather_data(temperature=True)], ) -def test_post_weather_data(client, post_message): +def test_post_weather_data(setup_api_test_data, client, post_message): """ Try to post wind speed data as a logged-in test user with the Supplier role, which should succeed. """ @@ -246,7 +209,7 @@ def test_post_weather_data(client, post_message): @pytest.mark.parametrize( "post_message", [message_for_post_weather_data(invalid_unit=True)] ) -def test_post_weather_data_invalid_unit(client, post_message): +def test_post_weather_data_invalid_unit(setup_api_test_data, client, post_message): """ Try to post wind speed data as a logged-in test user with the Supplier role, but with a wrong unit for wind speed, which should fail. @@ -269,7 +232,9 @@ def test_post_weather_data_invalid_unit(client, post_message): @pytest.mark.parametrize("post_message", [message_for_post_price_data()]) -def test_auto_fix_missing_registration_of_user_as_data_source(client, post_message): +def test_auto_fix_missing_registration_of_user_as_data_source( + setup_api_test_data, client, post_message +): """Try to post price data as a user that has not been properly registered as a data source. The API call should succeed and the user should be automatically registered as a data source. """ diff --git a/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py b/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py new file mode 100644 index 000000000..2a984fb94 --- /dev/null +++ b/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py @@ -0,0 +1,49 @@ +from datetime import timedelta + +import pytest +from flask import url_for +from isodate import duration_isoformat + +from flexmeasures.api.common.responses import unapplicable_resolution +from flexmeasures.api.tests.utils import get_auth_token +from flexmeasures.api.v1_1.tests.utils import ( + message_for_post_price_data, + verify_prices_in_db, +) + + +@pytest.mark.parametrize( + "post_message, status, msg", + [ + ( + message_for_post_price_data( + duration=duration_isoformat(timedelta(minutes=2)) + ), + 400, + unapplicable_resolution()[0]["message"], + ), + (message_for_post_price_data(compress_n=4), 200, "Request has been processed."), + ], +) +def test_post_price_data_unexpected_resolution( + setup_fresh_api_v1_1_test_data, app, client, post_message, status, msg +): + """ + Try to post price data with an unexpected resolution, + which might be fixed with upsampling or otherwise fail. + """ + db = setup_fresh_api_v1_1_test_data + auth_token = get_auth_token(client, "test_supplier@seita.nl", "testtest") + post_price_data_response = client.post( + url_for("flexmeasures_api_v1_1.post_price_data"), + json=post_message, + headers={"Authorization": auth_token}, + ) + print("Server responded with:\n%s" % post_price_data_response.json) + assert post_price_data_response.json["type"] == "PostPriceDataResponse" + assert post_price_data_response.status_code == status + assert msg in post_price_data_response.json["message"] + if "processed" in msg: + verify_prices_in_db( + post_message, [v for v in post_message["values"] for i in range(4)], db + ) diff --git a/flexmeasures/api/v1_2/tests/conftest.py b/flexmeasures/api/v1_2/tests/conftest.py index 7a0f45011..947a82df5 100644 --- a/flexmeasures/api/v1_2/tests/conftest.py +++ b/flexmeasures/api/v1_2/tests/conftest.py @@ -1,19 +1,9 @@ -from flask_security import SQLAlchemySessionUserDatastore import pytest -@pytest.fixture(scope="function", autouse=True) -def setup_api_test_data(db): +@pytest.fixture(scope="module", autouse=True) +def setup_api_test_data(db, add_market_prices, add_battery_assets): """ Set up data for API v1.2 tests. """ print("Setting up data for API v1.2 tests on %s" % db.engine) - - from flexmeasures.data.models.user import User, Role - from flexmeasures.data.models.assets import Asset - - user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) - test_prosumer = user_datastore.find_user(email="test_prosumer@seita.nl") - - battery = Asset.query.filter(Asset.name == "Test battery").one_or_none() - battery.owner = test_prosumer diff --git a/flexmeasures/api/v1_3/tests/conftest.py b/flexmeasures/api/v1_3/tests/conftest.py index a0c949463..c19263788 100644 --- a/flexmeasures/api/v1_3/tests/conftest.py +++ b/flexmeasures/api/v1_3/tests/conftest.py @@ -1,24 +1,17 @@ -from flask_security import SQLAlchemySessionUserDatastore import pytest -@pytest.fixture(scope="function", autouse=True) -def setup_api_test_data(db): +@pytest.fixture(scope="module", autouse=True) +def setup_api_test_data(db, add_market_prices, add_battery_assets): """ Set up data for API v1.3 tests. """ print("Setting up data for API v1.3 tests on %s" % db.engine) - from flexmeasures.data.models.user import User, Role - from flexmeasures.data.models.assets import Asset - user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) - test_prosumer = user_datastore.find_user(email="test_prosumer@seita.nl") - - battery = Asset.query.filter(Asset.name == "Test battery").one_or_none() - battery.owner = test_prosumer - - charging_station = Asset.query.filter( - Asset.name == "Test charging station" - ).one_or_none() - charging_station.owner = test_prosumer +@pytest.fixture(scope="function") +def setup_fresh_api_test_data(fresh_db, add_battery_assets_fresh_db): + """ + Set up data for API v1.3 tests. + """ + pass diff --git a/flexmeasures/api/v1_3/tests/test_api_v1_3.py b/flexmeasures/api/v1_3/tests/test_api_v1_3.py index f0b64193e..ac94590d3 100644 --- a/flexmeasures/api/v1_3/tests/test_api_v1_3.py +++ b/flexmeasures/api/v1_3/tests/test_api_v1_3.py @@ -6,7 +6,7 @@ import pandas as pd from rq.job import Job -from flexmeasures.api.common.responses import unrecognized_event, unknown_schedule +from flexmeasures.api.common.responses import unrecognized_event from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v1_3.tests.utils import ( message_for_get_device_message, @@ -45,7 +45,9 @@ def test_get_device_message_wrong_event_id(client, message): (message_for_post_udi_event(targets=True), "Test charging station"), ], ) -def test_post_udi_event_and_get_device_message(app, message, asset_name): +def test_post_udi_event_and_get_device_message( + app, add_charging_station_assets, message, asset_name +): auth_token = None with app.test_client() as client: asset = Asset.query.filter(Asset.name == asset_name).one_or_none() @@ -198,68 +200,3 @@ def test_post_udi_event_and_get_device_message(app, message, asset_name): ).is_failed is True ) - - -@pytest.mark.parametrize("message", [message_for_post_udi_event(unknown_prices=True)]) -def test_post_udi_event_and_get_device_message_with_unknown_prices(app, message): - auth_token = None - with app.test_client() as client: - asset = Asset.query.filter(Asset.name == "Test battery").one_or_none() - asset_id = asset.id - asset_owner_id = asset.owner_id - message["event"] = message["event"] % (asset.owner_id, asset.id) - auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest") - post_udi_event_response = client.post( - url_for("flexmeasures_api_v1_3.post_udi_event"), - json=message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_udi_event_response.json) - assert post_udi_event_response.status_code == 200 - assert post_udi_event_response.json["type"] == "PostUdiEventResponse" - - # look for scheduling jobs in queue - assert ( - len(app.queues["scheduling"]) == 1 - ) # only 1 schedule should be made for 1 asset - job = app.queues["scheduling"].jobs[0] - assert job.kwargs["asset_id"] == asset_id - assert job.kwargs["start"] == parse_datetime(message["datetime"]) - assert job.id == message["event"] - assert ( - Job.fetch(message["event"], connection=app.queues["scheduling"].connection) - == job - ) - - # process the scheduling queue - work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) - processed_job = Job.fetch( - message["event"], connection=app.queues["scheduling"].connection - ) - assert processed_job.is_failed is True - - # check results are not in the database - scheduler_source = DataSource.query.filter_by( - name="Seita", type="scheduling script" - ).one_or_none() - assert ( - scheduler_source is None - ) # Make sure the scheduler data source is still not there - - # try to retrieve the schedule through the getDeviceMessage api endpoint - message = message_for_get_device_message() - message["event"] = message["event"] % (asset_owner_id, asset_id) - auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest") - get_device_message_response = client.get( - url_for("flexmeasures_api_v1_3.get_device_message"), - query_string=message, - headers={"content-type": "application/json", "Authorization": auth_token}, - ) - print("Server responded with:\n%s" % get_device_message_response.json) - assert get_device_message_response.status_code == 400 - assert get_device_message_response.json["type"] == "GetDeviceMessageResponse" - assert ( - get_device_message_response.json["status"] - == unknown_schedule()[0]["status"] - ) - assert "prices unknown" in get_device_message_response.json["message"].lower() diff --git a/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py b/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py new file mode 100644 index 000000000..68c901d76 --- /dev/null +++ b/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py @@ -0,0 +1,82 @@ +import pytest +from flask import url_for +from isodate import parse_datetime +from rq.job import Job + +from flexmeasures.api.common.responses import unknown_schedule +from flexmeasures.api.tests.utils import get_auth_token +from flexmeasures.api.v1_3.tests.utils import ( + message_for_post_udi_event, + message_for_get_device_message, +) +from flexmeasures.data.models.assets import Asset +from flexmeasures.data.models.data_sources import DataSource +from flexmeasures.data.services.scheduling import handle_scheduling_exception +from flexmeasures.data.tests.utils import work_on_rq + + +@pytest.mark.parametrize("message", [message_for_post_udi_event(unknown_prices=True)]) +def test_post_udi_event_and_get_device_message_with_unknown_prices( + setup_fresh_api_test_data, clean_redis, app, message +): + auth_token = None + with app.test_client() as client: + asset = Asset.query.filter(Asset.name == "Test battery").one_or_none() + asset_id = asset.id + asset_owner_id = asset.owner_id + message["event"] = message["event"] % (asset.owner_id, asset.id) + auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest") + post_udi_event_response = client.post( + url_for("flexmeasures_api_v1_3.post_udi_event"), + json=message, + headers={"Authorization": auth_token}, + ) + print("Server responded with:\n%s" % post_udi_event_response.json) + assert post_udi_event_response.status_code == 200 + assert post_udi_event_response.json["type"] == "PostUdiEventResponse" + + # look for scheduling jobs in queue + assert ( + len(app.queues["scheduling"]) == 1 + ) # only 1 schedule should be made for 1 asset + job = app.queues["scheduling"].jobs[0] + assert job.kwargs["asset_id"] == asset_id + assert job.kwargs["start"] == parse_datetime(message["datetime"]) + assert job.id == message["event"] + assert ( + Job.fetch(message["event"], connection=app.queues["scheduling"].connection) + == job + ) + + # process the scheduling queue + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + processed_job = Job.fetch( + message["event"], connection=app.queues["scheduling"].connection + ) + assert processed_job.is_failed is True + + # check results are not in the database + scheduler_source = DataSource.query.filter_by( + name="Seita", type="scheduling script" + ).one_or_none() + assert ( + scheduler_source is None + ) # Make sure the scheduler data source is still not there + + # try to retrieve the schedule through the getDeviceMessage api endpoint + message = message_for_get_device_message() + message["event"] = message["event"] % (asset_owner_id, asset_id) + auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest") + get_device_message_response = client.get( + url_for("flexmeasures_api_v1_3.get_device_message"), + query_string=message, + headers={"content-type": "application/json", "Authorization": auth_token}, + ) + print("Server responded with:\n%s" % get_device_message_response.json) + assert get_device_message_response.status_code == 400 + assert get_device_message_response.json["type"] == "GetDeviceMessageResponse" + assert ( + get_device_message_response.json["status"] + == unknown_schedule()[0]["status"] + ) + assert "prices unknown" in get_device_message_response.json["message"].lower() diff --git a/flexmeasures/api/v2_0/tests/conftest.py b/flexmeasures/api/v2_0/tests/conftest.py index 55034397b..783e330f6 100644 --- a/flexmeasures/api/v2_0/tests/conftest.py +++ b/flexmeasures/api/v2_0/tests/conftest.py @@ -3,27 +3,33 @@ import pytest -@pytest.fixture(scope="function", autouse=True) -def setup_api_test_data(db): +@pytest.fixture(scope="module", autouse=True) +def setup_api_test_data(db, setup_roles_users, add_market_prices, add_battery_assets): """ Set up data for API v2.0 tests. """ print("Setting up data for API v2.0 tests on %s" % db.engine) from flexmeasures.data.models.user import User, Role - from flexmeasures.data.models.assets import Asset user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) - test_supplier = user_datastore.find_user(email="test_supplier@seita.nl") - battery = Asset.query.filter(Asset.name == "Test battery").one_or_none() - battery.owner = test_supplier + battery = add_battery_assets["Test battery"] + battery.owner = setup_roles_users["Test Supplier"] test_prosumer = user_datastore.find_user(email="test_prosumer@seita.nl") admin_role = user_datastore.create_role(name="admin", description="God powers") user_datastore.add_role_to_user(test_prosumer, admin_role) - # an inactive user + +@pytest.fixture(scope="module") +def setup_inactive_user(db, setup_roles_users): + """ + Set up one inactive user. + """ + from flexmeasures.data.models.user import User, Role + + user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) user_datastore.create_user( username="inactive test user", email="inactive@seita.nl", diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py index 201baddd3..edde65c43 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py @@ -1,6 +1,8 @@ from flask import url_for import pytest +import pandas as pd + from flexmeasures.data.models.assets import Asset from flexmeasures.data.services.users import find_user_by_email from flexmeasures.api.tests.utils import get_auth_token, UserContext @@ -68,8 +70,8 @@ def test_get_asset_nonadmin_access(client): assert "not found" in asset_response.json["message"] -@pytest.mark.parametrize("use_owner_id,num_assets", [(False, 7), (True, 1)]) -def test_get_assets(client, use_owner_id, num_assets): +@pytest.mark.parametrize("use_owner_id, num_assets", [(False, 7), (True, 1)]) +def test_get_assets(client, add_charging_station_assets, use_owner_id, num_assets): """ Get assets, either for all users (prosumer is admin, so is allowed to see all 7 assets) or for a unique one (supplier user has one asset ― "Test battery"). @@ -95,7 +97,9 @@ def test_get_assets(client, use_owner_id, num_assets): if asset["name"] == "Test battery": battery = asset assert battery - assert battery["soc_datetime"] == "2015-01-01T00:00:00+00:00" + assert pd.Timestamp(battery["soc_datetime"]) == pd.Timestamp( + "2015-01-01T00:00:00+00:00" + ) assert battery["owner_id"] == test_supplier_id assert battery["capacity_in_mw"] == 2 diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py index 6462e292e..31dfd55e4 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py @@ -1,60 +1,13 @@ from flask import url_for import pytest -from datetime import timedelta -from iso8601 import parse_date -from flexmeasures.api.common.utils.api_utils import get_generic_asset from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v2_0.tests.utils import ( - message_for_post_price_data, message_for_post_prognosis, verify_sensor_data_in_db, ) -@pytest.mark.parametrize( - "post_message", - [ - message_for_post_price_data(), - message_for_post_price_data(prior_instead_of_horizon=True), - ], -) -def test_post_price_data_2_0(db, app, post_message): - """ - Try to post price data as a logged-in test user with the Supplier role, which should succeed. - """ - # call with client whose context ends, so that we can test for, - # after-effects in the database after teardown committed. - with app.test_client() as client: - # post price data - auth_token = get_auth_token(client, "test_supplier@seita.nl", "testtest") - post_price_data_response = client.post( - url_for("flexmeasures_api_v2_0.post_price_data"), - json=post_message, - headers={"Authorization": auth_token}, - ) - print("Server responded with:\n%s" % post_price_data_response.json) - assert post_price_data_response.status_code == 200 - assert post_price_data_response.json["type"] == "PostPriceDataResponse" - - verify_sensor_data_in_db( - post_message, post_message["values"], db, entity_type="market" - ) - - # look for Forecasting jobs in queue - assert ( - len(app.queues["forecasting"]) == 2 - ) # only one market is affected, but two horizons - horizons = [timedelta(hours=24), timedelta(hours=48)] - jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"]) - market = get_generic_asset(post_message["market"], "market") - for job, horizon in zip(jobs, horizons): - assert job.kwargs["horizon"] == horizon - assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon - assert job.kwargs["timed_value_type"] == "Price" - assert job.kwargs["asset_id"] == market.id - - @pytest.mark.parametrize( "post_message", [ diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors_fresh_db.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors_fresh_db.py new file mode 100644 index 000000000..49c2fc50e --- /dev/null +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors_fresh_db.py @@ -0,0 +1,63 @@ +from datetime import timedelta + +import pytest +from flask import url_for +from iso8601 import parse_date + +from flexmeasures.api.common.utils.api_utils import get_generic_asset +from flexmeasures.api.tests.utils import get_auth_token +from flexmeasures.api.v2_0.tests.utils import ( + message_for_post_price_data, + verify_sensor_data_in_db, +) + + +@pytest.mark.parametrize( + "post_message", + [ + message_for_post_price_data(), + message_for_post_price_data(prior_instead_of_horizon=True), + ], +) +def test_post_price_data_2_0( + fresh_db, + setup_roles_users_fresh_db, + setup_markets_fresh_db, + clean_redis, + app, + post_message, +): + """ + Try to post price data as a logged-in test user with the Supplier role, which should succeed. + """ + db = fresh_db + # call with client whose context ends, so that we can test for, + # after-effects in the database after teardown committed. + with app.test_client() as client: + # post price data + auth_token = get_auth_token(client, "test_supplier@seita.nl", "testtest") + post_price_data_response = client.post( + url_for("flexmeasures_api_v2_0.post_price_data"), + json=post_message, + headers={"Authorization": auth_token}, + ) + print("Server responded with:\n%s" % post_price_data_response.json) + assert post_price_data_response.status_code == 200 + assert post_price_data_response.json["type"] == "PostPriceDataResponse" + + verify_sensor_data_in_db( + post_message, post_message["values"], db, entity_type="market" + ) + + # look for Forecasting jobs in queue + assert ( + len(app.queues["forecasting"]) == 2 + ) # only one market is affected, but two horizons + horizons = [timedelta(hours=24), timedelta(hours=48)] + jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"]) + market = get_generic_asset(post_message["market"], "market") + for job, horizon in zip(jobs, horizons): + assert job.kwargs["horizon"] == horizon + assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon + assert job.kwargs["timed_value_type"] == "Price" + assert job.kwargs["asset_id"] == market.id diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_users.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_users.py index 410d883ed..771d8a8f7 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_users.py +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_users.py @@ -1,4 +1,4 @@ -from flask import url_for, request +from flask import url_for import pytest # from flexmeasures.data.models.user import User @@ -32,7 +32,7 @@ def test_get_users_bad_auth(client, use_auth): @pytest.mark.parametrize("include_inactive", [False, True]) -def test_get_users_inactive(client, include_inactive): +def test_get_users_inactive(client, setup_inactive_user, include_inactive): headers = { "content-type": "application/json", "Authorization": get_auth_token(client, "test_prosumer@seita.nl", "testtest"), @@ -117,7 +117,7 @@ def test_edit_user(client): def test_edit_user_with_unexpected_fields(client): - """Sending unexpected fields (not in Schema) is an Unprocessible Entity error.""" + """Sending unexpected fields (not in Schema) is an Unprocessable Entity error.""" with UserContext("test_supplier@seita.nl") as supplier: supplier_id = supplier.id with UserContext("test_prosumer@seita.nl") as prosumer: @@ -132,54 +132,3 @@ def test_edit_user_with_unexpected_fields(client): ) print("Server responded with:\n%s" % user_edit_response.json) assert user_edit_response.status_code == 422 - - -@pytest.mark.parametrize( - "sender", - ( - (""), - ("test_supplier@seita.nl"), - ("test_prosumer@seita.nl"), - ("test_prosumer@seita.nl"), - ("test_prosumer@seita.nl"), - ), -) -def test_user_reset_password(app, client, sender): - """ - Reset the password of supplier. - Only the prosumer is allowed to do that (as admin). - """ - with UserContext("test_supplier@seita.nl") as supplier: - supplier_id = supplier.id - old_password = supplier.password - headers = {"content-type": "application/json"} - if sender != "": - headers["Authorization"] = (get_auth_token(client, sender, "testtest"),) - with app.mail.record_messages() as outbox: - pwd_reset_response = client.patch( - url_for("flexmeasures_api_v2_0.reset_user_password", id=supplier_id), - query_string={}, - headers=headers, - ) - print("Server responded with:\n%s" % pwd_reset_response.json) - - if sender == "": - assert pwd_reset_response.status_code == 401 - return - - if sender == "test_supplier@seita.nl": - assert pwd_reset_response.status_code == 403 - return - - assert pwd_reset_response.status_code == 200 - - supplier = find_user_by_email("test_supplier@seita.nl") - assert len(outbox) == 2 - assert "has been reset" in outbox[0].subject - pwd_reset_instructions = outbox[1] - assert old_password != supplier.password - assert "reset instructions" in pwd_reset_instructions.subject - assert ( - "reset your password:\n\n%sreset/" % request.host_url - in pwd_reset_instructions.body - ) diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_users_fresh_db.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_users_fresh_db.py new file mode 100644 index 000000000..cfb2da44d --- /dev/null +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_users_fresh_db.py @@ -0,0 +1,56 @@ +import pytest +from flask import url_for, request + +from flexmeasures.api.tests.utils import UserContext, get_auth_token +from flexmeasures.data.services.users import find_user_by_email + + +@pytest.mark.parametrize( + "sender", + ( + (""), + ("test_supplier@seita.nl"), + ("test_prosumer@seita.nl"), + ("test_prosumer@seita.nl"), + ("test_prosumer@seita.nl"), + ), +) +def test_user_reset_password(app, client, sender): + """ + Reset the password of supplier. + Only the prosumer is allowed to do that (as admin). + """ + with UserContext("test_supplier@seita.nl") as supplier: + supplier_id = supplier.id + old_password = supplier.password + headers = {"content-type": "application/json"} + if sender != "": + headers["Authorization"] = (get_auth_token(client, sender, "testtest"),) + with app.mail.record_messages() as outbox: + pwd_reset_response = client.patch( + url_for("flexmeasures_api_v2_0.reset_user_password", id=supplier_id), + query_string={}, + headers=headers, + ) + print("Server responded with:\n%s" % pwd_reset_response.json) + + if sender == "": + assert pwd_reset_response.status_code == 401 + return + + if sender == "test_supplier@seita.nl": + assert pwd_reset_response.status_code == 403 + return + + assert pwd_reset_response.status_code == 200 + + supplier = find_user_by_email("test_supplier@seita.nl") + assert len(outbox) == 2 + assert "has been reset" in outbox[0].subject + pwd_reset_instructions = outbox[1] + assert old_password != supplier.password + assert "reset instructions" in pwd_reset_instructions.subject + assert ( + "reset your password:\n\n%sreset/" % request.host_url + in pwd_reset_instructions.body + ) diff --git a/flexmeasures/api/v2_0/tests/utils.py b/flexmeasures/api/v2_0/tests/utils.py index a7d0e311f..410d83f00 100644 --- a/flexmeasures/api/v2_0/tests/utils.py +++ b/flexmeasures/api/v2_0/tests/utils.py @@ -134,7 +134,7 @@ def verify_sensor_data_in_db( def message_for_post_prognosis(): message = { "type": "PostPrognosisRequest", - "connection": "ea1.2018-06.localhost:1:2", + "connection": "ea1.2018-06.localhost:2:5", "values": [300, 300, 300, 0, 0, 300], "start": "2021-01-01T00:00:00Z", "duration": "PT1H30M", diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index ebde5931c..15c1dd3d6 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -1,13 +1,15 @@ +from contextlib import contextmanager import pytest from random import random from datetime import datetime, timedelta +from typing import Dict from isodate import parse_duration import pandas as pd import numpy as np from flask import request, jsonify from flask_sqlalchemy import SQLAlchemy -from flask_security import roles_accepted, SQLAlchemySessionUserDatastore +from flask_security import roles_accepted from flask_security.utils import hash_password from werkzeug.exceptions import ( InternalServerError, @@ -19,19 +21,38 @@ from flexmeasures.app import create as create_app from flexmeasures.utils.time_utils import as_server_time -from flexmeasures.data.services.users import create_user, find_user_by_email +from flexmeasures.data.services.users import create_user from flexmeasures.data.models.assets import AssetType, Asset, Power from flexmeasures.data.models.data_sources import DataSource -from flexmeasures.data.models.markets import Market, Price +from flexmeasures.data.models.markets import Market, MarketType, Price from flexmeasures.data.models.time_series import Sensor, TimedBelief +from flexmeasures.data.models.user import User """ Useful things for all tests. -One application is made per test session, but cleanup and recreation currently happens per test. -This can be sped up if needed by moving some functions to "module" or even "session" scope, -but then the tests need to share data and and data modifications can lead to tricky debugging. +# App + +One application is made per test session. + +# Database + +Database recreation and cleanup can happen per test (use fresh_db) or per module (use db). +Having tests inside a module share a database makes those tests faster. +Tests that use fresh_db should be put in a separate module to avoid clashing with the module scoped test db. +For example: +- test_api_v1_1.py contains tests that share a module scoped database +- test_api_v1_1_fresh_db.py contains tests that each get a fresh function-scoped database +Further speed-up may be possible by defining a "package" scoped or even "session" scoped database, +but then tests in different modules need to share data and data modifications can lead to tricky debugging. + +# Data + +Various fixture below set up data that many tests use. +In case a test needs to use such data with a fresh test database, +that test should also use a fixture that requires the fresh_db. +Such fixtures can be recognised by having fresh_db appended to their name. """ @@ -51,8 +72,22 @@ def app(): print("DONE WITH APP FIXTURE") -@pytest.fixture(scope="function") +@pytest.fixture(scope="module") def db(app): + """Fresh test db per module.""" + with create_test_db(app) as test_db: + yield test_db + + +@pytest.fixture(scope="function") +def fresh_db(app): + """Fresh test db per function.""" + with create_test_db(app) as test_db: + yield test_db + + +@contextmanager +def create_test_db(app): """ Provide a db object with the structure freshly created. This assumes a clean database. It does clean up after itself when it's done (drops everything). @@ -74,27 +109,45 @@ def db(app): _db.drop_all() +@pytest.fixture(scope="module") +def setup_roles_users() -> Dict[str, User]: + return create_roles_users() + + @pytest.fixture(scope="function") -def setup_roles_users(db): +def setup_roles_users_fresh_db() -> Dict[str, User]: + return create_roles_users() + + +def create_roles_users() -> Dict[str, User]: """Create a minimal set of roles and users""" - create_user( + test_prosumer = create_user( username="Test Prosumer", email="test_prosumer@seita.nl", password=hash_password("testtest"), user_roles=dict(name="Prosumer", description="A Prosumer with a few assets."), ) - create_user( + test_supplier = create_user( username="Test Supplier", email="test_supplier@seita.nl", password=hash_password("testtest"), user_roles=dict(name="Supplier", description="A Supplier trading on markets."), ) + return {"Test Prosumer": test_prosumer, "Test Supplier": test_supplier} + +@pytest.fixture(scope="module") +def setup_markets(db) -> Dict[str, Market]: + return create_test_markets(db) -@pytest.fixture(scope="function", autouse=True) -def setup_markets(db): + +@pytest.fixture(scope="function") +def setup_markets_fresh_db(fresh_db) -> Dict[str, Market]: + return create_test_markets(fresh_db) + + +def create_test_markets(db) -> Dict[str, Market]: """Create the epex_da market.""" - from flexmeasures.data.models.markets import Market, MarketType day_ahead = MarketType( name="day_ahead", @@ -112,37 +165,55 @@ def setup_markets(db): knowledge_horizon_par={"x": 1, "y": 12, "z": "Europe/Paris"}, ) db.session.add(epex_da) + return {"epex_da": epex_da} -@pytest.fixture(scope="function", autouse=True) -def setup_assets(db, setup_roles_users, setup_markets): - """Make some asset types and add assets to known test users.""" - +@pytest.fixture(scope="module") +def setup_sources(db) -> Dict[str, DataSource]: data_source = DataSource(name="Seita", type="demo script") db.session.add(data_source) + return {"Seita": data_source} - db.session.add( - AssetType( - name="solar", - is_producer=True, - can_curtail=True, - daily_seasonality=True, - yearly_seasonality=True, - ) + +@pytest.fixture(scope="module") +def setup_asset_types(db) -> Dict[str, AssetType]: + return create_test_asset_types(db) + + +@pytest.fixture(scope="function") +def setup_asset_types_fresh_db(fresh_db) -> Dict[str, AssetType]: + return create_test_asset_types(fresh_db) + + +def create_test_asset_types(db) -> Dict[str, AssetType]: + """Make some asset types used throughout.""" + + solar = AssetType( + name="solar", + is_producer=True, + can_curtail=True, + daily_seasonality=True, + yearly_seasonality=True, ) - db.session.add( - AssetType( - name="wind", - is_producer=True, - can_curtail=True, - daily_seasonality=True, - yearly_seasonality=True, - ) + db.session.add(solar) + wind = AssetType( + name="wind", + is_producer=True, + can_curtail=True, + daily_seasonality=True, + yearly_seasonality=True, ) + db.session.add(wind) + return dict(solar=solar, wind=wind) - test_prosumer = find_user_by_email("test_prosumer@seita.nl") - test_market = Market.query.filter_by(name="epex_da").one_or_none() +@pytest.fixture(scope="module") +def setup_assets( + db, setup_roles_users, setup_markets, setup_sources, setup_asset_types +) -> Dict[str, Asset]: + """Add assets to known test users.""" + + assets = [] for asset_name in ["wind-asset-1", "wind-asset-2", "solar-asset-1"]: asset = Asset( name=asset_name, @@ -155,10 +226,11 @@ def setup_assets(db, setup_roles_users, setup_markets): max_soc_in_mwh=0, soc_in_mwh=0, unit="MW", - market_id=test_market.id, + market_id=setup_markets["epex_da"].id, ) - asset.owner = test_prosumer + asset.owner = setup_roles_users["Test Prosumer"] db.session.add(asset) + assets.append(asset) # one day of test data (one complete sine curve) time_slots = pd.date_range( @@ -170,25 +242,23 @@ def setup_assets(db, setup_roles_users, setup_markets): datetime=as_server_time(dt), horizon=parse_duration("PT0M"), value=val, - data_source_id=data_source.id, + data_source_id=setup_sources["Seita"].id, ) p.asset = asset db.session.add(p) + return {asset.name: asset for asset in assets} -@pytest.fixture(scope="function") -def setup_beliefs(db: SQLAlchemy, setup_markets) -> int: +@pytest.fixture(scope="module") +def setup_beliefs(db: SQLAlchemy, setup_markets, setup_sources) -> int: """ :returns: the number of beliefs set up """ sensor = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() - data_source = DataSource.query.filter_by( - name="Seita", type="demo script" - ).one_or_none() beliefs = [ TimedBelief( sensor=sensor, - source=data_source, + source=setup_sources["Seita"], event_value=21, event_start="2021-03-28 16:00+01", belief_horizon=timedelta(0), @@ -198,13 +268,9 @@ def setup_beliefs(db: SQLAlchemy, setup_markets) -> int: return len(beliefs) -@pytest.fixture(scope="function", autouse=True) -def add_market_prices(db: SQLAlchemy, setup_assets, setup_markets): +@pytest.fixture(scope="module") +def add_market_prices(db: SQLAlchemy, setup_assets, setup_markets, setup_sources): """Add two days of market prices for the EPEX day-ahead market.""" - epex_da = Market.query.filter(Market.name == "epex_da").one_or_none() - data_source = DataSource.query.filter_by( - name="Seita", type="demo script" - ).one_or_none() # one day of test data (one complete sine curve) time_slots = pd.date_range( @@ -216,9 +282,9 @@ def add_market_prices(db: SQLAlchemy, setup_assets, setup_markets): datetime=as_server_time(dt), horizon=timedelta(hours=0), value=val, - data_source_id=data_source.id, + data_source_id=setup_sources["Seita"].id, ) - p.market = epex_da + p.market = setup_markets["epex_da"] db.session.add(p) # another day of test data (8 expensive hours, 8 cheap hours, and again 8 expensive hours) @@ -231,14 +297,31 @@ def add_market_prices(db: SQLAlchemy, setup_assets, setup_markets): datetime=as_server_time(dt), horizon=timedelta(hours=0), value=val, - data_source_id=data_source.id, + data_source_id=setup_sources["Seita"].id, ) - p.market = epex_da + p.market = setup_markets["epex_da"] db.session.add(p) -@pytest.fixture(scope="function", autouse=True) -def add_battery_assets(db: SQLAlchemy, setup_roles_users, setup_markets): +@pytest.fixture(scope="module") +def add_battery_assets( + db: SQLAlchemy, setup_roles_users, setup_markets +) -> Dict[str, Asset]: + return create_test_battery_assets(db, setup_roles_users, setup_markets) + + +@pytest.fixture(scope="function") +def add_battery_assets_fresh_db( + fresh_db, setup_roles_users_fresh_db, setup_markets_fresh_db +) -> Dict[str, Asset]: + return create_test_battery_assets( + fresh_db, setup_roles_users_fresh_db, setup_markets_fresh_db + ) + + +def create_test_battery_assets( + db: SQLAlchemy, setup_roles_users, setup_markets +) -> Dict[str, Asset]: """Add two battery assets, set their capacity values and their initial SOC.""" db.session.add( AssetType( @@ -253,13 +336,7 @@ def add_battery_assets(db: SQLAlchemy, setup_roles_users, setup_markets): ) ) - from flexmeasures.data.models.user import User, Role - - user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) - test_prosumer = user_datastore.find_user(email="test_prosumer@seita.nl") - epex_da = Market.query.filter(Market.name == "epex_da").one_or_none() - - battery = Asset( + test_battery = Asset( name="Test battery", asset_type_name="battery", event_resolution=timedelta(minutes=15), @@ -271,13 +348,13 @@ def add_battery_assets(db: SQLAlchemy, setup_roles_users, setup_markets): soc_udi_event_id=203, latitude=10, longitude=100, - market_id=epex_da.id, + market_id=setup_markets["epex_da"].id, unit="MW", ) - battery.owner = test_prosumer - db.session.add(battery) + test_battery.owner = setup_roles_users["Test Prosumer"] + db.session.add(test_battery) - battery = Asset( + test_battery_no_prices = Asset( name="Test battery with no known prices", asset_type_name="battery", event_resolution=timedelta(minutes=15), @@ -289,15 +366,21 @@ def add_battery_assets(db: SQLAlchemy, setup_roles_users, setup_markets): soc_udi_event_id=203, latitude=10, longitude=100, - market_id=epex_da.id, + market_id=setup_markets["epex_da"].id, unit="MW", ) - battery.owner = test_prosumer - db.session.add(battery) - - -@pytest.fixture(scope="function", autouse=True) -def add_charging_station_assets(db: SQLAlchemy, setup_roles_users, setup_markets): + test_battery_no_prices.owner = setup_roles_users["Test Prosumer"] + db.session.add(test_battery_no_prices) + return { + "Test battery": test_battery, + "Test battery with no known prices": test_battery_no_prices, + } + + +@pytest.fixture(scope="module") +def add_charging_station_assets( + db: SQLAlchemy, setup_roles_users, setup_markets +) -> Dict[str, Asset]: """Add uni- and bi-directional charging station assets, set their capacity value and their initial SOC.""" db.session.add( AssetType( @@ -324,12 +407,6 @@ def add_charging_station_assets(db: SQLAlchemy, setup_roles_users, setup_markets ) ) - from flexmeasures.data.models.user import User, Role - - user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) - test_prosumer = user_datastore.find_user(email="test_prosumer@seita.nl") - epex_da = Market.query.filter(Market.name == "epex_da").one_or_none() - charging_station = Asset( name="Test charging station", asset_type_name="one-way_evse", @@ -342,10 +419,10 @@ def add_charging_station_assets(db: SQLAlchemy, setup_roles_users, setup_markets soc_udi_event_id=203, latitude=10, longitude=100, - market_id=epex_da.id, + market_id=setup_markets["epex_da"].id, unit="MW", ) - charging_station.owner = test_prosumer + charging_station.owner = setup_roles_users["Test Prosumer"] db.session.add(charging_station) bidirectional_charging_station = Asset( @@ -360,14 +437,18 @@ def add_charging_station_assets(db: SQLAlchemy, setup_roles_users, setup_markets soc_udi_event_id=203, latitude=10, longitude=100, - market_id=epex_da.id, + market_id=setup_markets["epex_da"].id, unit="MW", ) - bidirectional_charging_station.owner = test_prosumer + bidirectional_charging_station.owner = setup_roles_users["Test Prosumer"] db.session.add(bidirectional_charging_station) + return { + "Test charging station": charging_station, + "Test charging station (bidirectional)": bidirectional_charging_station, + } -@pytest.fixture(scope="function", autouse=True) +@pytest.fixture(scope="function") def clean_redis(app): failed = app.queues["forecasting"].failed_job_registry app.queues["forecasting"].empty() diff --git a/flexmeasures/data/models/planning/tests/conftest.py b/flexmeasures/data/models/planning/tests/conftest.py index e69de29bb..359f4d79b 100644 --- a/flexmeasures/data/models/planning/tests/conftest.py +++ b/flexmeasures/data/models/planning/tests/conftest.py @@ -0,0 +1,9 @@ +import pytest + + +@pytest.fixture(scope="function", autouse=True) +def setup_planning_test_data(db, add_market_prices, add_charging_station_assets): + """ + Set up data for all planning tests. + """ + print("Setting up data for planning tests on %s" % db.engine) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 8184670f2..fd832b586 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -12,7 +12,7 @@ from flexmeasures.utils.time_utils import as_server_time -def test_battery_solver_day_1(): +def test_battery_solver_day_1(add_battery_assets): epex_da = Market.query.filter(Market.name == "epex_da").one_or_none() battery = Asset.query.filter(Asset.name == "Test battery").one_or_none() start = as_server_time(datetime(2015, 1, 1)) @@ -33,7 +33,7 @@ def test_battery_solver_day_1(): assert soc <= battery.max_soc_in_mwh -def test_battery_solver_day_2(): +def test_battery_solver_day_2(add_battery_assets): epex_da = Market.query.filter(Market.name == "epex_da").one_or_none() battery = Asset.query.filter(Asset.name == "Test battery").one_or_none() start = as_server_time(datetime(2015, 1, 2)) diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index 2a9fb4c48..900eef8ae 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -2,14 +2,15 @@ from datetime import datetime, timedelta from random import random +from isodate import parse_duration import pandas as pd import numpy as np from flask_sqlalchemy import SQLAlchemy from statsmodels.api import OLS +from flexmeasures.data.models.assets import Asset, Power from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.weather import WeatherSensorType, WeatherSensor, Weather -from flexmeasures.data.models.assets import AssetType from flexmeasures.data.models.forecasting import model_map from flexmeasures.data.models.forecasting.model_spec_factory import ( create_initial_model_specs, @@ -17,8 +18,14 @@ from flexmeasures.utils.time_utils import as_server_time -@pytest.fixture(scope="function", autouse=True) -def setup_test_data(db, app, remove_seasonality_for_power_forecasts): +@pytest.fixture(scope="module") +def setup_test_data( + db, + app, + add_market_prices, + setup_assets, + remove_seasonality_for_power_forecasts, +): """ Adding a few forecasting jobs (based on data made in flexmeasures.conftest). """ @@ -29,14 +36,72 @@ def setup_test_data(db, app, remove_seasonality_for_power_forecasts): print("Done setting up data for data tests") -@pytest.fixture(scope="function", autouse=True) -def remove_seasonality_for_power_forecasts(db): +@pytest.fixture(scope="function") +def setup_fresh_test_data( + fresh_db, + setup_markets_fresh_db, + setup_roles_users_fresh_db, + app, + fresh_remove_seasonality_for_power_forecasts, +): + db = fresh_db + setup_roles_users = setup_roles_users_fresh_db + setup_markets = setup_markets_fresh_db + + data_source = DataSource(name="Seita", type="demo script") + db.session.add(data_source) + db.session.flush() + + for asset_name in ["wind-asset-2", "solar-asset-1"]: + asset = Asset( + name=asset_name, + asset_type_name="wind" if "wind" in asset_name else "solar", + event_resolution=timedelta(minutes=15), + capacity_in_mw=1, + latitude=10, + longitude=100, + min_soc_in_mwh=0, + max_soc_in_mwh=0, + soc_in_mwh=0, + unit="MW", + market_id=setup_markets["epex_da"].id, + ) + asset.owner = setup_roles_users["Test Prosumer"] + db.session.add(asset) + + time_slots = pd.date_range( + datetime(2015, 1, 1), datetime(2015, 1, 1, 23, 45), freq="15T" + ) + values = [random() * (1 + np.sin(x / 15)) for x in range(len(time_slots))] + for dt, val in zip(time_slots, values): + p = Power( + datetime=as_server_time(dt), + horizon=parse_duration("PT0M"), + value=val, + data_source_id=data_source.id, + ) + p.asset = asset + db.session.add(p) + add_test_weather_sensor_and_forecasts(fresh_db) + + +@pytest.fixture(scope="module", autouse=True) +def remove_seasonality_for_power_forecasts(db, setup_asset_types): + """Make sure the AssetType specs make us query only data we actually have in the test db""" + for asset_type in setup_asset_types.keys(): + setup_asset_types[asset_type].daily_seasonality = False + setup_asset_types[asset_type].weekly_seasonality = False + setup_asset_types[asset_type].yearly_seasonality = False + + +@pytest.fixture(scope="function") +def fresh_remove_seasonality_for_power_forecasts(db, setup_asset_types_fresh_db): """Make sure the AssetType specs make us query only data we actually have in the test db""" - asset_types = AssetType.query.all() - for a in asset_types: - a.daily_seasonality = False - a.weekly_seasonality = False - a.yearly_seasonality = False + setup_asset_types = setup_asset_types_fresh_db + for asset_type in setup_asset_types.keys(): + setup_asset_types[asset_type].daily_seasonality = False + setup_asset_types[asset_type].weekly_seasonality = False + setup_asset_types[asset_type].yearly_seasonality = False def add_test_weather_sensor_and_forecasts(db: SQLAlchemy): @@ -72,7 +137,7 @@ def add_test_weather_sensor_and_forecasts(db: SQLAlchemy): ) -@pytest.fixture(scope="function", autouse=True) +@pytest.fixture(scope="module", autouse=True) def add_failing_test_model(db): """Add a test model specs to the lookup which should fail due to missing data. It falls back to linear OLS (which falls back to naive).""" diff --git a/flexmeasures/data/tests/test_forecasting_jobs.py b/flexmeasures/data/tests/test_forecasting_jobs.py index e92307c04..9c6f9c6a4 100644 --- a/flexmeasures/data/tests/test_forecasting_jobs.py +++ b/flexmeasures/data/tests/test_forecasting_jobs.py @@ -3,9 +3,7 @@ from datetime import datetime, timedelta import os -import pytest import numpy as np -from sqlalchemy.orm import Query from rq.job import Job from flexmeasures.data.models.data_sources import DataSource @@ -36,15 +34,19 @@ def get_data_source(model_identifier: str = "linear-OLS model v2"): ).one_or_none() -def check_aggregate(overall_expected: int, horizon: timedelta): +def check_aggregate(overall_expected: int, horizon: timedelta, asset_id: int): """Check that the expected number of forecasts were made for the given horizon, and check that each forecast is a number.""" - all_forecasts = Power.query.filter(Power.horizon == horizon).all() + all_forecasts = ( + Power.query.filter(Power.asset_id == asset_id) + .filter(Power.horizon == horizon) + .all() + ) assert len(all_forecasts) == overall_expected assert all([not np.isnan(f.value) for f in all_forecasts]) -def test_forecasting_an_hour_of_wind(db, app): +def test_forecasting_an_hour_of_wind(db, app, setup_test_data): """Test one clean run of one job: - data source was made, - forecasts have been made @@ -80,69 +82,10 @@ def test_forecasting_an_hour_of_wind(db, app): .all() ) assert len(forecasts) == 4 - check_aggregate(4, horizon) - - -def test_forecasting_three_hours_of_wind(db, app): - wind_device2: Asset = Asset.query.filter_by(name="wind-asset-2").one_or_none() - - # makes 12 forecasts - horizon = timedelta(hours=1) - job = create_forecasting_jobs( - timed_value_type="Power", - start_of_roll=as_server_time(datetime(2015, 1, 1, 10)), - end_of_roll=as_server_time(datetime(2015, 1, 1, 13)), - horizons=[horizon], - asset_id=wind_device2.id, - custom_model_params=custom_model_params(), - ) - print("Job: %s" % job[0].id) - - work_on_rq(app.queues["forecasting"], exc_handler=handle_forecasting_exception) - - forecasts = ( - Power.query.filter(Power.asset_id == wind_device2.id) - .filter(Power.horizon == horizon) - .filter( - (Power.datetime >= as_server_time(datetime(2015, 1, 1, 11))) - & (Power.datetime < as_server_time(datetime(2015, 1, 1, 14))) - ) - .all() - ) - assert len(forecasts) == 12 - check_aggregate(12, horizon) - - -def test_forecasting_two_hours_of_solar(db, app): - solar_device1: Asset = Asset.query.filter_by(name="solar-asset-1").one_or_none() - - # makes 8 forecasts - horizon = timedelta(hours=1) - job = create_forecasting_jobs( - timed_value_type="Power", - start_of_roll=as_server_time(datetime(2015, 1, 1, 12)), - end_of_roll=as_server_time(datetime(2015, 1, 1, 14)), - horizons=[horizon], - asset_id=solar_device1.id, - custom_model_params=custom_model_params(), - ) - print("Job: %s" % job[0].id) - - work_on_rq(app.queues["forecasting"], exc_handler=handle_forecasting_exception) - forecasts = ( - Power.query.filter(Power.asset_id == solar_device1.id) - .filter(Power.horizon == horizon) - .filter( - (Power.datetime >= as_server_time(datetime(2015, 1, 1, 13))) - & (Power.datetime < as_server_time(datetime(2015, 1, 1, 15))) - ) - .all() - ) - assert len(forecasts) == 8 - check_aggregate(8, horizon) + check_aggregate(4, horizon, wind_device_1.id) -def test_forecasting_two_hours_of_solar_at_edge_of_data_set(db, app): +def test_forecasting_two_hours_of_solar_at_edge_of_data_set(db, app, setup_test_data): solar_device1: Asset = Asset.query.filter_by(name="solar-asset-1").one_or_none() last_power_datetime = ( @@ -182,7 +125,7 @@ def test_forecasting_two_hours_of_solar_at_edge_of_data_set(db, app): .all() ) assert len(forecasts) == 1 - check_aggregate(4, horizon) + check_aggregate(4, horizon, solar_device1.id) def check_failures( @@ -227,7 +170,7 @@ def check_failures( assert job.meta["model_identifier"] == model_identifiers[job_idx] -def test_failed_forecasting_insufficient_data(app): +def test_failed_forecasting_insufficient_data(app, clean_redis, setup_test_data): """This one (as well as the fallback) should fail as there is no underlying data. (Power data is in 2015)""" solar_device1: Asset = Asset.query.filter_by(name="solar-asset-1").one_or_none() @@ -243,7 +186,7 @@ def test_failed_forecasting_insufficient_data(app): check_failures(app.queues["forecasting"], 2 * ["NotEnoughDataException"]) -def test_failed_forecasting_invalid_horizon(app): +def test_failed_forecasting_invalid_horizon(app, clean_redis, setup_test_data): """ This one (as well as the fallback) should fail as the horizon is invalid.""" solar_device1: Asset = Asset.query.filter_by(name="solar-asset-1").one_or_none() create_forecasting_jobs( @@ -258,7 +201,7 @@ def test_failed_forecasting_invalid_horizon(app): check_failures(app.queues["forecasting"], 2 * ["InvalidHorizonException"]) -def test_failed_unknown_model(app): +def test_failed_unknown_model(app, clean_redis, setup_test_data): """ This one should fail because we use a model search term which yields no model configurator.""" solar_device1: Asset = Asset.query.filter_by(name="solar-asset-1").one_or_none() horizon = timedelta(hours=1) @@ -278,98 +221,3 @@ def test_failed_unknown_model(app): work_on_rq(app.queues["forecasting"], exc_handler=handle_forecasting_exception) check_failures(app.queues["forecasting"], ["No model found for search term"]) - - -@pytest.mark.parametrize( - "model_to_start_with, model_version", [("failing-test", 1), ("linear-OLS", 2)] -) -def test_failed_model_with_too_much_training_then_succeed_with_fallback( - app, model_to_start_with, model_version -): - """ - Here we fail once - because we start with a model that needs too much training. - So we check for this failure happening as expected. - But then, we do succeed with the fallback model one level down. - (fail-test falls back to linear & linear falls back to naive). - As a result, there should be forecasts in the DB. - """ - solar_device1: Asset = Asset.query.filter_by(name="solar-asset-1").one_or_none() - horizon_hours = 1 - horizon = timedelta(hours=horizon_hours) - - cmp = custom_model_params() - hour_start = 5 - if model_to_start_with == "linear-OLS": - # making the linear model fail and fall back to naive - hour_start = 3 # Todo: explain this parameter; why would it fail to forecast if data is there for the full day? - - # The failed test model (this failure enqueues a new job) - create_forecasting_jobs( - timed_value_type="Power", - start_of_roll=as_server_time(datetime(2015, 1, 1, hour_start)), - end_of_roll=as_server_time(datetime(2015, 1, 1, hour_start + 2)), - horizons=[horizon], - asset_id=solar_device1.id, - model_search_term=model_to_start_with, - custom_model_params=cmp, - ) - work_on_rq(app.queues["forecasting"], exc_handler=handle_forecasting_exception) - - # Check if the correct model failed in the expected way - check_failures( - app.queues["forecasting"], - ["NotEnoughDataException"], - ["%s model v%d" % (model_to_start_with, model_version)], - ) - - # this query is useful to check data: - def make_query(the_horizon_hours: int) -> Query: - the_horizon = timedelta(hours=the_horizon_hours) - return ( - Power.query.filter(Power.asset_id == solar_device1.id) - .filter(Power.horizon == the_horizon) - .filter( - ( - Power.datetime - >= as_server_time( - datetime(2015, 1, 1, hour_start + the_horizon_hours) - ) - ) - & ( - Power.datetime - < as_server_time( - datetime(2015, 1, 1, hour_start + the_horizon_hours + 2) - ) - ) - ) - ) - - # The successful (linear or naive) OLS leads to these. - forecasts = make_query(the_horizon_hours=horizon_hours).all() - - assert len(forecasts) == 8 - check_aggregate(8, horizon) - - if model_to_start_with == "linear-OLS": - existing_data = make_query(the_horizon_hours=0).all() - - for ed, fd in zip(existing_data, forecasts): - assert ed.value == fd.value - - # Now to check which models actually got to work. - # We check which data sources do and do not exist by now: - assert ( - get_data_source("failing-test model v1") is None - ) # the test failure model failed -> no data source - if model_to_start_with == "linear-OLS": - assert ( - get_data_source() is None - ) # the default (linear regression) (was made to) fail, as well - assert ( - get_data_source("naive model v1") is not None - ) # the naive one had to be used - else: - assert get_data_source() is not None # the default (linear regression) - assert ( - get_data_source("naive model v1") is None - ) # the naive one did not have to be used diff --git a/flexmeasures/data/tests/test_forecasting_jobs_fresh_db.py b/flexmeasures/data/tests/test_forecasting_jobs_fresh_db.py new file mode 100644 index 000000000..6ba275264 --- /dev/null +++ b/flexmeasures/data/tests/test_forecasting_jobs_fresh_db.py @@ -0,0 +1,175 @@ +from datetime import timedelta, datetime + +import pytest +from sqlalchemy.orm import Query + +from flexmeasures.data.models.assets import Asset, Power +from flexmeasures.data.services.forecasting import ( + create_forecasting_jobs, + handle_forecasting_exception, +) +from flexmeasures.data.tests.test_forecasting_jobs import ( + custom_model_params, + check_aggregate, + check_failures, + get_data_source, +) +from flexmeasures.data.tests.utils import work_on_rq +from flexmeasures.utils.time_utils import as_server_time + + +def test_forecasting_three_hours_of_wind(app, setup_fresh_test_data, clean_redis): + wind_device2: Asset = Asset.query.filter_by(name="wind-asset-2").one_or_none() + + # makes 12 forecasts + horizon = timedelta(hours=1) + job = create_forecasting_jobs( + timed_value_type="Power", + start_of_roll=as_server_time(datetime(2015, 1, 1, 10)), + end_of_roll=as_server_time(datetime(2015, 1, 1, 13)), + horizons=[horizon], + asset_id=wind_device2.id, + custom_model_params=custom_model_params(), + ) + print("Job: %s" % job[0].id) + + work_on_rq(app.queues["forecasting"], exc_handler=handle_forecasting_exception) + + forecasts = ( + Power.query.filter(Power.asset_id == wind_device2.id) + .filter(Power.horizon == horizon) + .filter( + (Power.datetime >= as_server_time(datetime(2015, 1, 1, 11))) + & (Power.datetime < as_server_time(datetime(2015, 1, 1, 14))) + ) + .all() + ) + assert len(forecasts) == 12 + check_aggregate(12, horizon, wind_device2.id) + + +def test_forecasting_two_hours_of_solar(app, setup_fresh_test_data, clean_redis): + solar_device1: Asset = Asset.query.filter_by(name="solar-asset-1").one_or_none() + wind_device2: Asset = Asset.query.filter_by(name="wind-asset-2").one_or_none() + print(solar_device1) + print(wind_device2) + + # makes 8 forecasts + horizon = timedelta(hours=1) + job = create_forecasting_jobs( + timed_value_type="Power", + start_of_roll=as_server_time(datetime(2015, 1, 1, 12)), + end_of_roll=as_server_time(datetime(2015, 1, 1, 14)), + horizons=[horizon], + asset_id=solar_device1.id, + custom_model_params=custom_model_params(), + ) + print("Job: %s" % job[0].id) + + work_on_rq(app.queues["forecasting"], exc_handler=handle_forecasting_exception) + forecasts = ( + Power.query.filter(Power.asset_id == solar_device1.id) + .filter(Power.horizon == horizon) + .filter( + (Power.datetime >= as_server_time(datetime(2015, 1, 1, 13))) + & (Power.datetime < as_server_time(datetime(2015, 1, 1, 15))) + ) + .all() + ) + assert len(forecasts) == 8 + check_aggregate(8, horizon, solar_device1.id) + + +@pytest.mark.parametrize( + "model_to_start_with, model_version", [("failing-test", 1), ("linear-OLS", 2)] +) +def test_failed_model_with_too_much_training_then_succeed_with_fallback( + setup_fresh_test_data, app, clean_redis, model_to_start_with, model_version +): + """ + Here we fail once - because we start with a model that needs too much training. + So we check for this failure happening as expected. + But then, we do succeed with the fallback model one level down. + (fail-test falls back to linear & linear falls back to naive). + As a result, there should be forecasts in the DB. + """ + solar_device1: Asset = Asset.query.filter_by(name="solar-asset-1").one_or_none() + horizon_hours = 1 + horizon = timedelta(hours=horizon_hours) + + cmp = custom_model_params() + hour_start = 5 + if model_to_start_with == "linear-OLS": + # making the linear model fail and fall back to naive + hour_start = 3 # Todo: explain this parameter; why would it fail to forecast if data is there for the full day? + + # The failed test model (this failure enqueues a new job) + create_forecasting_jobs( + timed_value_type="Power", + start_of_roll=as_server_time(datetime(2015, 1, 1, hour_start)), + end_of_roll=as_server_time(datetime(2015, 1, 1, hour_start + 2)), + horizons=[horizon], + asset_id=solar_device1.id, + model_search_term=model_to_start_with, + custom_model_params=cmp, + ) + work_on_rq(app.queues["forecasting"], exc_handler=handle_forecasting_exception) + + # Check if the correct model failed in the expected way + check_failures( + app.queues["forecasting"], + ["NotEnoughDataException"], + ["%s model v%d" % (model_to_start_with, model_version)], + ) + + # this query is useful to check data: + def make_query(the_horizon_hours: int) -> Query: + the_horizon = timedelta(hours=the_horizon_hours) + return ( + Power.query.filter(Power.asset_id == solar_device1.id) + .filter(Power.horizon == the_horizon) + .filter( + ( + Power.datetime + >= as_server_time( + datetime(2015, 1, 1, hour_start + the_horizon_hours) + ) + ) + & ( + Power.datetime + < as_server_time( + datetime(2015, 1, 1, hour_start + the_horizon_hours + 2) + ) + ) + ) + ) + + # The successful (linear or naive) OLS leads to these. + forecasts = make_query(the_horizon_hours=horizon_hours).all() + + assert len(forecasts) == 8 + check_aggregate(8, horizon, solar_device1.id) + + if model_to_start_with == "linear-OLS": + existing_data = make_query(the_horizon_hours=0).all() + + for ed, fd in zip(existing_data, forecasts): + assert ed.value == fd.value + + # Now to check which models actually got to work. + # We check which data sources do and do not exist by now: + assert ( + get_data_source("failing-test model v1") is None + ) # the test failure model failed -> no data source + if model_to_start_with == "linear-OLS": + assert ( + get_data_source() is None + ) # the default (linear regression) (was made to) fail, as well + assert ( + get_data_source("naive model v1") is not None + ) # the naive one had to be used + else: + assert get_data_source() is not None # the default (linear regression) + assert ( + get_data_source("naive model v1") is None + ) # the naive one did not have to be used diff --git a/flexmeasures/data/tests/test_queries.py b/flexmeasures/data/tests/test_queries.py index b510809ad..145dd8820 100644 --- a/flexmeasures/data/tests/test_queries.py +++ b/flexmeasures/data/tests/test_queries.py @@ -39,7 +39,7 @@ # ), # test empty BeliefsDataFrame # todo: uncomment when this if fixed: https://github.com/pandas-dev/pandas/issues/30517 ], ) -def test_collect_power(db, app, query_start, query_end, num_values): +def test_collect_power(db, app, query_start, query_end, num_values, setup_test_data): wind_device_1 = Asset.query.filter_by(name="wind-asset-1").one_or_none() data = Power.query.filter(Power.asset_id == wind_device_1.id).all() print(data) @@ -88,7 +88,7 @@ def test_collect_power(db, app, query_start, query_end, num_values): ], ) def test_collect_power_resampled( - db, app, query_start, query_end, resolution, num_values + db, app, query_start, query_end, resolution, num_values, setup_test_data ): wind_device_1 = Asset.query.filter_by(name="wind-asset-1").one_or_none() bdf: tb.BeliefsDataFrame = Power.collect( @@ -204,7 +204,7 @@ def test_multiplication_with_both_empty_dataframe(): pd.testing.assert_frame_equal(df, df_compare) -def test_simplify_index(): +def test_simplify_index(setup_test_data): """Check whether simplify_index retains the event resolution.""" wind_device_1 = Asset.query.filter_by(name="wind-asset-1").one_or_none() bdf: tb.BeliefsDataFrame = Power.collect( @@ -238,7 +238,7 @@ def test_query_beliefs(setup_beliefs): assert len(bdf) == setup_beliefs -def test_persist_beliefs(setup_beliefs): +def test_persist_beliefs(setup_beliefs, setup_test_data): """Check whether persisting beliefs works. We load the already set up beliefs, and form new beliefs an hour later. diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py index 324188564..ec7f0a4cf 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_jobs.py @@ -1,9 +1,6 @@ # flake8: noqa: E402 from datetime import datetime, timedelta -import numpy as np -import pandas as pd - from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.assets import Asset, Power from flexmeasures.data.tests.utils import work_on_rq, exception_reporter @@ -11,7 +8,7 @@ from flexmeasures.utils.time_utils import as_server_time -def test_scheduling_a_battery(db, app): +def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data): """Test one clean run of one scheduling job: - data source was made, - schedule has been made @@ -49,68 +46,3 @@ def test_scheduling_a_battery(db, app): ) print([v.value for v in power_values]) assert len(power_values) == 96 - - -def test_scheduling_a_charging_station(db, app): - """Test one clean run of one scheduling job: - - data source was made, - - schedule has been made - - Starting with a state of charge 1 kWh, within 2 hours we should be able to reach 5 kWh. - """ - soc_at_start = 1 - target_soc = 5 - duration_until_target = timedelta(hours=2) - - charging_station = Asset.query.filter( - Asset.name == "Test charging station" - ).one_or_none() - start = as_server_time(datetime(2015, 1, 2)) - end = as_server_time(datetime(2015, 1, 3)) - resolution = timedelta(minutes=15) - target_soc_datetime = start + duration_until_target - soc_targets = pd.Series( - np.nan, index=pd.date_range(start, end, freq=resolution, closed="right") - ) - soc_targets.loc[target_soc_datetime] = target_soc - - assert ( - DataSource.query.filter_by(name="Seita", type="scheduling script").one_or_none() - is None - ) # Make sure the scheduler data source isn't there - - job = create_scheduling_job( - charging_station.id, - start, - end, - belief_time=start, - resolution=resolution, - soc_at_start=soc_at_start, - soc_targets=soc_targets, - ) - - print("Job: %s" % job.id) - - work_on_rq(app.queues["scheduling"], exc_handler=exception_reporter) - - scheduler_source = DataSource.query.filter_by( - name="Seita", type="scheduling script" - ).one_or_none() - assert ( - scheduler_source is not None - ) # Make sure the scheduler data source is now there - - power_values = ( - Power.query.filter(Power.asset_id == charging_station.id) - .filter(Power.data_source_id == scheduler_source.id) - .all() - ) - consumption_schedule = pd.Series( - [-v.value for v in power_values], - index=pd.DatetimeIndex([v.datetime for v in power_values]), - ) # For consumption schedules, positive values denote consumption. For the db, consumption is negative - assert len(consumption_schedule) == 96 - print(consumption_schedule.head(12)) - assert ( - consumption_schedule.head(8).sum() * (resolution / timedelta(hours=1)) == 4.0 - ) # The first 2 hours should consume 4 kWh to charge from 1 to 5 kWh diff --git a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py new file mode 100644 index 000000000..722b69adf --- /dev/null +++ b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py @@ -0,0 +1,77 @@ +from datetime import timedelta, datetime + +import numpy as np +import pandas as pd + +from flexmeasures.data.models.assets import Asset, Power +from flexmeasures.data.models.data_sources import DataSource +from flexmeasures.data.services.scheduling import create_scheduling_job +from flexmeasures.data.tests.utils import work_on_rq, exception_reporter +from flexmeasures.utils.time_utils import as_server_time + + +def test_scheduling_a_charging_station( + db, app, add_charging_station_assets, setup_test_data +): + """Test one clean run of one scheduling job: + - data source was made, + - schedule has been made + + Starting with a state of charge 1 kWh, within 2 hours we should be able to reach 5 kWh. + """ + soc_at_start = 1 + target_soc = 5 + duration_until_target = timedelta(hours=2) + + charging_station = Asset.query.filter( + Asset.name == "Test charging station" + ).one_or_none() + start = as_server_time(datetime(2015, 1, 2)) + end = as_server_time(datetime(2015, 1, 3)) + resolution = timedelta(minutes=15) + target_soc_datetime = start + duration_until_target + soc_targets = pd.Series( + np.nan, index=pd.date_range(start, end, freq=resolution, closed="right") + ) + soc_targets.loc[target_soc_datetime] = target_soc + + assert ( + DataSource.query.filter_by(name="Seita", type="scheduling script").one_or_none() + is None + ) # Make sure the scheduler data source isn't there + + job = create_scheduling_job( + charging_station.id, + start, + end, + belief_time=start, + resolution=resolution, + soc_at_start=soc_at_start, + soc_targets=soc_targets, + ) + + print("Job: %s" % job.id) + + work_on_rq(app.queues["scheduling"], exc_handler=exception_reporter) + + scheduler_source = DataSource.query.filter_by( + name="Seita", type="scheduling script" + ).one_or_none() + assert ( + scheduler_source is not None + ) # Make sure the scheduler data source is now there + + power_values = ( + Power.query.filter(Power.asset_id == charging_station.id) + .filter(Power.data_source_id == scheduler_source.id) + .all() + ) + consumption_schedule = pd.Series( + [-v.value for v in power_values], + index=pd.DatetimeIndex([v.datetime for v in power_values]), + ) # For consumption schedules, positive values denote consumption. For the db, consumption is negative + assert len(consumption_schedule) == 96 + print(consumption_schedule.head(12)) + assert ( + consumption_schedule.head(8).sum() * (resolution / timedelta(hours=1)) == 4.0 + ) # The first 2 hours should consume 4 kWh to charge from 1 to 5 kWh diff --git a/flexmeasures/data/tests/test_user_services.py b/flexmeasures/data/tests/test_user_services.py index ed73769b1..6008b7894 100644 --- a/flexmeasures/data/tests/test_user_services.py +++ b/flexmeasures/data/tests/test_user_services.py @@ -13,7 +13,7 @@ from flexmeasures.data.models.data_sources import DataSource -def test_create_user(app): +def test_create_user(fresh_db, setup_roles_users_fresh_db, app): """Create a user""" num_users = User.query.count() user = create_user( @@ -29,7 +29,7 @@ def test_create_user(app): assert DataSource.query.filter_by(name=user.username).one_or_none() -def test_create_invalid_user(app): +def test_create_invalid_user(fresh_db, setup_roles_users_fresh_db, app): """A few invalid attempts to create a user""" with pytest.raises(InvalidFlexMeasuresUser) as exc_info: create_user(password=hash_password("testtest"), user_roles=["Prosumer"]) @@ -67,7 +67,7 @@ def test_create_invalid_user(app): assert "already exists" in str(exc_info.value) -def test_delete_user(app): +def test_delete_user(fresh_db, setup_roles_users_fresh_db, app): """Assert user has assets and power measurements. Deleting removes all of that.""" prosumer: User = find_user_by_email("test_prosumer@seita.nl") num_users_before = User.query.count() diff --git a/flexmeasures/ui/tests/conftest.py b/flexmeasures/ui/tests/conftest.py index c09372725..f29d34b5a 100644 --- a/flexmeasures/ui/tests/conftest.py +++ b/flexmeasures/ui/tests/conftest.py @@ -28,8 +28,10 @@ def as_admin(client): logout(client) -@pytest.fixture(scope="function", autouse=True) -def setup_ui_test_data(db): +@pytest.fixture(scope="module", autouse=True) +def setup_ui_test_data( + db, setup_roles_users, setup_markets, setup_sources, setup_asset_types +): """ Create another prosumer, without data, and an admin Also, a weather sensor (and sensor type). diff --git a/flexmeasures/ui/tests/test_asset_crud.py b/flexmeasures/ui/tests/test_asset_crud.py index 4ae20f7c1..8a7b134f2 100644 --- a/flexmeasures/ui/tests/test_asset_crud.py +++ b/flexmeasures/ui/tests/test_asset_crud.py @@ -34,13 +34,13 @@ def test_assets_page_nonempty(db, client, requests_mock, as_prosumer, use_owned_ assert asset["display_name"].encode() in asset_index.data -def test_new_asset_page(client, as_admin): +def test_new_asset_page(client, setup_assets, as_admin): asset_page = client.get(url_for("AssetCrudUI:get", id="new"), follow_redirects=True) assert asset_page.status_code == 200 assert b"Creating a new asset" in asset_page.data -def test_asset_page(db, client, requests_mock, as_prosumer): +def test_asset_page(db, client, setup_assets, requests_mock, as_prosumer): prosumer = find_user_by_email("test_prosumer@seita.nl") asset = prosumer.assets[0] db.session.expunge(prosumer) @@ -61,7 +61,7 @@ def test_asset_page(db, client, requests_mock, as_prosumer): assert str(mock_asset["longitude"]).encode() in asset_page.data -def test_edit_asset(db, client, requests_mock, as_admin): +def test_edit_asset(db, client, setup_assets, requests_mock, as_admin): mock_asset = mock_asset_response(as_list=False) requests_mock.patch( "http://localhost//api/v2_0/asset/1", status_code=200, json=mock_asset @@ -78,7 +78,7 @@ def test_edit_asset(db, client, requests_mock, as_admin): assert str(mock_asset["longitude"]) in str(response.data) -def test_add_asset(db, client, requests_mock, as_admin): +def test_add_asset(db, client, setup_assets, requests_mock, as_admin): """Add a new asset""" prosumer = find_user_by_email("test_prosumer@seita.nl") mock_asset = mock_asset_response(owner_id=prosumer.id, as_list=False) diff --git a/flexmeasures/ui/tests/test_views.py b/flexmeasures/ui/tests/test_views.py index 6bb723766..27edd4049 100644 --- a/flexmeasures/ui/tests/test_views.py +++ b/flexmeasures/ui/tests/test_views.py @@ -4,7 +4,7 @@ from flexmeasures.ui.tests.utils import logout -def test_dashboard_responds(client, as_prosumer): +def test_dashboard_responds(client, setup_assets, as_prosumer): dashboard = client.get( url_for("flexmeasures_ui.dashboard_view"), follow_redirects=True ) @@ -21,7 +21,7 @@ def test_dashboard_responds_only_for_logged_in_users(client, as_prosumer): assert b"Please log in" in dashboard.data -def test_portfolio_responds(client, as_prosumer): +def test_portfolio_responds(client, setup_assets, as_prosumer): portfolio = client.get( url_for("flexmeasures_ui.portfolio_view"), follow_redirects=True ) @@ -42,7 +42,7 @@ def test_control_responds(client, as_prosumer): assert b"Control actions" in control.data -def test_analytics_responds(db, client, as_prosumer): +def test_analytics_responds(db, client, setup_assets, as_prosumer): analytics = client.get( url_for("flexmeasures_ui.analytics_view"), follow_redirects=True )