Make test suite run faster (#115)

Re-use the database between automated tests, if possible. This shaves 2/3rd off of the time it takes for the FlexMeasures test suite to run.



* Avoid redundant queries in test suite

* Separate data source setup

* Reduce the use of autouse=True

* Less queries

* Typo

* Fix test

* Separate setup of inactive user

* Clean redis only in API and data package

* Less setup of market prices

* Add type annotations for return values

* Set up charging stations only when tests need them

* Set up assets only when tests need them (stop autouse), and split off setup of asset types

* Create draft PR for #114

* Setup database with module scope

* Refactor test db setup at different scopes

* Refactor other main conftest setups at different scopes

* Rename test modules using fresh db

* Rename test modules using fresh db

* Update developer documentation in main conftest

* Rename fixtures using fresh test db according to a single rule (append _fresh_db)

* Rename fresh_test_db to fresh_db

Co-authored-by: F.N. Claessen <felix@seita.nl>
Co-authored-by: Flix6x <Flix6x@users.noreply.github.com>

documentation/changelog.rst

@@ -18,6 +18,7 @@ Bugfixes
 Infrastructure / Support
 ----------------------
 * Make assets use MW as their default unit and enforce that in CLI, as well (API already did) [see `PR #108 <http://www.github.com/SeitaBV/flexmeasures/pull/108>`_]
+* Re-use the database between automated tests, if possible. This shaves 2/3rd off of the time it takes for the FlexMeasures test suite to run [see `PR #115 <http://www.github.com/SeitaBV/flexmeasures/pull/115>`_]
 
 
 v0.4.1 | May 7, 2021

flexmeasures/api/tests/conftest.py

@@ -7,8 +7,8 @@
 from flask_security.utils import hash_password
 
 
-@pytest.fixture(scope="function", autouse=True)
-def setup_api_test_data(db):
+@pytest.fixture(scope="module", autouse=True)
+def setup_api_test_data(db, setup_roles_users):
     """
     Adding the task-runner
     """

flexmeasures/api/v1/tests/conftest.py

@@ -4,27 +4,23 @@
 import isodate
 import pytest
 
-from flask_security import SQLAlchemySessionUserDatastore
 from flask_security.utils import hash_password
 
 from flexmeasures.data.services.users import create_user
 
 
-@pytest.fixture(scope="function", autouse=True)
-def setup_api_test_data(db):
+@pytest.fixture(scope="module", autouse=True)
+def setup_api_test_data(db, setup_roles_users, add_market_prices):
     """
     Set up data for API v1 tests.
     """
     print("Setting up data for API v1 tests on %s" % db.engine)
 
-    from flexmeasures.data.models.user import User, Role
     from flexmeasures.data.models.assets import Asset, AssetType, Power
     from flexmeasures.data.models.data_sources import DataSource
 
-    user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role)
-
     # Create an anonymous user
-    create_user(
+    test_anonymous_prosumer = create_user(
         username="anonymous user with Prosumer role",
         email="demo@seita.nl",
         password=hash_password("testtest"),
@@ -35,7 +31,6 @@ def setup_api_test_data(db):
     )
 
     # Create 1 test asset for the anonymous user
-    test_prosumer = user_datastore.find_user(email="demo@seita.nl")
     test_asset_type = AssetType(name="test-type")
     db.session.add(test_asset_type)
     asset_names = ["CS 0"]
@@ -50,7 +45,7 @@ def setup_api_test_data(db):
             longitude=100,
             unit="MW",
         )
-        asset.owner = test_prosumer
+        asset.owner = test_anonymous_prosumer
         assets.append(asset)
         db.session.add(asset)
 
@@ -62,7 +57,7 @@ def setup_api_test_data(db):
     )
 
     # Create 5 test assets for the test_prosumer user
-    test_prosumer = user_datastore.find_user(email="test_prosumer@seita.nl")
+    test_prosumer = setup_roles_users["Test Prosumer"]
     asset_names = ["CS 1", "CS 2", "CS 3", "CS 4", "CS 5"]
     assets: List[Asset] = []
     for asset_name in asset_names:
@@ -83,7 +78,7 @@ def setup_api_test_data(db):
 
     # Add power forecasts to one of the assets, for two sources
     cs_5 = Asset.query.filter(Asset.name == "CS 5").one_or_none()
-    test_supplier = user_datastore.find_user(email="test_supplier@seita.nl")
+    test_supplier = setup_roles_users["Test Supplier"]
     prosumer_data_source = DataSource.query.filter(
         DataSource.user == test_prosumer
     ).one_or_none()
@@ -113,3 +108,32 @@ def setup_api_test_data(db):
     db.session.bulk_save_objects(meter_data)
 
     print("Done setting up data for API v1 tests")
+
+
+@pytest.fixture(scope="function")
+def setup_fresh_api_test_data(fresh_db, setup_roles_users_fresh_db):
+    db = fresh_db
+    setup_roles_users = setup_roles_users_fresh_db
+    from flexmeasures.data.models.assets import Asset, AssetType
+
+    # Create 5 test assets for the test_prosumer user
+    test_prosumer = setup_roles_users["Test Prosumer"]
+    test_asset_type = AssetType(name="test-type")
+    db.session.add(test_asset_type)
+    asset_names = ["CS 1", "CS 2", "CS 3", "CS 4", "CS 5"]
+    assets: List[Asset] = []
+    for asset_name in asset_names:
+        asset = Asset(
+            name=asset_name,
+            asset_type_name="test-type",
+            event_resolution=timedelta(minutes=15),
+            capacity_in_mw=1,
+            latitude=100,
+            longitude=100,
+            unit="MW",
+        )
+        asset.owner = test_prosumer
+        if asset_name == "CS 4":
+            asset.event_resolution = timedelta(hours=1)
+        assets.append(asset)
+        db.session.add(asset)

flexmeasures/api/v1/tests/test_api_v1.py

@@ -4,9 +4,6 @@
 import isodate
 import pandas as pd
 import pytest
-from iso8601 import parse_date
-from numpy import repeat
-
 
 from flexmeasures.api.common.responses import (
     invalid_domain,
@@ -24,7 +21,6 @@
     verify_power_in_db,
 )
 from flexmeasures.data.auth_setup import UNAUTH_ERROR_STATUS
-from flexmeasures.api.v1.tests.utils import count_connections_in_post_message
 from flexmeasures.data.models.assets import Asset
 
 
@@ -250,94 +246,7 @@ def test_get_meter_data(db, app, client, message):
     assert get_meter_data_response.json["values"] == [(100.0 + i) for i in range(6)]
 
 
-@pytest.mark.parametrize(
-    "post_message",
-    [
-        message_for_post_meter_data(),
-        message_for_post_meter_data(single_connection=True),
-        message_for_post_meter_data(single_connection_group=True),
-    ],
-)
-@pytest.mark.parametrize(
-    "get_message",
-    [
-        message_for_get_meter_data(),
-        message_for_get_meter_data(single_connection=False),
-        message_for_get_meter_data(resolution="PT30M"),
-    ],
-)
-def test_post_and_get_meter_data(db, app, client, post_message, get_message):
-    """
-    Tries to post meter data as a logged-in test user with the MDC role, which should succeed.
-    There should be some ForecastingJobs waiting now.
-    Then tries to get meter data, which should succeed, and should return the same meter data as was posted,
-    or a downsampled version, if that was requested.
-    """
-
-    # post meter data
-    auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest")
-    post_meter_data_response = client.post(
-        url_for("flexmeasures_api_v1.post_meter_data"),
-        json=message_replace_name_with_ea(post_message),
-        headers={"Authorization": auth_token},
-    )
-    print("Server responded with:\n%s" % post_meter_data_response.json)
-    assert post_meter_data_response.status_code == 200
-    assert post_meter_data_response.json["type"] == "PostMeterDataResponse"
-
-    # look for Forecasting jobs
-    expected_connections = count_connections_in_post_message(post_message)
-    assert (
-        len(app.queues["forecasting"]) == 4 * expected_connections
-    )  # four horizons times the number of assets
-    horizons = repeat(
-        [
-            timedelta(hours=1),
-            timedelta(hours=6),
-            timedelta(hours=24),
-            timedelta(hours=48),
-        ],
-        expected_connections,
-    )
-    jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"])
-    for job, horizon in zip(jobs, horizons):
-        assert job.kwargs["horizon"] == horizon
-        assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon
-    for asset_name in ("CS 1", "CS 2", "CS 3"):
-        if asset_name in str(post_message):
-            asset = Asset.query.filter_by(name=asset_name).one_or_none()
-            assert asset.id in [job.kwargs["asset_id"] for job in jobs]
-
-    # get meter data
-    get_meter_data_response = client.get(
-        url_for("flexmeasures_api_v1.get_meter_data"),
-        query_string=message_replace_name_with_ea(get_message),
-        headers={"Authorization": auth_token},
-    )
-    print("Server responded with:\n%s" % get_meter_data_response.json)
-    assert get_meter_data_response.status_code == 200
-    assert get_meter_data_response.json["type"] == "GetMeterDataResponse"
-    if "groups" in post_message:
-        posted_values = post_message["groups"][0]["values"]
-    else:
-        posted_values = post_message["values"]
-    if "groups" in get_meter_data_response.json:
-        gotten_values = get_meter_data_response.json["groups"][0]["values"]
-    else:
-        gotten_values = get_meter_data_response.json["values"]
-
-    if "resolution" not in get_message or get_message["resolution"] == "":
-        assert gotten_values == posted_values
-    else:
-        # We used a target resolution of 30 minutes, so double of 15 minutes.
-        # Six values went in, three come out.
-        if posted_values[1] > 0:  # see utils.py:message_for_post_meter_data
-            assert gotten_values == [306.66, -0.0, 306.66]
-        else:
-            assert gotten_values == [153.33, 0, 306.66]
-
-
-def test_post_meter_data_to_different_resolutions(db, app, client):
+def test_post_meter_data_to_different_resolutions(app, client):
     """
     Tries to post meter data to assets with different event_resolutions, which is not accepted.
     """

flexmeasures/api/v1/tests/test_api_v1_fresh_db.py

@@ -0,0 +1,104 @@
+from datetime import timedelta
+
+import pytest
+from flask import url_for
+from iso8601 import parse_date
+from numpy import repeat
+
+from flexmeasures.api.common.utils.api_utils import message_replace_name_with_ea
+from flexmeasures.api.tests.utils import get_auth_token
+from flexmeasures.api.v1.tests.utils import (
+    message_for_post_meter_data,
+    message_for_get_meter_data,
+    count_connections_in_post_message,
+)
+from flexmeasures.data.models.assets import Asset
+
+
+@pytest.mark.parametrize(
+    "post_message",
+    [
+        message_for_post_meter_data(),
+        message_for_post_meter_data(single_connection=True),
+        message_for_post_meter_data(single_connection_group=True),
+    ],
+)
+@pytest.mark.parametrize(
+    "get_message",
+    [
+        message_for_get_meter_data(),
+        message_for_get_meter_data(single_connection=False),
+        message_for_get_meter_data(resolution="PT30M"),
+    ],
+)
+def test_post_and_get_meter_data(
+    setup_fresh_api_test_data, app, clean_redis, client, post_message, get_message
+):
+    """
+    Tries to post meter data as a logged-in test user with the MDC role, which should succeed.
+    There should be some ForecastingJobs waiting now.
+    Then tries to get meter data, which should succeed, and should return the same meter data as was posted,
+    or a downsampled version, if that was requested.
+    """
+
+    # post meter data
+    auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest")
+    post_meter_data_response = client.post(
+        url_for("flexmeasures_api_v1.post_meter_data"),
+        json=message_replace_name_with_ea(post_message),
+        headers={"Authorization": auth_token},
+    )
+    print("Server responded with:\n%s" % post_meter_data_response.json)
+    assert post_meter_data_response.status_code == 200
+    assert post_meter_data_response.json["type"] == "PostMeterDataResponse"
+
+    # look for Forecasting jobs
+    expected_connections = count_connections_in_post_message(post_message)
+    assert (
+        len(app.queues["forecasting"]) == 4 * expected_connections
+    )  # four horizons times the number of assets
+    horizons = repeat(
+        [
+            timedelta(hours=1),
+            timedelta(hours=6),
+            timedelta(hours=24),
+            timedelta(hours=48),
+        ],
+        expected_connections,
+    )
+    jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"])
+    for job, horizon in zip(jobs, horizons):
+        assert job.kwargs["horizon"] == horizon
+        assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon
+    for asset_name in ("CS 1", "CS 2", "CS 3"):
+        if asset_name in str(post_message):
+            asset = Asset.query.filter_by(name=asset_name).one_or_none()
+            assert asset.id in [job.kwargs["asset_id"] for job in jobs]
+
+    # get meter data
+    get_meter_data_response = client.get(
+        url_for("flexmeasures_api_v1.get_meter_data"),
+        query_string=message_replace_name_with_ea(get_message),
+        headers={"Authorization": auth_token},
+    )
+    print("Server responded with:\n%s" % get_meter_data_response.json)
+    assert get_meter_data_response.status_code == 200
+    assert get_meter_data_response.json["type"] == "GetMeterDataResponse"
+    if "groups" in post_message:
+        posted_values = post_message["groups"][0]["values"]
+    else:
+        posted_values = post_message["values"]
+    if "groups" in get_meter_data_response.json:
+        gotten_values = get_meter_data_response.json["groups"][0]["values"]
+    else:
+        gotten_values = get_meter_data_response.json["values"]
+
+    if "resolution" not in get_message or get_message["resolution"] == "":
+        assert gotten_values == posted_values
+    else:
+        # We used a target resolution of 30 minutes, so double of 15 minutes.
+        # Six values went in, three come out.
+        if posted_values[1] > 0:  # see utils.py:message_for_post_meter_data
+            assert gotten_values == [306.66, -0.0, 306.66]
+        else:
+            assert gotten_values == [153.33, 0, 306.66]

flexmeasures/api/v1_1/tests/conftest.py

@@ -11,8 +11,8 @@
 from flexmeasures.data.services.users import create_user
 
 
-@pytest.fixture(scope="function", autouse=True)
-def setup_api_test_data(db):
+@pytest.fixture(scope="module")
+def setup_api_test_data(db, setup_roles_users, add_market_prices):
     """
     Set up data for API v1.1 tests.
     """
@@ -41,7 +41,7 @@ def setup_api_test_data(db):
     )
 
     # Create 3 test assets for the test_prosumer user
-    test_prosumer = user_datastore.find_user(email="test_prosumer@seita.nl")
+    test_prosumer = setup_roles_users["Test Prosumer"]
     test_asset_type = AssetType(name="test-type")
     db.session.add(test_asset_type)
     asset_names = ["CS 1", "CS 2", "CS 3"]
@@ -124,3 +124,10 @@ def setup_api_test_data(db):
     db.session.add(sensor)
 
     print("Done setting up data for API v1.1 tests")
+
+
+@pytest.fixture(scope="function")
+def setup_fresh_api_v1_1_test_data(
+    fresh_db, setup_roles_users_fresh_db, setup_markets_fresh_db
+):
+    return fresh_db