FlexMeasures · Flix6x · May 1, 2023 · Apr 25, 2023 · Apr 25, 2023 · Apr 25, 2023
diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
@@ -5,6 +5,12 @@ API change log
 
 .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace.
 
+v3.0-9 | 2023-04-26
+"""""""""""""""""""
+
+- Added missing documentation for the public endpoints for authentication and listing active API versions.
+- Added REST endpoint for listing available services for a specific API version: `/api/v3_0` (GET). This functionality is similar to the *getService* endpoint for older API versions, but now also returns the full URL for each available service.
+
 v3.0-8 | 2023-03-23
 """""""""""""""""""
 
@@ -82,7 +88,7 @@ v3.0-0 | 2022-03-25
     - *getDeviceMessage* -> use `/sensors/<id>/schedules/<uuid>` (GET) instead, where <id> is the sensor id from the "event" field and <uuid> is the value of the "schedule" field returned by `/sensors/<id>/schedules/trigger` (POST)
     - *getMeterData* -> use `/sensors/data` (GET) instead, replacing the "connection" field with "sensor"
     - *getPrognosis* -> use `/sensors/data` (GET) instead, replacing the "connection" field with "sensor"
-    - *getService* -> consult the public API documentation instead, at https://flexmeasures.readthedocs.io
+    - *getService* -> use `/api/v3_0` (GET) instead (since v3.0-9), or consult the public API documentation instead, at https://flexmeasures.readthedocs.io
     - *postMeterData* -> use `/sensors/data` (POST) instead, replacing the "connection" field with "sensor"
     - *postPriceData* -> use `/sensors/data` (POST) instead, replacing the "market" field with "sensor"
     - *postPrognosis* -> use `/sensors/data` (POST) instead, replacing the "connection" field with "sensor"
@@ -106,8 +112,13 @@ v3.0-0 | 2022-03-25
     - Rewrote the sections on roles and sources into a combined section that refers to account roles rather than USEF roles.
     - Deprecated the section on group notation.
 
+v2.0-6 | 2022-04-26
+"""""""""""""""""""
+
+*API v2.0 is sunset.*
+
 v2.0-5 | 2022-02-13
-""""""""""""""""""""
+"""""""""""""""""""
 
 *API v2.0 is deprecated.*
 
@@ -152,6 +163,11 @@ v2.0-0 | 2020-11-14
 
 - Added REST endpoints for managing assets: `/assets/` (GET, POST) and `/asset/<id>` (GET, PATCH, DELETE).
 
+v1.3-13 | 2022-04-26
+""""""""""""""""""""
+
+*API v1.3 is sunset.*
+
 v1.3-12 | 2022-02-13
 """"""""""""""""""""
 
@@ -243,8 +259,13 @@ v1.3-0 | 2020-01-28
 - The *postUdiEvent* endpoint now triggers scheduling jobs to be set up (rather than scheduling directly triggered by the *getDeviceMessage* endpoint)
 - The *getDeviceMessage* now queries the job queue and database for an available schedule
 
+v1.2-5 | 2022-04-26
+"""""""""""""""""""
+
+*API v1.2 is sunset.*
+
 v1.2-4 | 2022-02-13
-""""""""""""""""""""
+"""""""""""""""""""
 
 *API v1.2 is deprecated.*
 
@@ -294,8 +315,13 @@ v1.2-0 | 2018-09-08
 - Added a description of the *postUdiEvent* endpoint in the Prosumer and Simulation sections
 - Added a description of the *getDeviceMessage* endpoint in the Prosumer and Simulation sections
 
+v1.1-7 | 2022-04-26
+"""""""""""""""""""
+
+*API v1.1 is sunset.*
+
 v1.1-6 | 2022-02-13
-""""""""""""""""""""
+"""""""""""""""""""
 
 *API v1.1 is deprecated.*
 
@@ -352,8 +378,13 @@ v1.1-0 | 2018-07-15
 
 - Added a description of the *getPrognosis* endpoint in the Supplier section
 
+v1.0-3 | 2022-04-26
+"""""""""""""""""""
+
+*API v1.0 is sunset.*
+
 v1.0-2 | 2022-02-13
-""""""""""""""""""""
+"""""""""""""""""""
 
 *API v1.0 is deprecated.*
 

diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst
@@ -107,6 +107,11 @@ which gives a response like this if the credentials are correct:
 Deprecation and sunset
 ----------------------
 
+Some sunsetting options are available for FlexMeasures hosts. See :ref:`api_deprecation_hosts`.
+
+FlexMeasures clients
+^^^^^^^^^^^^^^^^^^^^
+
 Professional API users should monitor API responses for the ``"Deprecation"`` and ``"Sunset"`` response headers [see `draft-ietf-httpapi-deprecation-header-02 <https://datatracker.ietf.org/doc/draft-ietf-httpapi-deprecation-header/>`_ and `RFC 8594 <https://www.rfc-editor.org/rfc/rfc8594>`_, respectively], so system administrators can be warned when using API endpoints that are flagged for deprecation and/or are likely to become unresponsive in the future.
 
 The deprecation header field shows an `IMF-fixdate <https://www.rfc-editor.org/rfc/rfc7231#section-7.1.1.1>`_ indicating when the API endpoint was deprecated.
@@ -139,3 +144,19 @@ Here is a client-side code example in Python (this merely prints out the depreca
                 print(f"Your request to {url} returned a sunset warning. Sunset: {content}")
             elif header == "Link" and ('rel="deprecation";' in content or 'rel="sunset";' in content):
                 print(f"Further info is available: {content}")
+
+.. _api_deprecation_hosts:
+
+FlexMeasures hosts
+^^^^^^^^^^^^^^^^^^
+
+When upgrading to a FlexMeasures version that sunsets an API version, clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints.
+After upgrading to one of the next FlexMeasures versions, they will receive ``HTTP status 404 (Not Found)`` responses.
+
+Hosts should not expect every client to monitor response headers and proactively upgrade to newer API versions.
+Please make sure that your users have upgraded before you upgrade to a FlexMeasures version that sunsets an API version.
+You can do this by checking your server logs for warnings about users who are still calling deprecated endpoints.
+
+In case you have users that haven't upgraded yet, and would still like to upgrade FlexMeasures, you can.
+Just set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = False`` and consider announcing some blackout tests to your users, during which you can set this setting to ``True`` to activate the sunset.
+During such a blackout test, clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints.
diff --git a/documentation/api/v1.rst b/documentation/api/v1.rst
@@ -3,20 +3,5 @@
 Version 1.0
 ===========
 
-.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`.
 
-Summary
--------
-
-.. qrefflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1
-    :order: path
-    :include-empty-docstring:
-
-API Details
------------
-
-.. autoflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1
-    :order: path
-    :include-empty-docstring:
+.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 1.0 will stay available `here on readthedocs.org <https://flexmeasures.readthedocs.io/en/v0.12.3/api/v1.html>`_.
diff --git a/documentation/api/v1_1.rst b/documentation/api/v1_1.rst
@@ -3,20 +3,5 @@
 Version 1.1
 ===========
 
-.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`.
 
-Summary
--------
-
-.. qrefflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_1
-    :order: path
-    :include-empty-docstring:
-
-API Details
------------
-
-.. autoflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_1
-    :order: path
-    :include-empty-docstring:
+.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 1.1 will stay available `here on readthedocs.org <https://flexmeasures.readthedocs.io/en/v0.12.3/api/v1_1.html>`_.
diff --git a/documentation/api/v1_2.rst b/documentation/api/v1_2.rst
@@ -3,20 +3,5 @@
 Version 1.2
 ===========
 
-.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`.
 
-Summary
--------
-
-.. qrefflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_2
-    :order: path
-    :include-empty-docstring:
-
-API Details
------------
-
-.. autoflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_2
-    :order: path
-    :include-empty-docstring:
+.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 1.2 will stay available `here on readthedocs.org <https://flexmeasures.readthedocs.io/en/v0.12.3/api/v1_2.html>`_.
diff --git a/documentation/api/v1_3.rst b/documentation/api/v1_3.rst
@@ -3,20 +3,5 @@
 Version 1.3
 ===========
 
-.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`.
 
-Summary
--------
-
-.. qrefflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_3
-    :order: path
-    :include-empty-docstring:
-
-API Details
------------
-
-.. autoflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_3
-    :order: path
-    :include-empty-docstring:
+.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 1.3 will stay available `here on readthedocs.org <https://flexmeasures.readthedocs.io/en/v0.12.3/api/v1_3.html>`_.
diff --git a/documentation/api/v2_0.rst b/documentation/api/v2_0.rst
@@ -3,20 +3,4 @@
 Version 2.0
 ===========
 
-.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`.
-
-Summary
--------
-
-.. qrefflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v2_0
-    :order: path
-    :include-empty-docstring:
-
-API Details
------------
-
-.. autoflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v2_0
-    :order: path
-    :include-empty-docstring:
+.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 2.0 will stay available `here on readthedocs.org <https://flexmeasures.readthedocs.io/en/v0.12.3/api/v2_0.html>`_.
diff --git a/documentation/api/v3_0.rst b/documentation/api/v3_0.rst
@@ -7,14 +7,14 @@ Summary
 -------
 
 .. qrefflask:: flexmeasures.app:create(env="documentation")
-    :modules: flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health
+    :modules: flexmeasures.api, flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health, flexmeasures.api.v3_0.public
     :order: path
     :include-empty-docstring:
 
 API Details
 -----------
 
 .. autoflask:: flexmeasures.app:create(env="documentation")
-    :modules: flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health
+    :modules: flexmeasures.api, flexmeasures.api.v3_0.assets, flexmeasures.api.v3_0.sensors, flexmeasures.api.v3_0.users, flexmeasures.api.v3_0.health, flexmeasures.api.v3_0.public
     :order: path
     :include-empty-docstring:
diff --git a/documentation/changelog.rst b/documentation/changelog.rst
@@ -6,6 +6,10 @@ FlexMeasures Changelog
 v0.13.0 | April XX, 2023
 ============================
 
+.. warning:: Sunset notice for API versions 1.0, 1.1, 1.2, 1.3 and 2.0: after upgrading to ``flexmeasures==0.13``, users of these API versions may receive ``HTTP status 410 (Gone)`` responses.
+             See the `documentation for deprecation and sunset <https://flexmeasures.readthedocs.io/en/latest/api/introduction.html#deprecation-and-sunset>`_.
+             The relevant endpoints have been deprecated since ``flexmeasures==0.12``.
+
 .. warning:: The API endpoint (`[POST] /sensors/(id)/schedules/trigger <api/v3_0.html#post--api-v3_0-sensors-(id)-schedules-trigger>`_) to make new schedules sunsets the deprecated (since v0.12) storage flexibility parameters (they move to the ``flex-model`` parameter group), as well as the parameters describing other sensors (they move to ``flex-context``).
 
 .. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``).
@@ -26,6 +30,8 @@ Bugfixes
 
 Infrastructure / Support
 ----------------------
+* Support blackout tests for sunset API versions [see `PR #651 <https://www.github.com/FlexMeasures/flexmeasures/pull/651>`_]
+* Sunset API versions 1.0, 1.1, 1.2, 1.3 and 2.0 [see `PR #650 <https://www.github.com/FlexMeasures/flexmeasures/pull/650>`_]
 * Sunset several API fields for `/sensors/<id>/schedules/trigger` (POST) that have moved into the ``flex-model`` or ``flex-context`` fields [see `PR #580 <https://www.github.com/FlexMeasures/flexmeasures/pull/580>`_]
 * Fix broken `make show-data-model` command [see `PR #638 <https://www.github.com/FlexMeasures/flexmeasures/pull/638>`_]
 
@@ -532,7 +538,7 @@ Infrastructure / Support
 * Integration with `timely beliefs <https://github.com/SeitaBV/timely-beliefs>`__ lib: Sensors [see `PR #13 <https://www.github.com/FlexMeasures/flexmeasures/pull/13>`_]
 * Apache 2.0 license [see `PR #16 <https://www.github.com/FlexMeasures/flexmeasures/pull/16>`_]
 * Load js & css from CDN [see `PR #21 <https://www.github.com/FlexMeasures/flexmeasures/pull/21>`_]
-* Start using marshmallow for input validation, also introducing ``HTTP status 422`` in the API [see `PR #25 <https://www.github.com/FlexMeasures/flexmeasures/pull/25>`_]
+* Start using marshmallow for input validation, also introducing ``HTTP status 422 (Unprocessable Entity)`` in the API [see `PR #25 <https://www.github.com/FlexMeasures/flexmeasures/pull/25>`_]
 * Replace ``solarpy`` with ``pvlib`` (due to license conflict) [see `PR #16 <https://www.github.com/FlexMeasures/flexmeasures/pull/16>`_]
 * Stop supporting the creation of new users on asset creation (to reduce complexity) [see `PR #36 <https://www.github.com/FlexMeasures/flexmeasures/pull/36>`_]
 
diff --git a/documentation/configuration.rst b/documentation/configuration.rst
@@ -592,3 +592,29 @@ When ``FLEXMEASURES_MODE=demo``\ , this setting can be used to make the FlexMeas
 so that old imported data can be demoed as if it were current.
 
 Default: ``None``
+
+Sunset
+------
+
+FLEXMEASURES_API_SUNSET_ACTIVE
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Allow control over the effect of sunsetting API versions.
+Specifically, if True, the endpoints in sunset versions will return ``HTTP status 410 (Gone)`` status codes.
+If False, the endpoints will work like before, including Deprecation and Sunset headers in their response.
+
+Default: ``True``
+
+FLEXMEASURES_API_SUNSET_DATE
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Allow to override the default sunset date for your clients.
+
+Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"2023-05-01"`` for v2.0)
+
+FLEXMEASURES_API_SUNSET_LINK
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Allow to override the default sunset link for your clients.
+
+Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html"`` for v2.0)
diff --git a/documentation/dev/api.rst b/documentation/dev/api.rst
@@ -7,6 +7,8 @@ The FlexMeasures API is the main way that third-parties can automate their inter
 
 This is a small guide for creating new versions of the API and its docs.
 
+.. warning:: This guide was written for API versions below v3.0 and is currently out of date.
+
 .. todo:: A guide for endpoint design, e.g. using Marshmallow schemas and common validators.
 
 .. contents:: Table of contents

diff --git a/flexmeasures/__init__.py b/flexmeasures/__init__.py
@@ -1,7 +1,12 @@
 from importlib_metadata import version, PackageNotFoundError
 
 from flexmeasures.data.models.annotations import Annotation  # noqa F401
-from flexmeasures.data.models.user import Account, AccountRole, User  # noqa F401
+from flexmeasures.data.models.user import (  # noqa F401
+    Account,
+    AccountRole,
+    User,
+    Role as UserRole,
+)
 from flexmeasures.data.models.data_sources import DataSource as Source  # noqa F401
 from flexmeasures.data.models.generic_assets import (  # noqa F401
     GenericAsset as Asset,

diff --git a/flexmeasures/api/__init__.py b/flexmeasures/api/__init__.py
@@ -73,9 +73,9 @@ def get_versions() -> dict:
     """
     response = {
         "message": "For these API versions a public endpoint is available, listing its service. For example: "
-        "/api/v2_0/getService and /api/v3_0/getService. An authentication token can be requested at: "
+        "/api/v3_0. An authentication token can be requested at: "
         "/api/requestAuthToken",
-        "versions": ["v1", "v1_1", "v1_2", "v1_3", "v2_0", "v3_0"],
+        "versions": ["v3_0"],
         "flexmeasures_version": flexmeasures_version,
     }
     return response

diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py
@@ -10,12 +10,12 @@
 import pandas as pd
 
 from flexmeasures.data import ma
-from flexmeasures.data.models.data_sources import DataSource
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.api.common.schemas.sensors import SensorField
 from flexmeasures.api.common.utils.api_utils import upsample_values
 from flexmeasures.data.models.planning.utils import initialize_index
 from flexmeasures.data.schemas import AwareDateTimeField, DurationField, SourceIdField
+from flexmeasures.data.services.data_sources import get_or_create_source
 from flexmeasures.data.services.time_series import simplify_index
 from flexmeasures.utils.time_utils import (
     decide_resolution,
@@ -350,14 +350,7 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame:
         """
         Turn the de-serialized and validated data into a BeliefsDataFrame.
         """
-        source = DataSource.query.filter(
-            DataSource.user_id == current_user.id
-        ).one_or_none()
-        if not source:
-            raise ValidationError(
-                f"User {current_user.id} is not an accepted data source."
-            )
-
+        source = get_or_create_source(current_user)
         num_values = len(sensor_data["values"])
         event_resolution = sensor_data["duration"] / num_values
         dt_index = pd.date_range(