diff --git a/documentation/_static/css/custom.css b/documentation/_static/css/custom.css index 1de969a7d..9cd328ec0 100644 --- a/documentation/_static/css/custom.css +++ b/documentation/_static/css/custom.css @@ -20,4 +20,8 @@ div .contents li { -webkit-column-break-inside: avoid; page-break-inside: avoid; break-inside: avoid-column; -} \ No newline at end of file +} + +div.admonition.info-icon > .admonition-title:before { + content: "\f05a"; /* the fa-circle-info icon */ +} diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 6a7b2b974..5f200b962 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -107,6 +107,11 @@ which gives a response like this if the credentials are correct: Deprecation and sunset ---------------------- +Some sunsetting options are available for FlexMeasures hosts. See :ref:`api_deprecation_hosts`. + +FlexMeasures clients +^^^^^^^^^^^^^^^^^^^^ + Professional API users should monitor API responses for the ``"Deprecation"`` and ``"Sunset"`` response headers [see `draft-ietf-httpapi-deprecation-header-02 `_ and `RFC 8594 `_, respectively], so system administrators can be warned when using API endpoints that are flagged for deprecation and/or are likely to become unresponsive in the future. The deprecation header field shows an `IMF-fixdate `_ indicating when the API endpoint was deprecated. @@ -139,3 +144,33 @@ Here is a client-side code example in Python (this merely prints out the depreca print(f"Your request to {url} returned a sunset warning. Sunset: {content}") elif header == "Link" and ('rel="deprecation";' in content or 'rel="sunset";' in content): print(f"Further info is available: {content}") + +.. _api_deprecation_hosts: + +FlexMeasures hosts +^^^^^^^^^^^^^^^^^^ + +When upgrading to a FlexMeasures version that sunsets an API version (e.g. ``flexmeasures==0.13.0`` sunsets API version 2), clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints. +After upgrading to one of the next FlexMeasures versions (e.g. ``flexmeasures==0.14.0``), they will receive ``HTTP status 404 (Not Found)`` responses. + +Hosts should not expect every client to monitor response headers and proactively upgrade to newer API versions. +Please make sure that your users have upgraded before you upgrade to a FlexMeasures version that sunsets an API version. +You can do this by checking your server logs for warnings about users who are still calling deprecated endpoints. + +In addition, we recommend running blackout tests during the deprecation notice phase. +You (and your users) can learn which systems need attention and how to deal with them. +Be sure to announce these beforehand. +Here is an example of how to run a blackout test: +If a sunset happens in version ``0.13``, and you are hosting a version which includes the deprecation notice (e.g. ``0.12``), FlexMeasures will simulate the sunset if you set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = True`` (see :ref:`Sunset Configuration`). +During such a blackout test, clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints. + +.. admonition:: What is a blackout test + :class: info-icon + + A blackout test is a planned, timeboxed event when a host will turn off a certain API or some of the API capabilities. + The test is meant to help developers understand the impact the retirement will have on the applications and users. + `Source: Platform of Trust `_ + +In case you have users that haven't upgraded yet, and would still like to upgrade FlexMeasures (to the version that officially sunsets the API version), you can. +For a little while after sunset (usually one more minor version), we will continue to support "letting the sun unset". +To enable this, just set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = False`` and consider announcing some more blackout tests to your users, during which you can set this setting to ``True`` to activate the sunset. diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 49fd58fa4..490cf169d 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -7,6 +7,7 @@ v0.13.0 | April XX, 2023 ============================ .. warning:: Sunset notice for API versions 1.0, 1.1, 1.2, 1.3 and 2.0: after upgrading to ``flexmeasures==0.13``, users of these API versions may receive ``HTTP status 410 (Gone)`` responses. + See the `documentation for deprecation and sunset `_. The relevant endpoints have been deprecated since ``flexmeasures==0.12``. .. warning:: The API endpoint (`[POST] /sensors/(id)/schedules/trigger `_) to make new schedules sunsets the deprecated (since v0.12) storage flexibility parameters (they move to the ``flex-model`` parameter group), as well as the parameters describing other sensors (they move to ``flex-context``). @@ -30,6 +31,7 @@ Bugfixes Infrastructure / Support ---------------------- +* Support blackout tests for sunset API versions [see `PR #651 `_] * Sunset API versions 1.0, 1.1, 1.2, 1.3 and 2.0 [see `PR #650 `_] * Sunset several API fields for `/sensors//schedules/trigger` (POST) that have moved into the ``flex-model`` or ``flex-context`` fields [see `PR #580 `_] * Fix broken `make show-data-model` command [see `PR #638 `_] diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 27c4766fc..b3568f5f3 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -592,3 +592,31 @@ When ``FLEXMEASURES_MODE=demo``\ , this setting can be used to make the FlexMeas so that old imported data can be demoed as if it were current. Default: ``None`` + +.. _sunset-config: + +Sunset +------ + +FLEXMEASURES_API_SUNSET_ACTIVE +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Allow control over the effect of sunsetting API versions. +Specifically, if True, the endpoints in sunset versions will return ``HTTP status 410 (Gone)`` status codes. +If False, the endpoints will work like before, including Deprecation and Sunset headers in their response. + +Default: ``True`` + +FLEXMEASURES_API_SUNSET_DATE +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Allow to override the default sunset date for your clients. + +Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"2023-05-01"`` for v2.0) + +FLEXMEASURES_API_SUNSET_LINK +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Allow to override the default sunset link for your clients. + +Default: ``None`` (defaults are set internally for each sunset API version, e.g. ``"https://flexmeasures.readthedocs.io/en/v0.13.0/api/v2_0.html"`` for v2.0) diff --git a/flexmeasures/api/common/utils/deprecation_utils.py b/flexmeasures/api/common/utils/deprecation_utils.py index 84263de38..f2b207ddf 100644 --- a/flexmeasures/api/common/utils/deprecation_utils.py +++ b/flexmeasures/api/common/utils/deprecation_utils.py @@ -1,12 +1,39 @@ from __future__ import annotations -from flask import current_app, request, Blueprint, Response, after_this_request +from typing import Any + +from flask import abort, current_app, request, Blueprint, Response, after_this_request from flask_security.core import current_user import pandas as pd from flexmeasures.utils.time_utils import to_http_time +def sunset_blueprint( + blueprint, + api_version_sunset: str, + sunset_link: str, + api_version_upgrade_to: str = "3.0", +): + """Sunsets every route on a blueprint by returning 410 (Gone) responses. + + Such errors will be logged by utils.error_utils.error_handling_router. + """ + + def let_host_switch_to_returning_410(): + + # Override with custom info link, if set by host + _sunset_link = override_from_config(sunset_link, "FLEXMEASURES_API_SUNSET_LINK") + + if current_app.config["FLEXMEASURES_API_SUNSET_ACTIVE"]: + abort( + 410, + f"API version {api_version_sunset} has been sunset. Please upgrade to API version {api_version_upgrade_to}. See {_sunset_link} for more information.", + ) + + blueprint.before_request(let_host_switch_to_returning_410) + + def deprecate_fields( fields: str | list[str], deprecation_date: pd.Timestamp | str | None = None, @@ -50,7 +77,8 @@ def post_item(color, length): """ if not isinstance(fields, list): fields = [fields] - deprecation, sunset = _format_deprecation_and_sunset(deprecation_date, sunset_date) + deprecation = _format_deprecation(deprecation_date) + sunset = _format_sunset(sunset_date) @after_this_request def _after_request_handler(response: Response) -> Response: @@ -63,12 +91,21 @@ def _after_request_handler(response: Response) -> Response: current_app.logger.warning( f"Endpoint {request.endpoint} called by {current_user} with deprecated fields: {deprecated_fields_used}" ) + + # Override sunset date if host used corresponding config setting + _sunset = override_from_config(sunset, "FLEXMEASURES_API_SUNSET_DATE") + + # Override sunset link if host used corresponding config setting + _sunset_link = override_from_config( + sunset_link, "FLEXMEASURES_API_SUNSET_LINK" + ) + return _add_headers( response, deprecation, deprecation_link, - sunset, - sunset_link, + _sunset, + _sunset_link, ) return response @@ -109,18 +146,26 @@ def deprecate_blueprint( - Deprecation header: https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header - Sunset header: https://www.rfc-editor.org/rfc/rfc8594 """ - deprecation, sunset = _format_deprecation_and_sunset(deprecation_date, sunset_date) + deprecation = _format_deprecation(deprecation_date) + sunset = _format_sunset(sunset_date) def _after_request_handler(response: Response) -> Response: current_app.logger.warning( f"Deprecated endpoint {request.endpoint} called by {current_user}" ) + + # Override sunset date if host used corresponding config setting + _sunset = override_from_config(sunset, "FLEXMEASURES_API_SUNSET_DATE") + + # Override sunset link if host used corresponding config setting + _sunset_link = override_from_config(sunset_link, "FLEXMEASURES_API_SUNSET_LINK") + return _add_headers( response, deprecation, deprecation_link, - sunset, - sunset_link, + _sunset, + _sunset_link, ) blueprint.after_request(_after_request_handler) @@ -149,13 +194,27 @@ def _add_link(response: Response, link: str, rel: str) -> Response: return response -def _format_deprecation_and_sunset(deprecation_date, sunset_date): +def _format_deprecation(deprecation_date): if deprecation_date: deprecation = to_http_time(pd.Timestamp(deprecation_date) - pd.Timedelta("1s")) else: deprecation = "true" + return deprecation + + +def _format_sunset(sunset_date): if sunset_date: sunset = to_http_time(pd.Timestamp(sunset_date) - pd.Timedelta("1s")) else: sunset = None - return deprecation, sunset + return sunset + + +def override_from_config(setting: Any, config_setting_name: str) -> Any: + """Override setting by config setting, unless the latter is None or is missing.""" + config_setting = current_app.config.get(config_setting_name) + if config_setting is not None: + _setting = config_setting + else: + _setting = setting + return _setting diff --git a/flexmeasures/api/v1/__init__.py b/flexmeasures/api/v1/__init__.py index 8f8632cff..0fdf4d196 100644 --- a/flexmeasures/api/v1/__init__.py +++ b/flexmeasures/api/v1/__init__.py @@ -1,6 +1,9 @@ from flask import Flask, Blueprint -from flexmeasures.api.common.utils.deprecation_utils import deprecate_blueprint +from flexmeasures.api.common.utils.deprecation_utils import ( + deprecate_blueprint, + sunset_blueprint, +) # The api blueprint. It is registered with the Flask app (see register_at) @@ -9,8 +12,13 @@ flexmeasures_api, deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/introduction.html#deprecation-and-sunset", - sunset_date="2023-02-01", - sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v1.html", + sunset_date="2023-05-01", + sunset_link="https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1.html", +) +sunset_blueprint( + flexmeasures_api, + "1.0", + "https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1.html", ) diff --git a/flexmeasures/api/v1_1/__init__.py b/flexmeasures/api/v1_1/__init__.py index 45fa12d61..774808627 100644 --- a/flexmeasures/api/v1_1/__init__.py +++ b/flexmeasures/api/v1_1/__init__.py @@ -1,6 +1,9 @@ from flask import Flask, Blueprint -from flexmeasures.api.common.utils.deprecation_utils import deprecate_blueprint +from flexmeasures.api.common.utils.deprecation_utils import ( + deprecate_blueprint, + sunset_blueprint, +) # The api blueprint. It is registered with the Flask app (see app.py) flexmeasures_api = Blueprint("flexmeasures_api_v1_1", __name__) @@ -8,8 +11,13 @@ flexmeasures_api, deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_1.html", - sunset_date="2023-02-01", - sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_1.html", + sunset_date="2023-05-01", + sunset_link="https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1_1.html", +) +sunset_blueprint( + flexmeasures_api, + "1.1", + "https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1_1.html", ) diff --git a/flexmeasures/api/v1_2/__init__.py b/flexmeasures/api/v1_2/__init__.py index b34ee17a5..09c18def0 100644 --- a/flexmeasures/api/v1_2/__init__.py +++ b/flexmeasures/api/v1_2/__init__.py @@ -1,6 +1,9 @@ from flask import Flask, Blueprint -from flexmeasures.api.common.utils.deprecation_utils import deprecate_blueprint +from flexmeasures.api.common.utils.deprecation_utils import ( + deprecate_blueprint, + sunset_blueprint, +) # The api blueprint. It is registered with the Flask app (see app.py) flexmeasures_api = Blueprint("flexmeasures_api_v1_2", __name__) @@ -8,8 +11,13 @@ flexmeasures_api, deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_2.html", - sunset_date="2023-02-01", - sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_2.html", + sunset_date="2023-05-01", + sunset_link="https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1_2.html", +) +sunset_blueprint( + flexmeasures_api, + "1.2", + "https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1_2.html", ) diff --git a/flexmeasures/api/v1_3/__init__.py b/flexmeasures/api/v1_3/__init__.py index d6b14b119..ce6308d9f 100644 --- a/flexmeasures/api/v1_3/__init__.py +++ b/flexmeasures/api/v1_3/__init__.py @@ -1,6 +1,9 @@ from flask import Flask, Blueprint -from flexmeasures.api.common.utils.deprecation_utils import deprecate_blueprint +from flexmeasures.api.common.utils.deprecation_utils import ( + deprecate_blueprint, + sunset_blueprint, +) # The api blueprint. It is registered with the Flask app (see app.py) flexmeasures_api = Blueprint("flexmeasures_api_v1_3", __name__) @@ -8,8 +11,13 @@ flexmeasures_api, deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_3.html", - sunset_date="2023-02-01", - sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v1_3.html", + sunset_date="2023-05-01", + sunset_link="https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1_3.html", +) +sunset_blueprint( + flexmeasures_api, + "1.3", + "https://flexmeasures.readthedocs.io/en/v0.13.0/api/v1_3.html", ) diff --git a/flexmeasures/api/v2_0/__init__.py b/flexmeasures/api/v2_0/__init__.py index c32eb09c6..3ee0746e7 100644 --- a/flexmeasures/api/v2_0/__init__.py +++ b/flexmeasures/api/v2_0/__init__.py @@ -1,14 +1,22 @@ from flask import Flask, Blueprint -from flexmeasures.api.common.utils.deprecation_utils import deprecate_blueprint +from flexmeasures.api.common.utils.deprecation_utils import ( + deprecate_blueprint, + sunset_blueprint, +) flexmeasures_api = Blueprint("flexmeasures_api_v2_0", __name__) deprecate_blueprint( flexmeasures_api, deprecation_date="2022-12-14", deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html", - sunset_date="2023-02-01", - sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html", + sunset_date="2023-05-01", + sunset_link="https://flexmeasures.readthedocs.io/en/v0.13.0/api/v2_0.html", +) +sunset_blueprint( + flexmeasures_api, + "2.0", + "https://flexmeasures.readthedocs.io/en/v0.13.0/api/v2_0.html", ) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 967bac318..aa2fef97d 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -130,6 +130,11 @@ class Config(object): # todo: expand with other js versions used in FlexMeasures ) + # Custom sunset switches + FLEXMEASURES_API_SUNSET_ACTIVE: bool = True # if True, sunset endpoints return 410 (Gone) responses; if False, they will work as before + FLEXMEASURES_API_SUNSET_DATE: str | None = None # e.g. 2023-05-01 + FLEXMEASURES_API_SUNSET_LINK: str | None = None # e.g. https://flexmeasures.readthedocs.io/en/latest/api/introduction.html#deprecation-and-sunset + # names of settings which cannot be None # SECRET_KEY is also required but utils.app_utils.set_secret_key takes care of this better. diff --git a/flexmeasures/utils/error_utils.py b/flexmeasures/utils/error_utils.py index 7882581f1..faa8544de 100644 --- a/flexmeasures/utils/error_utils.py +++ b/flexmeasures/utils/error_utils.py @@ -7,6 +7,7 @@ InternalServerError, BadRequest, NotFound, + Gone, ) from sqlalchemy.orm import Query @@ -70,7 +71,9 @@ def error_handling_router(error: HTTPException): error, "description", f"Something went wrong: {error.__class__.__name__}" ) - if request.is_json: + if request.is_json or ( + request.url_rule is not None and request.url_rule.rule.startswith("/api") + ): response = jsonify( dict( message=getattr(error, "description", str(error)), @@ -99,6 +102,7 @@ def add_basic_error_handlers(app: Flask): app.register_error_handler(BadRequest, error_handling_router) app.register_error_handler(HTTPException, error_handling_router) app.register_error_handler(NotFound, error_handling_router) + app.register_error_handler(Gone, error_handling_router) app.register_error_handler(Exception, error_handling_router)