feat: add pyinstrument integration to Flask API endpoints

.gitignore

@@ -37,3 +37,5 @@ db_schema.png
 
 .coverage
 htmlcov
+test/*
+profile_reports/*

documentation/changelog.rst

@@ -17,6 +17,8 @@ Bugfixes
 Infrastructure / Support
 ----------------------
 
+* Add support for profiling Flask API calls using ``pyinstrument`` (if installed). Can be enabled by setting the environment variable ``FLEXMEASURES_PROFILE_REQUESTS`` to ``True`` [see `PR #722 <https://www.github.com/FlexMeasures/flexmeasures/pull/722>`_]
+
 
 v0.14.1 | June XX, 2023
 ============================

documentation/configuration.rst

@@ -98,7 +98,15 @@ Default: ``"migrations/dumps"``
 FLEXMEASURES_PROFILE_REQUESTS
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Whether to turn on a feature which times requests made through FlexMeasures. Interesting for developers.
+If True, the processing time of requests are profiled.
+
+The overall time used by requests are logged to the console. In addiition, if `pyinstrument` is installed, then a profiling report is made (of time being spent in different function calls) for all Flask API endpoints.
+
+The profiling results are stored in the ``profile_reports`` folder in the instance directory.
+
+Note: Profile reports for API endpoints are overwritten on repetition of the same request.
+
+Interesting for developers.
 
 Default: ``False``
 

flexmeasures/app.py

@@ -5,6 +5,9 @@
 from __future__ import annotations
 
 import time
+import os
+from pathlib import Path
+from datetime import date
 
 from flask import Flask, g, request
 from flask.cli import load_dotenv
@@ -97,6 +100,17 @@ def create(  # noqa C901
     if app.env not in ("documentation", "development"):
         SSLify(app)
 
+    # Prepare profiling, if needed
+
+    if app.config.get("FLEXMEASURES_PROFILE_REQUESTS", False):
+        Path("profile_reports").mkdir(parents=True, exist_ok=True)
+        try:
+            import pyinstrument  # noqa F401
+        except ImportError:
+            app.logger.warning(
+                "FLEXMEASURES_PROFILE_REQUESTS is True, but pyinstrument not installed ― I cannot produce profiling reports for requests."
+            )
+
     # Register database and models, including user auth security handlers
 
     from flexmeasures.data import register_at as register_db_at
@@ -150,6 +164,13 @@ def create(  # noqa C901
     def before_request():
         if app.config.get("FLEXMEASURES_PROFILE_REQUESTS", False):
             g.start = time.time()
+            try:
+                import pyinstrument  # noqa F401
+
+                g.profiler = pyinstrument.Profiler()
+                g.profiler.start()
+            except ImportError:
+                pass
 
     @app.teardown_request
     def teardown_request(exception=None):
@@ -159,5 +180,22 @@ def teardown_request(exception=None):
                 app.logger.info(
                     f"[PROFILE] {str(round(diff, 2)).rjust(6)} seconds to serve {request.url}."
                 )
+                if not hasattr(g, "profiler"):
+                    return app
+                g.profiler.stop()
+                output_html = g.profiler.output_html(timeline=True)
+                endpoint = request.endpoint
+                if endpoint is None:
+                    endpoint = "unknown"
+                today = date.today()
+                profile_filename = f"pyinstrument_{endpoint}.html"
+                profile_output_path = Path(
+                    "profile_reports", today.strftime("%Y-%m-%d")
+                )
+                profile_output_path.mkdir(parents=True, exist_ok=True)
+                with open(
+                    os.path.join(profile_output_path, profile_filename), "w+"
+                ) as f:
+                    f.write(output_html)
 
     return app

requirements/app.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
 #
 #    pip-compile --output-file=requirements/app.txt requirements/app.in
 #

requirements/dev.in

@@ -10,4 +10,5 @@ flake8-blind-except
 mypy
 pytest-runner
 setuptools_scm
-watchdog
+watchdog
+pyinstrument

requirements/dev.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
 #
 #    pip-compile --output-file=requirements/dev.txt requirements/dev.in
 #
@@ -52,6 +52,8 @@ pycodestyle==2.8.0
     # via flake8
 pyflakes==2.4.0
     # via flake8
+pyinstrument==4.5.0
+    # via -r requirements/dev.in
 pytest-runner==6.0.0
     # via -r requirements/dev.in
 pyyaml==6.0

requirements/docs.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
 #
 #    pip-compile --output-file=requirements/docs.txt requirements/docs.in
 #

requirements/test.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
 #
 #    pip-compile --output-file=requirements/test.txt requirements/test.in
 #