diff --git a/.gitignore b/.gitignore index 169f946a7..dd173e77f 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ db_schema.png .coverage htmlcov +test/* +profile_reports/* diff --git a/documentation/changelog.rst b/documentation/changelog.rst index c2c485c92..1aa34023a 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,6 +17,8 @@ Bugfixes Infrastructure / Support ---------------------- +* Add support for profiling Flask API calls using ``pyinstrument`` (if installed). Can be enabled by setting the environment variable ``FLEXMEASURES_PROFILE_REQUESTS`` to ``True`` [see `PR #722 `_] + v0.14.1 | June XX, 2023 ============================ diff --git a/documentation/configuration.rst b/documentation/configuration.rst index cdb1769cc..899a06d98 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -98,7 +98,15 @@ Default: ``"migrations/dumps"`` FLEXMEASURES_PROFILE_REQUESTS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Whether to turn on a feature which times requests made through FlexMeasures. Interesting for developers. +If True, the processing time of requests are profiled. + +The overall time used by requests are logged to the console. In addiition, if `pyinstrument` is installed, then a profiling report is made (of time being spent in different function calls) for all Flask API endpoints. + +The profiling results are stored in the ``profile_reports`` folder in the instance directory. + +Note: Profile reports for API endpoints are overwritten on repetition of the same request. + +Interesting for developers. Default: ``False`` diff --git a/flexmeasures/app.py b/flexmeasures/app.py index e9c2a5554..051600508 100644 --- a/flexmeasures/app.py +++ b/flexmeasures/app.py @@ -5,6 +5,9 @@ from __future__ import annotations import time +import os +from pathlib import Path +from datetime import date from flask import Flask, g, request from flask.cli import load_dotenv @@ -97,6 +100,17 @@ def create( # noqa C901 if app.env not in ("documentation", "development"): SSLify(app) + # Prepare profiling, if needed + + if app.config.get("FLEXMEASURES_PROFILE_REQUESTS", False): + Path("profile_reports").mkdir(parents=True, exist_ok=True) + try: + import pyinstrument # noqa F401 + except ImportError: + app.logger.warning( + "FLEXMEASURES_PROFILE_REQUESTS is True, but pyinstrument not installed ― I cannot produce profiling reports for requests." + ) + # Register database and models, including user auth security handlers from flexmeasures.data import register_at as register_db_at @@ -150,6 +164,13 @@ def create( # noqa C901 def before_request(): if app.config.get("FLEXMEASURES_PROFILE_REQUESTS", False): g.start = time.time() + try: + import pyinstrument # noqa F401 + + g.profiler = pyinstrument.Profiler() + g.profiler.start() + except ImportError: + pass @app.teardown_request def teardown_request(exception=None): @@ -159,5 +180,22 @@ def teardown_request(exception=None): app.logger.info( f"[PROFILE] {str(round(diff, 2)).rjust(6)} seconds to serve {request.url}." ) + if not hasattr(g, "profiler"): + return app + g.profiler.stop() + output_html = g.profiler.output_html(timeline=True) + endpoint = request.endpoint + if endpoint is None: + endpoint = "unknown" + today = date.today() + profile_filename = f"pyinstrument_{endpoint}.html" + profile_output_path = Path( + "profile_reports", today.strftime("%Y-%m-%d") + ) + profile_output_path.mkdir(parents=True, exist_ok=True) + with open( + os.path.join(profile_output_path, profile_filename), "w+" + ) as f: + f.write(output_html) return app diff --git a/requirements/app.txt b/requirements/app.txt index c5aa68400..3b4170198 100644 --- a/requirements/app.txt +++ b/requirements/app.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/app.txt requirements/app.in # diff --git a/requirements/dev.in b/requirements/dev.in index e1f392c32..63c71088b 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -10,4 +10,5 @@ flake8-blind-except mypy pytest-runner setuptools_scm -watchdog \ No newline at end of file +watchdog +pyinstrument \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt index 6e019a511..625d3f8a8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/dev.txt requirements/dev.in # @@ -52,6 +52,8 @@ pycodestyle==2.8.0 # via flake8 pyflakes==2.4.0 # via flake8 +pyinstrument==4.5.0 + # via -r requirements/dev.in pytest-runner==6.0.0 # via -r requirements/dev.in pyyaml==6.0 diff --git a/requirements/docs.txt b/requirements/docs.txt index f185451c4..baa53083b 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/docs.txt requirements/docs.in # diff --git a/requirements/test.txt b/requirements/test.txt index 827b7d973..edd5791ca 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/test.txt requirements/test.in #