From cb3de7cad9e03e0eb78106a98f2ad79de28b120f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Mon, 27 Sep 2021 15:14:25 +0200 Subject: [PATCH] allow the @task_with_status_report decorator to accept a name to be used in monitoring (#193) * allow the latest_task_run_monitor to accept a name * changelog entry * Grammar fix Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> --- documentation/changelog.rst | 2 +- documentation/dev/error-monitoring.rst | 15 ++++++++++- flexmeasures/data/transactional.py | 24 +++++++++++------ flexmeasures/utils/coding_utils.py | 37 ++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 02a5933cb..f5f715450 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -11,7 +11,6 @@ New features ----------- * Set a logo for the top left corner with the new FLEXMEASURES_MENU_LOGO_PATH setting [see `PR #184 `_] * Add an extra style-sheet which applies to all pages with the new FLEXMEASURES_EXTRA_CSS_PATH setting [see `PR #185 `_] - Bugfixes ----------- @@ -21,6 +20,7 @@ Infrastructure / Support ---------------------- * FlexMeasures plugins can be Python packages now. We provide `a cookie-cutter template `_ for this approach. [see `PR #182 `_] * Set default timezone for new users using the FLEXMEASURES_TIMEZONE config setting [see `PR #190 `_] +* Monitored CLI tasks can get better names for identification [see `PR #193 `_] v0.6.1 | September XX, 2021 diff --git a/documentation/dev/error-monitoring.rst b/documentation/dev/error-monitoring.rst index 25941920c..94424b503 100644 --- a/documentation/dev/error-monitoring.rst +++ b/documentation/dev/error-monitoring.rst @@ -21,5 +21,18 @@ This task status monitoring is enabled by decorating the functions behind these .. code-block:: python @task_with_status_report + def my_function(): + ... + +Then, FlexMeasures will log if this task ran, and if it succeeded or failed. The result is in the table ``latest_task_runs``, and that's where the ``flexmeasures monitor tasks`` will look. + +.. note:: The decorator should be placed right before the function (after all other decorators). + +Per default the function name is used as task name. If the number of tasks accumulate (e.g. by using multiple plugins that each define a task or two), it is useful to come up with more dedicated names. You can add a custom name as argument to the decorator: + +.. code-block:: python + + @task_with_status_report("pluginA_myFunction") + def my_function(): + ... -Then, FlexMeasures will log if this task ran, and if it succeeded or failed. \ No newline at end of file diff --git a/flexmeasures/data/transactional.py b/flexmeasures/data/transactional.py index adbade62c..db2c3c7a1 100644 --- a/flexmeasures/data/transactional.py +++ b/flexmeasures/data/transactional.py @@ -4,6 +4,7 @@ """ import sys from datetime import datetime +from typing import Optional import pytz import click @@ -12,6 +13,7 @@ from flexmeasures.data.config import db from flexmeasures.utils.error_utils import get_err_source_info +from flexmeasures.utils.coding_utils import optional_arg_decorator from flexmeasures.data.models.task_runs import LatestTaskRun @@ -75,7 +77,8 @@ class PartialTaskCompletionException(Exception): pass -def task_with_status_report(task_function): +@optional_arg_decorator +def task_with_status_report(task_function, task_name: Optional[str] = None): """Decorator for tasks which should report their runtime and status in the db (as LatestTaskRun entries). Tasks decorated with this endpoint should also leave committing or rolling back the session to this decorator (for the reasons that it is nice to centralise that but also practically, this decorator @@ -84,18 +87,24 @@ def task_with_status_report(task_function): it can raise a PartialTaskCompletionException and we recommend to use save-points (db.session.being_nested) to do partial rollbacks (see https://docs.sqlalchemy.org/en/latest/orm/session_transaction.html#using-savepoint).""" + task_name_to_report = ( + task_name # store this closure var somewhere else before we might assign to it + ) + if task_name_to_report is None: + task_name_to_report = task_function.__name__ + def wrap(*args, **kwargs): status: bool = True partial: bool = False try: task_function(*args, **kwargs) - click.echo("[FLEXMEASURES] Task %s ran fine." % task_function.__name__) + click.echo("[FLEXMEASURES] Task %s ran fine." % task_name_to_report) except Exception as e: exc_info = sys.exc_info() last_traceback = exc_info[2] click.echo( '[FLEXMEASURES] Task %s encountered a problem: "%s". More details: %s' - % (task_function.__name__, str(e), get_err_source_info(last_traceback)) + % (task_name_to_report, str(e), get_err_source_info(last_traceback)) ) status = False if e.__class__ == PartialTaskCompletionException: @@ -108,24 +117,23 @@ def wrap(*args, **kwargs): # now save the status of the task db.session.begin_nested() # any failure here does not invalidate any task results we might commit try: - task_name = task_function.__name__ task_run = LatestTaskRun.query.filter( - LatestTaskRun.name == task_name + LatestTaskRun.name == task_name_to_report ).one_or_none() if task_run is None: - task_run = LatestTaskRun(name=task_name) + task_run = LatestTaskRun(name=task_name_to_report) db.session.add(task_run) task_run.datetime = datetime.utcnow().replace(tzinfo=pytz.utc) task_run.status = status click.echo( "[FLEXMEASURES] Reported task %s status as %s" - % (task_function.__name__, status) + % (task_name_to_report, status) ) db.session.commit() except Exception as e: click.echo( "[FLEXMEASURES] Could not report the running of task %s. Encountered the following problem: [%s]." - " The task might have run fine." % (task_function.__name__, str(e)) + " The task might have run fine." % (task_name_to_report, str(e)) ) db.session.rollback() diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index ec4da4f05..4c75c9c02 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -74,6 +74,43 @@ def _getattr(obj, attr): return functools.reduce(_getattr, [obj] + attr.split(".")) +def optional_arg_decorator(fn): + """ + A decorator which _optionally_ accepts arguments. + + So a decorator like this: + + @optional_arg_decorator + def register_something(fn, optional_arg = 'Default Value'): + ... + return fn + + will work in both of these usage scenarios: + + @register_something('Custom Name') + def custom_name(): + pass + + @register_something + def default_name(): + pass + + Thanks to https://stackoverflow.com/questions/3888158/making-decorators-with-optional-arguments#comment65959042_24617244 + """ + + def wrapped_decorator(*args): + if len(args) == 1 and callable(args[0]): + return fn(args[0]) + else: + + def real_decorator(decoratee): + return fn(decoratee, *args) + + return real_decorator + + return wrapped_decorator + + def sort_dict(unsorted_dict: dict) -> dict: sorted_dict = dict(sorted(unsorted_dict.items(), key=lambda item: item[0])) return sorted_dict