Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add --custom-message param to flexmeasures monitor task CLI #489

Merged
merged 3 commits into from Aug 27, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions documentation/changelog.rst
Expand Up @@ -32,6 +32,7 @@ Infrastructure / Support
* Queue workers can get initialised without a custom name and name collisions are handled [see `PR #455 <http://www.github.com/FlexMeasures/flexmeasures/pull/455>`_]
* New API endpoint to get public assets [see `PR #461 <http://www.github.com/FlexMeasures/flexmeasures/pull/461>`_]
* Allow editing an asset's JSON attributes through the UI [see `PR #474 <http://www.github.com/FlexMeasures/flexmeasures/pull/474>`_]
* Allow a custom message when monitoring latest run of tasks [see `PR #489 <http://www.github.com/FlexMeasures/flexmeasures/pull/489>`_]


v0.10.1 | August 12, 2022
Expand Down
40 changes: 29 additions & 11 deletions flexmeasures/cli/monitor.py
Expand Up @@ -20,7 +20,10 @@ def fm_monitor():


def send_monitoring_alert(
task_name: str, msg: str, latest_run: Optional[LatestTaskRun] = None
task_name: str,
msg: str,
latest_run: Optional[LatestTaskRun] = None,
custom_msg: Optional[str] = None,
):
"""
Send any monitoring message per Sentry and per email. Also log an error.
Expand All @@ -34,18 +37,24 @@ def send_monitoring_alert(
f"Last run was at {latest_run.datetime}, status was: {latest_run.status}"
)

custom_msg_txt = ""
if custom_msg:
custom_msg_txt = f"\n\nNote: {custom_msg}"

capture_message_for_sentry(msg)

email_recipients = app.config.get("FLEXMEASURES_MONITORING_MAIL_RECIPIENTS", [])
if len(email_recipients) > 0:
email = Message(subject=f"Problem with task {task_name}", bcc=email_recipients)
email.body = f"{msg}\n\n{latest_run_txt}\nWe suggest to check the logs."
email.body = (
f"{msg}\n\n{latest_run_txt}\nWe suggest to check the logs.{custom_msg_txt}"
)
app.mail.send(email)

app.logger.error(f"{msg} {latest_run_txt}")
app.logger.error(f"{msg} {latest_run_txt} NOTE: {custom_msg}")


@fm_monitor.command("tasks")
@fm_monitor.command("tasks") # TODO: a better name would be "latest-run"
@with_appcontext
@click.option(
"--task",
Expand All @@ -54,7 +63,13 @@ def send_monitoring_alert(
required=True,
help="The name of the task and the maximal allowed minutes between successful runs. Use multiple times if needed.",
)
def monitor_tasks(task):
@click.option(
"--custom-message",
type=str,
default="",
help="Add this message to the monitoring alert (if one is sent).",
)
def monitor_tasks(task, custom_message):
"""
Check if the given task's last successful execution happened less than the allowed time ago.
If not, alert someone, via email or sentry.
Expand All @@ -65,7 +80,7 @@ def monitor_tasks(task):
latest_run: LatestTaskRun = LatestTaskRun.query.get(task_name)
if latest_run is None:
msg = f"Task {task_name} has no last run and thus cannot be monitored. Is it configured properly?"
send_monitoring_alert(task_name, msg)
send_monitoring_alert(task_name, msg, custom_msg=custom_message)
return
now = server_now()
acceptable_interval = timedelta(minutes=t[1])
Expand All @@ -74,14 +89,17 @@ def monitor_tasks(task):
# latest run time is okay, let's check the status
if latest_run.status is False:
msg = f"A failure has been reported on task {task_name}."
send_monitoring_alert(task_name, msg, latest_run)
send_monitoring_alert(
task_name, msg, latest_run=latest_run, custom_msg=custom_message
)
else:
msg = (
f"Task {task_name}'s latest run time is outside of the acceptable range "
f"({acceptable_interval})."
f"Task {task_name}'s latest run time is outside of the acceptable range"
f" ({acceptable_interval})."
)
send_monitoring_alert(
task_name, msg, latest_run=latest_run, custom_msg=custom_message
)
app.logger.error(msg)
nhoening marked this conversation as resolved.
Show resolved Hide resolved
send_monitoring_alert(task_name, msg, latest_run)
app.logger.info("Done checking task runs ...")


Expand Down