Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add --custom-message param to flexmeasures monitor task CLI #489

Merged
merged 3 commits into from Aug 27, 2022
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
40 changes: 30 additions & 10 deletions flexmeasures/cli/monitor.py
Expand Up @@ -20,7 +20,10 @@ def fm_monitor():


def send_monitoring_alert(
task_name: str, msg: str, latest_run: Optional[LatestTaskRun] = None
task_name: str,
msg: str,
latest_run: Optional[LatestTaskRun] = None,
custom_msg: Optional[str] = None,
):
"""
Send any monitoring message per Sentry and per email. Also log an error.
Expand All @@ -34,18 +37,26 @@ def send_monitoring_alert(
f"Last run was at {latest_run.datetime}, status was: {latest_run.status}"
)

custom_msg_txt = ""
if custom_msg:
custom_msg_txt = f"\n\nNote: {custom_msg}"

capture_message_for_sentry(msg)

email_recipients = app.config.get("FLEXMEASURES_MONITORING_MAIL_RECIPIENTS", [])
if len(email_recipients) > 0:
email = Message(subject=f"Problem with task {task_name}", bcc=email_recipients)
email.body = f"{msg}\n\n{latest_run_txt}\nWe suggest to check the logs."
email.body = (
f"{msg}\n\n{latest_run_txt}\nWe suggest to check the logs. {custom_msg_txt}"
nhoening marked this conversation as resolved.
Show resolved Hide resolved
)
app.mail.send(email)

app.logger.error(f"{msg} {latest_run_txt}")
if custom_msg:
app.logger.info(custom_msg)
nhoening marked this conversation as resolved.
Show resolved Hide resolved


@fm_monitor.command("tasks")
@fm_monitor.command("tasks") # TODO: a better name would be "latest-run"
@with_appcontext
@click.option(
"--task",
Expand All @@ -54,7 +65,13 @@ def send_monitoring_alert(
required=True,
help="The name of the task and the maximal allowed minutes between successful runs. Use multiple times if needed.",
)
def monitor_tasks(task):
@click.option(
"--custom-message",
type=str,
default="",
help="Add this message to the monitoring alert (if one is sent).",
)
def monitor_tasks(task, custom_message):
"""
Check if the given task's last successful execution happened less than the allowed time ago.
If not, alert someone, via email or sentry.
Expand All @@ -65,7 +82,7 @@ def monitor_tasks(task):
latest_run: LatestTaskRun = LatestTaskRun.query.get(task_name)
if latest_run is None:
msg = f"Task {task_name} has no last run and thus cannot be monitored. Is it configured properly?"
send_monitoring_alert(task_name, msg)
send_monitoring_alert(task_name, msg, custom_msg=custom_message)
return
now = server_now()
acceptable_interval = timedelta(minutes=t[1])
Expand All @@ -74,14 +91,17 @@ def monitor_tasks(task):
# latest run time is okay, let's check the status
if latest_run.status is False:
msg = f"A failure has been reported on task {task_name}."
send_monitoring_alert(task_name, msg, latest_run)
send_monitoring_alert(
task_name, msg, latest_run=latest_run, custom_msg=custom_message
)
else:
msg = (
f"Task {task_name}'s latest run time is outside of the acceptable range "
f"({acceptable_interval})."
f"Task {task_name}'s latest run time is outside of the acceptable range"
f" ({acceptable_interval})."
)
send_monitoring_alert(
task_name, msg, latest_run=latest_run, custom_msg=custom_message
)
app.logger.error(msg)
nhoening marked this conversation as resolved.
Show resolved Hide resolved
send_monitoring_alert(task_name, msg, latest_run)
app.logger.info("Done checking task runs ...")


Expand Down