Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ref(rules): Fire delayed rules #69830

Merged
merged 14 commits into from
May 2, 2024
52 changes: 31 additions & 21 deletions src/sentry/rules/processing/delayed_processing.py
Copy link
Member

@schew2381 schew2381 May 1, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just wanna double check this, but do we need to worry about snoozed rules at all? I realize that we only add to the buffer if the rule is not snoozed in the first place, and at worst we have a minute delay so theoretically an action could be fired in a tiny bit >1min after a snooze.

Is it worth considering doing another snooze check in delayed processing?

snoozed_rules = RuleSnooze.objects.filter(rule__in=rules, user_id=None).values_list(
"rule", flat=True
)
rule_statuses = bulk_get_rule_status(rules, self.group)
for rule in rules:
if rule.id not in snoozed_rules:
self.apply_rule(rule, rule_statuses[rule.id])

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems pretty minor if we fire a rule slightly after it's snoozed, but probably makes sense to recheck. I'd put that in a separate pr to keep this simple

Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from sentry.models.grouprulestatus import GroupRuleStatus
from sentry.models.project import Project
from sentry.models.rule import Rule
from sentry.models.rulesnooze import RuleSnooze
from sentry.rules import history, rules
from sentry.rules.conditions.event_frequency import (
BaseEventFrequencyCondition,
Expand Down Expand Up @@ -276,28 +277,37 @@ def apply_delayed(project_id: int) -> None:
)
# Step 7: Fire the rule's actions
now = timezone.now()
snoozed_rules = RuleSnooze.objects.filter(
rule__in=[rule for rule in rules_to_fire.keys()], user_id=None
ceorourke marked this conversation as resolved.
Show resolved Hide resolved
).values_list("rule", flat=True)
for rule, group_ids in rules_to_fire.items():
frequency = rule.data.get("frequency") or Rule.DEFAULT_FREQUENCY
freq_offset = now - timedelta(minutes=frequency)
group_to_groupevent = get_group_to_groupevent(rulegroup_to_events, project.id, group_ids)
for group, groupevent in group_to_groupevent.items():
rule_statuses = bulk_get_rule_status(alert_rules, group, project)
status = rule_statuses[rule.id]
if status.last_active and status.last_active > freq_offset:
return

updated = (
GroupRuleStatus.objects.filter(id=status.id)
.exclude(last_active__gt=freq_offset)
.update(last_active=now)
if rule.id not in snoozed_rules:
ceorourke marked this conversation as resolved.
Show resolved Hide resolved
frequency = rule.data.get("frequency") or Rule.DEFAULT_FREQUENCY
freq_offset = now - timedelta(minutes=frequency)
group_to_groupevent = get_group_to_groupevent(
rulegroup_to_events, project.id, group_ids
)
for group, groupevent in group_to_groupevent.items():
rule_statuses = bulk_get_rule_status(alert_rules, group, project)
status = rule_statuses[rule.id]
if status.last_active and status.last_active > freq_offset:
return

updated = (
GroupRuleStatus.objects.filter(id=status.id)
.exclude(last_active__gt=freq_offset)
.update(last_active=now)
)

if not updated:
return
if not updated:
return

notification_uuid = str(uuid.uuid4())
groupevent = group_to_groupevent[group]
rule_fire_history = history.record(rule, group, groupevent.event_id, notification_uuid)
safe_execute(
activate_downstream_actions, rule, groupevent, notification_uuid, rule_fire_history
)
notification_uuid = str(uuid.uuid4())
groupevent = group_to_groupevent[group]
rule_fire_history = history.record(
rule, group, groupevent.event_id, notification_uuid
)
for callback, futures in activate_downstream_actions(
rule, groupevent, notification_uuid, rule_fire_history
).values():
safe_execute(callback, groupevent, futures, _with_transaction=False)
46 changes: 44 additions & 2 deletions tests/sentry/rules/processing/test_delayed_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
apply_delayed,
process_delayed_alert_conditions,
)
from sentry.testutils.cases import APITestCase, TestCase
from sentry.testutils.cases import APITestCase, PerformanceIssueTestCase, TestCase
from sentry.testutils.factories import DEFAULT_EVENT_DATA
from sentry.testutils.helpers.datetime import iso_format
from sentry.utils import json
Expand All @@ -23,7 +23,9 @@
pytestmark = pytest.mark.sentry_metrics


class ProcessDelayedAlertConditionsTest(TestCase, APITestCase, BaseEventFrequencyPercentTest):
class ProcessDelayedAlertConditionsTest(
TestCase, APITestCase, BaseEventFrequencyPercentTest, PerformanceIssueTestCase
):
def create_event(
self, project_id, timestamp, fingerprint, environment=None, user: bool = True
) -> Event:
Expand Down Expand Up @@ -197,6 +199,46 @@ def test_apply_delayed_rules_to_fire(self):
assert (self.rule3.id, self.group3.id) in rule_fire_histories
assert (self.rule4.id, self.group4.id) in rule_fire_histories

def test_apply_delayed_issue_platform_event(self):
"""
Test that we fire rules triggered from issue platform events
"""
rule5 = self.create_project_rule(
project=self.project,
condition_match=[self.event_frequency_condition2],
environment_id=self.environment.id,
)
tags = [["foo", "guux"], ["sentry:release", "releaseme"]]
contexts = {"trace": {"trace_id": "b" * 32, "span_id": "c" * 16, "op": ""}}
for i in range(3):
event5 = self.create_performance_issue(
tags=tags,
fingerprint="group-5",
contexts=contexts,
)
group5 = event5.group
assert group5
assert self.group1
self.push_to_hash(
self.project.id,
rule5.id,
group5.id,
event5.event_id,
occurrence_id=event5.occurrence_id,
)
with patch("sentry.buffer.backend.get_hash", self.redis_buffer.get_hash):
apply_delayed(self.project.id)
rule_fire_histories = RuleFireHistory.objects.filter(
rule__in=[self.rule1, rule5],
group__in=[self.group1, group5],
event_id__in=[self.event1.event_id, event5.event_id],
project=self.project,
).values_list("rule", "group")
assert len(rule_fire_histories) == 3
assert (self.rule1.id, self.group1.id) in rule_fire_histories
assert (self.rule1.id, group5.id) in rule_fire_histories
assert (rule5.id, group5.id) in rule_fire_histories

def test_apply_delayed_same_condition_diff_value(self):
"""
Test that two rules with the same condition and interval but a different value are both fired
Expand Down