Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ref(snuba): Drop bulk_snql_query #69402

Merged
merged 1 commit into from Apr 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/sentry/api/endpoints/organization_events_trace.py
Expand Up @@ -31,7 +31,7 @@
from sentry.utils.iterators import chunked
from sentry.utils.numbers import base32_encode, format_grouped_length
from sentry.utils.sdk import set_measurement
from sentry.utils.snuba import bulk_snql_query
from sentry.utils.snuba import bulk_snuba_queries
from sentry.utils.validators import INVALID_ID_DETAILS, is_event_id

logger: logging.Logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -565,7 +565,7 @@ def query_trace_data(
auto_fields=False,
),
)
results = bulk_snql_query(
results = bulk_snuba_queries(
[
transaction_query.get_snql_query(),
error_query.get_snql_query(),
Expand Down Expand Up @@ -744,7 +744,7 @@ def augment_transactions_with_spans(
sentry_sdk.set_measurement("trace_view.span_query.total_chunks", total_chunks)
chunks = chunked(list_spans, (len(list_spans) // total_chunks) + 1)
queries = [build_span_query(trace_id, spans_params, chunk) for chunk in chunks]
results = bulk_snql_query(
results = bulk_snuba_queries(
[query.get_snql_query() for query in queries],
referrer=Referrer.API_TRACE_VIEW_GET_PARENTS.value,
)
Expand Down
4 changes: 2 additions & 2 deletions src/sentry/api/endpoints/organization_profiling_functions.py
Expand Up @@ -25,7 +25,7 @@
from sentry.snuba.referrer import Referrer
from sentry.utils.dates import parse_stats_period, validate_interval
from sentry.utils.sdk import set_measurement
from sentry.utils.snuba import bulk_snql_query
from sentry.utils.snuba import bulk_snuba_queries

TOP_FUNCTIONS_LIMIT = 50
FUNCTIONS_PER_QUERY = 10
Expand Down Expand Up @@ -138,7 +138,7 @@ def get_event_stats(_columns, query, params, _rollup, zerofill_results, _compari
)
for chunk in chunks
]
bulk_results = bulk_snql_query(
bulk_results = bulk_snuba_queries(
[builder.get_snql_query() for builder in builders],
Referrer.API_PROFILING_FUNCTION_TRENDS_STATS.value,
)
Expand Down
4 changes: 2 additions & 2 deletions src/sentry/api/endpoints/organization_traces.py
Expand Up @@ -23,7 +23,7 @@
from sentry.snuba.dataset import Dataset
from sentry.snuba.referrer import Referrer
from sentry.utils.numbers import clip
from sentry.utils.snuba import bulk_snql_query
from sentry.utils.snuba import bulk_snuba_queries


class TraceInterval(TypedDict):
Expand Down Expand Up @@ -225,7 +225,7 @@ def data_fn(offset: int, limit: int):
query.add_conditions([trace_id_condition])

with handle_query_errors():
results = bulk_snql_query(
results = bulk_snuba_queries(
[query.get_snql_query() for query in queries],
Referrer.API_TRACE_EXPLORER_TRACES_META.value,
)
Expand Down
6 changes: 3 additions & 3 deletions src/sentry/search/events/builder/metrics.py
Expand Up @@ -72,7 +72,7 @@
MetricsQuery,
)
from sentry.snuba.metrics.utils import get_num_intervals
from sentry.utils.snuba import DATASETS, bulk_snql_query, raw_snql_query
from sentry.utils.snuba import DATASETS, bulk_snuba_queries, raw_snql_query


class MetricsQueryBuilder(QueryBuilder):
Expand Down Expand Up @@ -1636,7 +1636,7 @@ def run_query(self, referrer: str, use_cache: bool = False) -> Any:

queries = self.get_snql_query()
if queries:
results = bulk_snql_query(queries, referrer, use_cache)
results = bulk_snuba_queries(queries, referrer, use_cache)
else:
results = []

Expand Down Expand Up @@ -1910,7 +1910,7 @@ def run_query(self, referrer: str, use_cache: bool = False) -> Any:
else:
queries = self.get_snql_query()
if queries:
results = bulk_snql_query(queries, referrer, use_cache)
results = bulk_snuba_queries(queries, referrer, use_cache)

time_map: dict[str, dict[str, Any]] = defaultdict(dict)
for current_result in results:
Expand Down
2 changes: 1 addition & 1 deletion src/sentry/search/snuba/executors.py
Expand Up @@ -1457,7 +1457,7 @@ def query(
)
queries.append(request)

bulk_result = snuba.bulk_snql_query(
bulk_result = snuba.bulk_snuba_queries(
queries, referrer="search.snuba.group_attributes_search.query"
)

Expand Down
8 changes: 5 additions & 3 deletions src/sentry/snuba/discover.py
Expand Up @@ -32,7 +32,7 @@
from sentry.utils.math import nice_int
from sentry.utils.snuba import (
SnubaTSResult,
bulk_snql_query,
bulk_snuba_queries,
get_array_column_alias,
get_array_column_field,
get_measurement_name,
Expand Down Expand Up @@ -335,7 +335,9 @@ def timeseries_query(
)
query_list.append(comparison_builder)

query_results = bulk_snql_query([query.get_snql_query() for query in query_list], referrer)
query_results = bulk_snuba_queries(
[query.get_snql_query() for query in query_list], referrer
)

with sentry_sdk.start_span(op="discover.discover", description="timeseries.transform_results"):
results = []
Expand Down Expand Up @@ -495,7 +497,7 @@ def top_events_timeseries(
timeseries_columns=timeseries_columns,
equations=equations,
)
result, other_result = bulk_snql_query(
result, other_result = bulk_snuba_queries(
[top_events_builder.get_snql_query(), other_events_builder.get_snql_query()],
referrer=referrer,
)
Expand Down
28 changes: 16 additions & 12 deletions src/sentry/snuba/errors.py
Expand Up @@ -14,7 +14,7 @@
from sentry.snuba.dataset import Dataset
from sentry.snuba.discover import EventsResponse, transform_tips, zerofill
from sentry.snuba.metrics.extraction import MetricSpecType
from sentry.utils.snuba import SnubaTSResult, bulk_snql_query
from sentry.utils.snuba import SnubaTSResult, bulk_snuba_queries

is_filter_translation = {}
for status_key, status_value in STATUS_QUERY_CHOICES.items():
Expand Down Expand Up @@ -128,23 +128,27 @@ def timeseries_query(
)
query_list.append(comparison_builder)

query_results = bulk_snql_query([query.get_snql_query() for query in query_list], referrer)
query_results = bulk_snuba_queries(
[query.get_snql_query() for query in query_list], referrer
)

with sentry_sdk.start_span(op="errors", description="timeseries.transform_results"):
results = []
for snql_query, result in zip(query_list, query_results):
results.append(
{
"data": zerofill(
result["data"],
snql_query.params.start,
snql_query.params.end,
rollup,
"time",
time_col_name="events.time",
)
if zerofill_results
else result["data"],
"data": (
zerofill(
result["data"],
snql_query.params.start,
snql_query.params.end,
rollup,
"time",
time_col_name="events.time",
)
if zerofill_results
else result["data"]
),
"meta": result["meta"],
}
)
Expand Down
26 changes: 15 additions & 11 deletions src/sentry/snuba/issue_platform.py
Expand Up @@ -12,7 +12,7 @@
from sentry.snuba.dataset import Dataset
from sentry.snuba.discover import EventsResponse, transform_tips, zerofill
from sentry.snuba.metrics.extraction import MetricSpecType
from sentry.utils.snuba import SnubaTSResult, bulk_snql_query
from sentry.utils.snuba import SnubaTSResult, bulk_snuba_queries


def query(
Expand Down Expand Up @@ -172,22 +172,26 @@ def timeseries_query(
)
query_list.append(comparison_builder)

query_results = bulk_snql_query([query.get_snql_query() for query in query_list], referrer)
query_results = bulk_snuba_queries(
[query.get_snql_query() for query in query_list], referrer
)

with sentry_sdk.start_span(op="issueplatform", description="timeseries.transform_results"):
results = []
for snql_query, result in zip(query_list, query_results):
results.append(
{
"data": zerofill(
result["data"],
snql_query.params.start,
snql_query.params.end,
rollup,
"time",
)
if zerofill_results
else result["data"],
"data": (
zerofill(
result["data"],
snql_query.params.start,
snql_query.params.end,
rollup,
"time",
)
if zerofill_results
else result["data"]
),
"meta": result["meta"],
}
)
Expand Down
20 changes: 11 additions & 9 deletions src/sentry/snuba/metrics/datasource.py
Expand Up @@ -89,7 +89,7 @@
get_intervals,
to_intervals,
)
from sentry.utils.snuba import bulk_snql_query, raw_snql_query
from sentry.utils.snuba import bulk_snuba_queries, raw_snql_query

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -409,7 +409,7 @@ def get_stored_metrics_of_projects(
use_case_id_to_index[use_case_id].append(len(requests) - 1)

# We run the queries all in parallel.
results = bulk_snql_query(
results = bulk_snuba_queries(
requests=requests,
referrer="snuba.metrics.datasource.get_stored_metrics_of_projects",
use_cache=True,
Expand Down Expand Up @@ -1228,11 +1228,13 @@ def get_series(
"end": metrics_query.end,
"intervals": intervals,
"groups": result_groups,
"meta": translate_meta_results(
meta=meta,
alias_to_metric_field=converter._alias_to_metric_field,
alias_to_metric_group_by_field=groupby_aliases,
)
if include_meta
else [],
"meta": (
translate_meta_results(
meta=meta,
alias_to_metric_field=converter._alias_to_metric_field,
alias_to_metric_group_by_field=groupby_aliases,
)
if include_meta
else []
),
}
40 changes: 23 additions & 17 deletions src/sentry/snuba/metrics_performance.py
Expand Up @@ -21,7 +21,7 @@
from sentry.snuba import discover
from sentry.snuba.dataset import Dataset
from sentry.snuba.metrics.extraction import MetricSpecType
from sentry.utils.snuba import SnubaTSResult, bulk_snql_query
from sentry.utils.snuba import SnubaTSResult, bulk_snuba_queries

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -177,7 +177,7 @@ def bulk_timeseries_query(
metrics_queries.append(snql_query[0])

metrics_referrer = referrer + ".metrics-enhanced"
bulk_result = bulk_snql_query(metrics_queries, metrics_referrer)
bulk_result = bulk_snuba_queries(metrics_queries, metrics_referrer)
_result: dict[str, Any] = {"data": []}
for br in bulk_result:
_result["data"] = [*_result["data"], *br["data"]]
Expand Down Expand Up @@ -221,9 +221,11 @@ def bulk_timeseries_query(
)
return SnubaTSResult(
{
"data": discover.zerofill([], params["start"], params["end"], rollup, "time")
if zerofill_results
else [],
"data": (
discover.zerofill([], params["start"], params["end"], rollup, "time")
if zerofill_results
else []
),
},
params["start"],
params["end"],
Expand Down Expand Up @@ -360,9 +362,11 @@ def run_metrics_query(inner_params: ParamsType):
# In case the query was not compatible with metrics we return empty data.
return SnubaTSResult(
{
"data": discover.zerofill([], params["start"], params["end"], rollup, "time")
if zerofill_results
else [],
"data": (
discover.zerofill([], params["start"], params["end"], rollup, "time")
if zerofill_results
else []
),
},
params["start"],
params["end"],
Expand Down Expand Up @@ -435,7 +439,7 @@ def top_events_timeseries(
),
)

# TODO: use bulk_snql_query
# TODO: use bulk_snuba_queries
other_result = other_events_builder.run_query(referrer)
result = top_events_builder.run_query(referrer)
else:
Expand All @@ -448,9 +452,11 @@ def top_events_timeseries(
):
return SnubaTSResult(
{
"data": discover.zerofill([], params["start"], params["end"], rollup, "time")
if zerofill_results
else [],
"data": (
discover.zerofill([], params["start"], params["end"], rollup, "time")
if zerofill_results
else []
),
},
params["start"],
params["end"],
Expand Down Expand Up @@ -482,11 +488,11 @@ def top_events_timeseries(
for key, item in results.items():
results[key] = SnubaTSResult(
{
"data": discover.zerofill(
item["data"], params["start"], params["end"], rollup, "time"
)
if zerofill_results
else item["data"],
"data": (
discover.zerofill(item["data"], params["start"], params["end"], rollup, "time")
if zerofill_results
else item["data"]
),
"order": item["order"],
# One of the queries in the builder has required, thus, we mark all of them
# This could mislead downstream consumers of the meta data
Expand Down