Skip to content

Commit

Permalink
ref(snuba): Drop bulk_snql_query (#69402)
Browse files Browse the repository at this point in the history
  • Loading branch information
armenzg authored and MichaelSun48 committed Apr 25, 2024
1 parent 37e8104 commit bc95787
Show file tree
Hide file tree
Showing 14 changed files with 108 additions and 95 deletions.
6 changes: 3 additions & 3 deletions src/sentry/api/endpoints/organization_events_trace.py
Expand Up @@ -31,7 +31,7 @@
from sentry.utils.iterators import chunked
from sentry.utils.numbers import base32_encode, format_grouped_length
from sentry.utils.sdk import set_measurement
from sentry.utils.snuba import bulk_snql_query
from sentry.utils.snuba import bulk_snuba_queries
from sentry.utils.validators import INVALID_ID_DETAILS, is_event_id

logger: logging.Logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -565,7 +565,7 @@ def query_trace_data(
auto_fields=False,
),
)
results = bulk_snql_query(
results = bulk_snuba_queries(
[
transaction_query.get_snql_query(),
error_query.get_snql_query(),
Expand Down Expand Up @@ -744,7 +744,7 @@ def augment_transactions_with_spans(
sentry_sdk.set_measurement("trace_view.span_query.total_chunks", total_chunks)
chunks = chunked(list_spans, (len(list_spans) // total_chunks) + 1)
queries = [build_span_query(trace_id, spans_params, chunk) for chunk in chunks]
results = bulk_snql_query(
results = bulk_snuba_queries(
[query.get_snql_query() for query in queries],
referrer=Referrer.API_TRACE_VIEW_GET_PARENTS.value,
)
Expand Down
4 changes: 2 additions & 2 deletions src/sentry/api/endpoints/organization_profiling_functions.py
Expand Up @@ -25,7 +25,7 @@
from sentry.snuba.referrer import Referrer
from sentry.utils.dates import parse_stats_period, validate_interval
from sentry.utils.sdk import set_measurement
from sentry.utils.snuba import bulk_snql_query
from sentry.utils.snuba import bulk_snuba_queries

TOP_FUNCTIONS_LIMIT = 50
FUNCTIONS_PER_QUERY = 10
Expand Down Expand Up @@ -138,7 +138,7 @@ def get_event_stats(_columns, query, params, _rollup, zerofill_results, _compari
)
for chunk in chunks
]
bulk_results = bulk_snql_query(
bulk_results = bulk_snuba_queries(
[builder.get_snql_query() for builder in builders],
Referrer.API_PROFILING_FUNCTION_TRENDS_STATS.value,
)
Expand Down
4 changes: 2 additions & 2 deletions src/sentry/api/endpoints/organization_traces.py
Expand Up @@ -23,7 +23,7 @@
from sentry.snuba.dataset import Dataset
from sentry.snuba.referrer import Referrer
from sentry.utils.numbers import clip
from sentry.utils.snuba import bulk_snql_query
from sentry.utils.snuba import bulk_snuba_queries


class TraceInterval(TypedDict):
Expand Down Expand Up @@ -224,7 +224,7 @@ def data_fn(offset: int, limit: int):
]

with handle_query_errors():
results = bulk_snql_query(
results = bulk_snuba_queries(
[query.get_snql_query() for query in queries],
Referrer.API_TRACE_EXPLORER_TRACES_META.value,
)
Expand Down
6 changes: 3 additions & 3 deletions src/sentry/search/events/builder/metrics.py
Expand Up @@ -72,7 +72,7 @@
MetricsQuery,
)
from sentry.snuba.metrics.utils import get_num_intervals
from sentry.utils.snuba import DATASETS, bulk_snql_query, raw_snql_query
from sentry.utils.snuba import DATASETS, bulk_snuba_queries, raw_snql_query


class MetricsQueryBuilder(QueryBuilder):
Expand Down Expand Up @@ -1636,7 +1636,7 @@ def run_query(self, referrer: str, use_cache: bool = False) -> Any:

queries = self.get_snql_query()
if queries:
results = bulk_snql_query(queries, referrer, use_cache)
results = bulk_snuba_queries(queries, referrer, use_cache)
else:
results = []

Expand Down Expand Up @@ -1910,7 +1910,7 @@ def run_query(self, referrer: str, use_cache: bool = False) -> Any:
else:
queries = self.get_snql_query()
if queries:
results = bulk_snql_query(queries, referrer, use_cache)
results = bulk_snuba_queries(queries, referrer, use_cache)

time_map: dict[str, dict[str, Any]] = defaultdict(dict)
for current_result in results:
Expand Down
2 changes: 1 addition & 1 deletion src/sentry/search/snuba/executors.py
Expand Up @@ -1457,7 +1457,7 @@ def query(
)
queries.append(request)

bulk_result = snuba.bulk_snql_query(
bulk_result = snuba.bulk_snuba_queries(
queries, referrer="search.snuba.group_attributes_search.query"
)

Expand Down
8 changes: 5 additions & 3 deletions src/sentry/snuba/discover.py
Expand Up @@ -32,7 +32,7 @@
from sentry.utils.math import nice_int
from sentry.utils.snuba import (
SnubaTSResult,
bulk_snql_query,
bulk_snuba_queries,
get_array_column_alias,
get_array_column_field,
get_measurement_name,
Expand Down Expand Up @@ -335,7 +335,9 @@ def timeseries_query(
)
query_list.append(comparison_builder)

query_results = bulk_snql_query([query.get_snql_query() for query in query_list], referrer)
query_results = bulk_snuba_queries(
[query.get_snql_query() for query in query_list], referrer
)

with sentry_sdk.start_span(op="discover.discover", description="timeseries.transform_results"):
results = []
Expand Down Expand Up @@ -495,7 +497,7 @@ def top_events_timeseries(
timeseries_columns=timeseries_columns,
equations=equations,
)
result, other_result = bulk_snql_query(
result, other_result = bulk_snuba_queries(
[top_events_builder.get_snql_query(), other_events_builder.get_snql_query()],
referrer=referrer,
)
Expand Down
28 changes: 16 additions & 12 deletions src/sentry/snuba/errors.py
Expand Up @@ -14,7 +14,7 @@
from sentry.snuba.dataset import Dataset
from sentry.snuba.discover import EventsResponse, transform_tips, zerofill
from sentry.snuba.metrics.extraction import MetricSpecType
from sentry.utils.snuba import SnubaTSResult, bulk_snql_query
from sentry.utils.snuba import SnubaTSResult, bulk_snuba_queries

is_filter_translation = {}
for status_key, status_value in STATUS_QUERY_CHOICES.items():
Expand Down Expand Up @@ -128,23 +128,27 @@ def timeseries_query(
)
query_list.append(comparison_builder)

query_results = bulk_snql_query([query.get_snql_query() for query in query_list], referrer)
query_results = bulk_snuba_queries(
[query.get_snql_query() for query in query_list], referrer
)

with sentry_sdk.start_span(op="errors", description="timeseries.transform_results"):
results = []
for snql_query, result in zip(query_list, query_results):
results.append(
{
"data": zerofill(
result["data"],
snql_query.params.start,
snql_query.params.end,
rollup,
"time",
time_col_name="events.time",
)
if zerofill_results
else result["data"],
"data": (
zerofill(
result["data"],
snql_query.params.start,
snql_query.params.end,
rollup,
"time",
time_col_name="events.time",
)
if zerofill_results
else result["data"]
),
"meta": result["meta"],
}
)
Expand Down
26 changes: 15 additions & 11 deletions src/sentry/snuba/issue_platform.py
Expand Up @@ -12,7 +12,7 @@
from sentry.snuba.dataset import Dataset
from sentry.snuba.discover import EventsResponse, transform_tips, zerofill
from sentry.snuba.metrics.extraction import MetricSpecType
from sentry.utils.snuba import SnubaTSResult, bulk_snql_query
from sentry.utils.snuba import SnubaTSResult, bulk_snuba_queries


def query(
Expand Down Expand Up @@ -172,22 +172,26 @@ def timeseries_query(
)
query_list.append(comparison_builder)

query_results = bulk_snql_query([query.get_snql_query() for query in query_list], referrer)
query_results = bulk_snuba_queries(
[query.get_snql_query() for query in query_list], referrer
)

with sentry_sdk.start_span(op="issueplatform", description="timeseries.transform_results"):
results = []
for snql_query, result in zip(query_list, query_results):
results.append(
{
"data": zerofill(
result["data"],
snql_query.params.start,
snql_query.params.end,
rollup,
"time",
)
if zerofill_results
else result["data"],
"data": (
zerofill(
result["data"],
snql_query.params.start,
snql_query.params.end,
rollup,
"time",
)
if zerofill_results
else result["data"]
),
"meta": result["meta"],
}
)
Expand Down
20 changes: 11 additions & 9 deletions src/sentry/snuba/metrics/datasource.py
Expand Up @@ -89,7 +89,7 @@
get_intervals,
to_intervals,
)
from sentry.utils.snuba import bulk_snql_query, raw_snql_query
from sentry.utils.snuba import bulk_snuba_queries, raw_snql_query

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -409,7 +409,7 @@ def get_stored_metrics_of_projects(
use_case_id_to_index[use_case_id].append(len(requests) - 1)

# We run the queries all in parallel.
results = bulk_snql_query(
results = bulk_snuba_queries(
requests=requests,
referrer="snuba.metrics.datasource.get_stored_metrics_of_projects",
use_cache=True,
Expand Down Expand Up @@ -1228,11 +1228,13 @@ def get_series(
"end": metrics_query.end,
"intervals": intervals,
"groups": result_groups,
"meta": translate_meta_results(
meta=meta,
alias_to_metric_field=converter._alias_to_metric_field,
alias_to_metric_group_by_field=groupby_aliases,
)
if include_meta
else [],
"meta": (
translate_meta_results(
meta=meta,
alias_to_metric_field=converter._alias_to_metric_field,
alias_to_metric_group_by_field=groupby_aliases,
)
if include_meta
else []
),
}
40 changes: 23 additions & 17 deletions src/sentry/snuba/metrics_performance.py
Expand Up @@ -21,7 +21,7 @@
from sentry.snuba import discover
from sentry.snuba.dataset import Dataset
from sentry.snuba.metrics.extraction import MetricSpecType
from sentry.utils.snuba import SnubaTSResult, bulk_snql_query
from sentry.utils.snuba import SnubaTSResult, bulk_snuba_queries

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -177,7 +177,7 @@ def bulk_timeseries_query(
metrics_queries.append(snql_query[0])

metrics_referrer = referrer + ".metrics-enhanced"
bulk_result = bulk_snql_query(metrics_queries, metrics_referrer)
bulk_result = bulk_snuba_queries(metrics_queries, metrics_referrer)
_result: dict[str, Any] = {"data": []}
for br in bulk_result:
_result["data"] = [*_result["data"], *br["data"]]
Expand Down Expand Up @@ -221,9 +221,11 @@ def bulk_timeseries_query(
)
return SnubaTSResult(
{
"data": discover.zerofill([], params["start"], params["end"], rollup, "time")
if zerofill_results
else [],
"data": (
discover.zerofill([], params["start"], params["end"], rollup, "time")
if zerofill_results
else []
),
},
params["start"],
params["end"],
Expand Down Expand Up @@ -360,9 +362,11 @@ def run_metrics_query(inner_params: ParamsType):
# In case the query was not compatible with metrics we return empty data.
return SnubaTSResult(
{
"data": discover.zerofill([], params["start"], params["end"], rollup, "time")
if zerofill_results
else [],
"data": (
discover.zerofill([], params["start"], params["end"], rollup, "time")
if zerofill_results
else []
),
},
params["start"],
params["end"],
Expand Down Expand Up @@ -435,7 +439,7 @@ def top_events_timeseries(
),
)

# TODO: use bulk_snql_query
# TODO: use bulk_snuba_queries
other_result = other_events_builder.run_query(referrer)
result = top_events_builder.run_query(referrer)
else:
Expand All @@ -448,9 +452,11 @@ def top_events_timeseries(
):
return SnubaTSResult(
{
"data": discover.zerofill([], params["start"], params["end"], rollup, "time")
if zerofill_results
else [],
"data": (
discover.zerofill([], params["start"], params["end"], rollup, "time")
if zerofill_results
else []
),
},
params["start"],
params["end"],
Expand Down Expand Up @@ -482,11 +488,11 @@ def top_events_timeseries(
for key, item in results.items():
results[key] = SnubaTSResult(
{
"data": discover.zerofill(
item["data"], params["start"], params["end"], rollup, "time"
)
if zerofill_results
else item["data"],
"data": (
discover.zerofill(item["data"], params["start"], params["end"], rollup, "time")
if zerofill_results
else item["data"]
),
"order": item["order"],
# One of the queries in the builder has required, thus, we mark all of them
# This could mislead downstream consumers of the meta data
Expand Down

0 comments on commit bc95787

Please sign in to comment.