From eaaacca416f919caeca78fa0f1ff2528d277b128 Mon Sep 17 00:00:00 2001 From: Shruthi Date: Mon, 18 Mar 2024 10:01:42 -0400 Subject: [PATCH] feat(spans): Add topic definition for buffered-segment topic (#237) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're building a new pipeline to support performance issues with span streaming, at a high level, it looks like this Screenshot 2024-03-13 at 5 00 35 PM the buffered-segment topic will expect to receive a list of spans. As of this moment, these spans will have the exact same schema as ones we consume from the spans kafka topic, the SpanEvent definition is just copied from the snuba-span schema. we might deserialize the spans to drop fields before pushing to redis as a memory optimization at which point the span schemas will diverge. --- CODEOWNERS | 5 + .../1/basic_buffered_segment.json | 52 ++++ schemas/buffered-segments.v1.schema.json | 239 ++++++++++++++++++ topics/buffered-segments.yaml | 16 ++ 4 files changed, 312 insertions(+) create mode 100644 examples/buffered-segments/1/basic_buffered_segment.json create mode 100644 schemas/buffered-segments.v1.schema.json create mode 100644 topics/buffered-segments.yaml diff --git a/CODEOWNERS b/CODEOWNERS index 572f17d6..d4c11567 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -26,6 +26,9 @@ /topics/snuba-spans.yaml @getsentry/owners-snuba @getsentry/profiling /topics/snuba-metrics-summaries.yaml @getsentry/owners-snuba @getsentry/profiling +# Topics produced by Sentry +/topics/buffered-segments.yaml @getsentry/owners-snuba @getsentry/performance + # DLQs for Snuba topics /topics/snuba-dead-letter-metrics.yaml @getsentry/owners-snuba /topics/snuba-dead-letter-generic-metrics.yaml @getsentry/owners-snuba @@ -51,6 +54,7 @@ /schemas/profile-functions.v1.schema.json @getsentry/owners-snuba @getsentry/profiling /schemas/group-attributes.v1.schema.json @getsentry/owners-snuba @getsentry/issues /schemas/snuba-spans.v1.schema.json @getsentry/owners-snuba @getsentry/profiling +/schemas/buffered-segments.v1.schema.json @getsentry/owners-snuba @getsentry/performance /schemas/snuba-metrics-summaries.v1.schema.json @getsentry/owners-snuba @getsentry/profiling # Examples @@ -62,6 +66,7 @@ /examples/profile-functions/ @getsentry/owners-snuba @getsentry/profiling /examples/group-attributes/ @getsentry/owners-snuba @getsentry/issues /examples/snuba-spans/ @getsentry/owners-snuba @getsentry/profiling +/examples/buffered-segments/ @getsentry/owners-snuba @getsentry/performance /examples/snuba-metrics-summaries/ @getsentry/owners-snuba @getsentry/profiling # Internal Snuba topics diff --git a/examples/buffered-segments/1/basic_buffered_segment.json b/examples/buffered-segments/1/basic_buffered_segment.json new file mode 100644 index 00000000..3168e596 --- /dev/null +++ b/examples/buffered-segments/1/basic_buffered_segment.json @@ -0,0 +1,52 @@ +{ + "spans": [ + { + "duration_ms": 0, + "event_id": "72fcea47d44a444fb132f8d462eeb0b4", + "exclusive_time_ms": 0.006, + "is_segment": true, + "parent_span_id": "93f0e87ad9cc709e", + "profile_id": "7ce060d7ea62432b8355bc9e612676e4", + "project_id": 1, + "received": 1706734067.029479, + "retention_days": 90, + "segment_id": "ace31e54d65652aa", + "sentry_tags": { + "environment": "development", + "op": "relay_fetch_org_options", + "release": "backend@24.2.0.dev0+df7615f2ff7dc3c8802f806477f920bb934bd198", + "transaction": "/api/0/relays/projectconfigs/", + "transaction.method": "POST", + "transaction.op": "http.server", + "user": "ip:127.0.0.1" + }, + "span_id": "95acbe6d30a66717", + "start_timestamp_ms": 1706734066840, + "trace_id": "8e6f22e6169545cc963255d0f29cb76b" + }, + { + "duration_ms": 0, + "event_id": "72fcea47d44a444fb132f8d462eeb0b4", + "exclusive_time_ms": 0.006, + "is_segment": false, + "parent_span_id": "93f0e87ad9cc709e", + "profile_id": "7ce060d7ea62432b8355bc9e612676e4", + "project_id": 1, + "received": 1706734067.029479, + "retention_days": 90, + "segment_id": "ace31e54d65652aa", + "sentry_tags": { + "environment": "development", + "op": "relay_fetch_org_options", + "release": "backend@24.2.0.dev0+df7615f2ff7dc3c8802f806477f920bb934bd198", + "transaction": "/api/0/relays/projectconfigs/", + "transaction.method": "POST", + "transaction.op": "http.server", + "user": "ip:127.0.0.1" + }, + "span_id": "95acbe6d30a66717", + "start_timestamp_ms": 1706734066840, + "trace_id": "8e6f22e6169545cc963255d0f29cb76b" + } + ] +} diff --git a/schemas/buffered-segments.v1.schema.json b/schemas/buffered-segments.v1.schema.json new file mode 100644 index 00000000..135a4ef6 --- /dev/null +++ b/schemas/buffered-segments.v1.schema.json @@ -0,0 +1,239 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "buffered_segment", + "type": "object", + "additionalProperties": true, + "properties": { + "spans": { + "$ref": "#/definitions/Spans" + } + }, + "required": ["spans"], + "definitions": { + "Spans": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/Span" + } + }, + "Span": { + "type": "object", + "additionalProperties": true, + "properties": { + "event_id": { + "$ref": "#/definitions/UUID" + }, + "organization_id": { + "$ref": "#/definitions/UInt" + }, + "project_id": { + "$ref": "#/definitions/UInt" + }, + "trace_id": { + "$ref": "#/definitions/UUID", + "description": "The trace ID is a unique identifier for a trace. It is a 16 byte hexadecimal string." + }, + "span_id": { + "type": "string", + "description": "The span ID is a unique identifier for a span within a trace. It is an 8 byte hexadecimal string." + }, + "parent_span_id": { + "type": "string", + "description": "The parent span ID is the ID of the span that caused this span. It is an 8 byte hexadecimal string." + }, + "segment_id": { + "type": "string", + "description": "The segment ID is a unique identifier for a segment within a trace. It is an 8 byte hexadecimal string." + }, + "profile_id": { + "$ref": "#/definitions/UUID", + "description": "The profile ID. It is an 16 byte hexadecimal string." + }, + "is_segment": { + "type": "boolean", + "description": "Whether this span is a segment or not." + }, + "start_timestamp_ms": { + "$ref": "#/definitions/UInt", + "description": "The start timestamp of the span in milliseconds since epoch." + }, + "duration_ms": { + "$ref": "#/definitions/UInt32", + "description": "The duration of the span in milliseconds." + }, + "exclusive_time_ms": { + "$ref": "#/definitions/PositiveFloat", + "description": "The exclusive time of the span in milliseconds." + }, + "retention_days": { + "$ref": "#/definitions/UInt16" + }, + "received": { + "$ref": "#/definitions/PositiveFloat", + "description": "Unix timestamp when the span was received by Sentry." + }, + "description": { + "type": "string" + }, + "tags": { + "description": " Manual key/value tag pairs.", + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/TagValue" + } + }, + { + "type": "null" + } + ] + }, + "sentry_tags": { + "$ref": "#/definitions/SentryExtractedTags" + }, + "measurements": { + "$ref": "#/definitions/Measurements" + }, + "_metrics_summary": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/MetricsSummary" + } + } + }, + "required": [ + "duration_ms", + "exclusive_time_ms", + "is_segment", + "project_id", + "received", + "retention_days", + "span_id", + "start_timestamp_ms", + "trace_id" + ] + }, + "SentryExtractedTags": { + "description": "Tags extracted by sentry. These are kept separate from customer tags", + "anyOf": [ + { + "type": "object", + "properties": { + "http.method": { + "type": "string" + }, + "action": { + "type": "string" + }, + "domain": { + "type": "string" + }, + "module": { + "type": "string" + }, + "group": { + "type": "string" + }, + "system": { + "type": "string" + }, + "status": { + "type": "string" + }, + "status_code": { + "type": "string" + }, + "transaction": { + "type": "string" + }, + "transaction.op": { + "type": "string" + }, + "op": { + "type": "string" + }, + "transaction.method": { + "type": "string" + } + }, + "additionalProperties": true + } + ] + }, + "TagValue": { + "type": "string" + }, + "UUID": { + "type": "string", + "minLength": 32, + "maxLength": 36 + }, + "UInt16": { + "type": "integer", + "minimum": 0, + "maximum": 65535 + }, + "UInt32": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295 + }, + "UInt": { + "type": "integer", + "minimum": 0 + }, + "PositiveFloat": { + "type": "number", + "minimum": 0 + }, + "Measurements": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/MeasurementValue" + } + }, + "MeasurementValue": { + "type": "object", + "properties": { + "value": { + "type": "number" + }, + "unit": { + "type": "string" + } + }, + "required": ["value"] + }, + "MetricsSummary": { + "type": "array", + "items": { + "$ref": "#/definitions/MetricSummaryValue" + } + }, + "MetricSummaryValue": { + "type": "object", + "properties": { + "min": { + "type": "number" + }, + "max": { + "type": "number" + }, + "sum": { + "type": "number" + }, + "count": { + "type": "number" + }, + "tags": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } +} diff --git a/topics/buffered-segments.yaml b/topics/buffered-segments.yaml new file mode 100644 index 00000000..ef466c2e --- /dev/null +++ b/topics/buffered-segments.yaml @@ -0,0 +1,16 @@ +topic: buffered-segments +description: List of buffered spans for a segment +services: + producers: + - getsentry/sentry + consumers: + - getsentry/sentry +schemas: + - version: 1 + compatibility_mode: none + type: json + resource: buffered-segments.v1.schema.json + examples: + - buffered-segments/1/ +topic_creation_config: + compression.type: lz4