Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(spans): Add topic definition for buffered-segment topic (#237)
We're building a new pipeline to support performance issues with span streaming, at a high level, it looks like this Screenshot 2024-03-13 at 5 00 35 PM the buffered-segment topic will expect to receive a list of spans. As of this moment, these spans will have the exact same schema as ones we consume from the spans kafka topic, the SpanEvent definition is just copied from the snuba-span schema. we might deserialize the spans to drop fields before pushing to redis as a memory optimization at which point the span schemas will diverge.
- Loading branch information
1 parent
0159e5a
commit eaaacca
Showing
4 changed files
with
312 additions
and
0 deletions.
There are no files selected for viewing
Validating CODEOWNERS rules …
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| { | ||
| "spans": [ | ||
| { | ||
| "duration_ms": 0, | ||
| "event_id": "72fcea47d44a444fb132f8d462eeb0b4", | ||
| "exclusive_time_ms": 0.006, | ||
| "is_segment": true, | ||
| "parent_span_id": "93f0e87ad9cc709e", | ||
| "profile_id": "7ce060d7ea62432b8355bc9e612676e4", | ||
| "project_id": 1, | ||
| "received": 1706734067.029479, | ||
| "retention_days": 90, | ||
| "segment_id": "ace31e54d65652aa", | ||
| "sentry_tags": { | ||
| "environment": "development", | ||
| "op": "relay_fetch_org_options", | ||
| "release": "backend@24.2.0.dev0+df7615f2ff7dc3c8802f806477f920bb934bd198", | ||
| "transaction": "/api/0/relays/projectconfigs/", | ||
| "transaction.method": "POST", | ||
| "transaction.op": "http.server", | ||
| "user": "ip:127.0.0.1" | ||
| }, | ||
| "span_id": "95acbe6d30a66717", | ||
| "start_timestamp_ms": 1706734066840, | ||
| "trace_id": "8e6f22e6169545cc963255d0f29cb76b" | ||
| }, | ||
| { | ||
| "duration_ms": 0, | ||
| "event_id": "72fcea47d44a444fb132f8d462eeb0b4", | ||
| "exclusive_time_ms": 0.006, | ||
| "is_segment": false, | ||
| "parent_span_id": "93f0e87ad9cc709e", | ||
| "profile_id": "7ce060d7ea62432b8355bc9e612676e4", | ||
| "project_id": 1, | ||
| "received": 1706734067.029479, | ||
| "retention_days": 90, | ||
| "segment_id": "ace31e54d65652aa", | ||
| "sentry_tags": { | ||
| "environment": "development", | ||
| "op": "relay_fetch_org_options", | ||
| "release": "backend@24.2.0.dev0+df7615f2ff7dc3c8802f806477f920bb934bd198", | ||
| "transaction": "/api/0/relays/projectconfigs/", | ||
| "transaction.method": "POST", | ||
| "transaction.op": "http.server", | ||
| "user": "ip:127.0.0.1" | ||
| }, | ||
| "span_id": "95acbe6d30a66717", | ||
| "start_timestamp_ms": 1706734066840, | ||
| "trace_id": "8e6f22e6169545cc963255d0f29cb76b" | ||
| } | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,239 @@ | ||
| { | ||
| "$schema": "http://json-schema.org/draft-07/schema#", | ||
| "title": "buffered_segment", | ||
| "type": "object", | ||
| "additionalProperties": true, | ||
| "properties": { | ||
| "spans": { | ||
| "$ref": "#/definitions/Spans" | ||
| } | ||
| }, | ||
| "required": ["spans"], | ||
| "definitions": { | ||
| "Spans": { | ||
| "type": "array", | ||
| "minItems": 1, | ||
| "items": { | ||
| "$ref": "#/definitions/Span" | ||
| } | ||
| }, | ||
| "Span": { | ||
| "type": "object", | ||
| "additionalProperties": true, | ||
| "properties": { | ||
| "event_id": { | ||
| "$ref": "#/definitions/UUID" | ||
| }, | ||
| "organization_id": { | ||
| "$ref": "#/definitions/UInt" | ||
| }, | ||
| "project_id": { | ||
| "$ref": "#/definitions/UInt" | ||
| }, | ||
| "trace_id": { | ||
| "$ref": "#/definitions/UUID", | ||
| "description": "The trace ID is a unique identifier for a trace. It is a 16 byte hexadecimal string." | ||
| }, | ||
| "span_id": { | ||
| "type": "string", | ||
| "description": "The span ID is a unique identifier for a span within a trace. It is an 8 byte hexadecimal string." | ||
| }, | ||
| "parent_span_id": { | ||
| "type": "string", | ||
| "description": "The parent span ID is the ID of the span that caused this span. It is an 8 byte hexadecimal string." | ||
| }, | ||
| "segment_id": { | ||
| "type": "string", | ||
| "description": "The segment ID is a unique identifier for a segment within a trace. It is an 8 byte hexadecimal string." | ||
| }, | ||
| "profile_id": { | ||
| "$ref": "#/definitions/UUID", | ||
| "description": "The profile ID. It is an 16 byte hexadecimal string." | ||
| }, | ||
| "is_segment": { | ||
| "type": "boolean", | ||
| "description": "Whether this span is a segment or not." | ||
| }, | ||
| "start_timestamp_ms": { | ||
| "$ref": "#/definitions/UInt", | ||
| "description": "The start timestamp of the span in milliseconds since epoch." | ||
| }, | ||
| "duration_ms": { | ||
| "$ref": "#/definitions/UInt32", | ||
| "description": "The duration of the span in milliseconds." | ||
| }, | ||
| "exclusive_time_ms": { | ||
| "$ref": "#/definitions/PositiveFloat", | ||
| "description": "The exclusive time of the span in milliseconds." | ||
| }, | ||
| "retention_days": { | ||
| "$ref": "#/definitions/UInt16" | ||
| }, | ||
| "received": { | ||
| "$ref": "#/definitions/PositiveFloat", | ||
| "description": "Unix timestamp when the span was received by Sentry." | ||
| }, | ||
| "description": { | ||
| "type": "string" | ||
| }, | ||
| "tags": { | ||
| "description": " Manual key/value tag pairs.", | ||
| "anyOf": [ | ||
| { | ||
| "type": "object", | ||
| "additionalProperties": { | ||
| "$ref": "#/definitions/TagValue" | ||
| } | ||
| }, | ||
| { | ||
| "type": "null" | ||
| } | ||
| ] | ||
| }, | ||
| "sentry_tags": { | ||
| "$ref": "#/definitions/SentryExtractedTags" | ||
| }, | ||
| "measurements": { | ||
| "$ref": "#/definitions/Measurements" | ||
| }, | ||
| "_metrics_summary": { | ||
| "type": "object", | ||
| "additionalProperties": { | ||
| "$ref": "#/definitions/MetricsSummary" | ||
| } | ||
| } | ||
| }, | ||
| "required": [ | ||
| "duration_ms", | ||
| "exclusive_time_ms", | ||
| "is_segment", | ||
| "project_id", | ||
| "received", | ||
| "retention_days", | ||
| "span_id", | ||
| "start_timestamp_ms", | ||
| "trace_id" | ||
| ] | ||
| }, | ||
| "SentryExtractedTags": { | ||
| "description": "Tags extracted by sentry. These are kept separate from customer tags", | ||
| "anyOf": [ | ||
| { | ||
| "type": "object", | ||
| "properties": { | ||
| "http.method": { | ||
| "type": "string" | ||
| }, | ||
| "action": { | ||
| "type": "string" | ||
| }, | ||
| "domain": { | ||
| "type": "string" | ||
| }, | ||
| "module": { | ||
| "type": "string" | ||
| }, | ||
| "group": { | ||
| "type": "string" | ||
| }, | ||
| "system": { | ||
| "type": "string" | ||
| }, | ||
| "status": { | ||
| "type": "string" | ||
| }, | ||
| "status_code": { | ||
| "type": "string" | ||
| }, | ||
| "transaction": { | ||
| "type": "string" | ||
| }, | ||
| "transaction.op": { | ||
| "type": "string" | ||
| }, | ||
| "op": { | ||
| "type": "string" | ||
| }, | ||
| "transaction.method": { | ||
| "type": "string" | ||
| } | ||
| }, | ||
| "additionalProperties": true | ||
| } | ||
| ] | ||
| }, | ||
| "TagValue": { | ||
| "type": "string" | ||
| }, | ||
| "UUID": { | ||
| "type": "string", | ||
| "minLength": 32, | ||
| "maxLength": 36 | ||
| }, | ||
| "UInt16": { | ||
| "type": "integer", | ||
| "minimum": 0, | ||
| "maximum": 65535 | ||
| }, | ||
| "UInt32": { | ||
| "type": "integer", | ||
| "minimum": 0, | ||
| "maximum": 4294967295 | ||
| }, | ||
| "UInt": { | ||
| "type": "integer", | ||
| "minimum": 0 | ||
| }, | ||
| "PositiveFloat": { | ||
| "type": "number", | ||
| "minimum": 0 | ||
| }, | ||
| "Measurements": { | ||
| "type": "object", | ||
| "additionalProperties": { | ||
| "$ref": "#/definitions/MeasurementValue" | ||
| } | ||
| }, | ||
| "MeasurementValue": { | ||
| "type": "object", | ||
| "properties": { | ||
| "value": { | ||
| "type": "number" | ||
| }, | ||
| "unit": { | ||
| "type": "string" | ||
| } | ||
| }, | ||
| "required": ["value"] | ||
| }, | ||
| "MetricsSummary": { | ||
| "type": "array", | ||
| "items": { | ||
| "$ref": "#/definitions/MetricSummaryValue" | ||
| } | ||
| }, | ||
| "MetricSummaryValue": { | ||
| "type": "object", | ||
| "properties": { | ||
| "min": { | ||
| "type": "number" | ||
| }, | ||
| "max": { | ||
| "type": "number" | ||
| }, | ||
| "sum": { | ||
| "type": "number" | ||
| }, | ||
| "count": { | ||
| "type": "number" | ||
| }, | ||
| "tags": { | ||
| "type": "object", | ||
| "additionalProperties": { | ||
| "type": "string" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| topic: buffered-segments | ||
| description: List of buffered spans for a segment | ||
| services: | ||
| producers: | ||
| - getsentry/sentry | ||
| consumers: | ||
| - getsentry/sentry | ||
| schemas: | ||
| - version: 1 | ||
| compatibility_mode: none | ||
| type: json | ||
| resource: buffered-segments.v1.schema.json | ||
| examples: | ||
| - buffered-segments/1/ | ||
| topic_creation_config: | ||
| compression.type: lz4 |