From 18932e462b3bde567e5d7c08445d52562627f4f8 Mon Sep 17 00:00:00 2001 From: Lyn Nagara Date: Tue, 12 Mar 2024 13:34:49 -0700 Subject: [PATCH] feat: Ensure compression.type is specified on all topics (#234) --- python/tests/test_sentry_kafka_schemas.py | 11 ++++------- python/tests/test_valid_topic_data.py | 7 +++++++ topics/events-subscription-results.yaml | 2 ++ topics/events.yaml | 1 + topics/generic-events.yaml | 1 + topics/generic-metrics-subscription-results.yaml | 2 ++ topics/group-attributes.yaml | 1 + topics/ingest-events-dlq.yaml | 1 + topics/ingest-events.yaml | 1 + topics/ingest-generic-metrics-dlq.yaml | 1 + topics/ingest-metrics-dlq.yaml | 1 + topics/ingest-metrics.yaml | 2 ++ topics/ingest-performance-metrics.yaml | 2 ++ topics/ingest-replay-events.yaml | 1 + topics/ingest-replay-recordings.yaml | 2 ++ topics/metrics-subscription-results.yaml | 2 ++ topics/outcomes-billing.yaml | 2 ++ topics/outcomes.yaml | 2 ++ topics/processed-profiles.yaml | 1 + topics/profiles-call-tree.yaml | 2 ++ topics/sessions-subscription-results.yaml | 2 ++ topics/shared-resources-usage.yaml | 2 ++ topics/snuba-commit-log.yaml | 1 + topics/snuba-dead-letter-generic-metrics.yaml | 1 + topics/snuba-dead-letter-group-attributes.yaml | 1 + topics/snuba-dead-letter-metrics.yaml | 1 + topics/snuba-dead-letter-querylog.yaml | 1 + topics/snuba-dead-letter-replays.yaml | 1 + topics/snuba-generic-events-commit-log.yaml | 1 + topics/snuba-generic-metrics-counters-commit-log.yaml | 1 + ...nuba-generic-metrics-distributions-commit-log.yaml | 1 + topics/snuba-generic-metrics-sets-commit-log.yaml | 1 + topics/snuba-generic-metrics.yaml | 1 + topics/snuba-metrics-commit-log.yaml | 1 + topics/snuba-metrics-summaries.yaml | 2 ++ topics/snuba-metrics.yaml | 1 + topics/snuba-queries.yaml | 1 + topics/snuba-spans.yaml | 2 ++ topics/snuba-transactions-commit-log.yaml | 1 + topics/transactions-subscription-results.yaml | 2 ++ topics/transactions.yaml | 1 + 41 files changed, 64 insertions(+), 7 deletions(-) diff --git a/python/tests/test_sentry_kafka_schemas.py b/python/tests/test_sentry_kafka_schemas.py index e0a5d365..66549390 100644 --- a/python/tests/test_sentry_kafka_schemas.py +++ b/python/tests/test_sentry_kafka_schemas.py @@ -3,15 +3,12 @@ def test_get_topic() -> None: - # Topic with creation config topic_name = "snuba-queries" topic_data = get_topic(topic_name) - assert topic_data["topic_creation_config"] == {"max.message.bytes": "2000000"} - - # Topic without creation config - topic_name_no_config = "outcomes" - topic_data = get_topic(topic_name_no_config) - assert topic_data["topic_creation_config"] == {} + assert topic_data["topic_creation_config"] == { + "compression.type": "lz4", + "max.message.bytes": "2000000", + } def test_get_schema() -> None: diff --git a/python/tests/test_valid_topic_data.py b/python/tests/test_valid_topic_data.py index 4fd04ea2..6ab60701 100644 --- a/python/tests/test_valid_topic_data.py +++ b/python/tests/test_valid_topic_data.py @@ -98,6 +98,13 @@ def test_all_topics() -> None: # Check description provided for topic assert topic_data["description"] + # Check every topic has an explicit, valid compression type + # Today we use lz4 everywhere, this list can be extended if needed + valid_types = ["lz4"] + assert ( + topic_data["topic_creation_config"]["compression.type"] in valid_types + ) + # Check valid schema versions topic_schemas = topic_data["schemas"] for i, schema_raw in enumerate(topic_schemas): diff --git a/topics/events-subscription-results.yaml b/topics/events-subscription-results.yaml index 5e140920..9a3ceb85 100644 --- a/topics/events-subscription-results.yaml +++ b/topics/events-subscription-results.yaml @@ -13,3 +13,5 @@ schemas: resource: subscription-results.v1.schema.json examples: - subscription-results/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/events.yaml b/topics/events.yaml index b5c64b49..4a40399b 100644 --- a/topics/events.yaml +++ b/topics/events.yaml @@ -15,5 +15,6 @@ schemas: examples: - events/1/ topic_creation_config: + compression.type: lz4 max.message.bytes: "25000000" message.timestamp.type: LogAppendTime diff --git a/topics/generic-events.yaml b/topics/generic-events.yaml index 82ba06cf..177a6416 100644 --- a/topics/generic-events.yaml +++ b/topics/generic-events.yaml @@ -14,4 +14,5 @@ schemas: examples: - generic-events/1/ topic_creation_config: + compression.type: lz4 message.timestamp.type: LogAppendTime diff --git a/topics/generic-metrics-subscription-results.yaml b/topics/generic-metrics-subscription-results.yaml index 9e4d9930..64f586e8 100644 --- a/topics/generic-metrics-subscription-results.yaml +++ b/topics/generic-metrics-subscription-results.yaml @@ -13,3 +13,5 @@ schemas: resource: subscription-results.v1.schema.json examples: - subscription-results/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/group-attributes.yaml b/topics/group-attributes.yaml index e8fd5cfb..de72d6cd 100644 --- a/topics/group-attributes.yaml +++ b/topics/group-attributes.yaml @@ -13,4 +13,5 @@ schemas: examples: - group-attributes/1/ topic_creation_config: + compression.type: lz4 message.timestamp.type: LogAppendTime diff --git a/topics/ingest-events-dlq.yaml b/topics/ingest-events-dlq.yaml index 5d7ea7fc..893935a0 100644 --- a/topics/ingest-events-dlq.yaml +++ b/topics/ingest-events-dlq.yaml @@ -13,4 +13,5 @@ schemas: examples: - any/ topic_creation_config: + compression.type: lz4 log.retention.minutes: "10080" diff --git a/topics/ingest-events.yaml b/topics/ingest-events.yaml index 0b63fa36..62607e4b 100644 --- a/topics/ingest-events.yaml +++ b/topics/ingest-events.yaml @@ -14,4 +14,5 @@ schemas: examples: - ingest-events/1/ topic_creation_config: + compression.type: lz4 max.message.bytes: "10000000" diff --git a/topics/ingest-generic-metrics-dlq.yaml b/topics/ingest-generic-metrics-dlq.yaml index d02820a9..48d57024 100644 --- a/topics/ingest-generic-metrics-dlq.yaml +++ b/topics/ingest-generic-metrics-dlq.yaml @@ -13,4 +13,5 @@ schemas: examples: - any/ topic_creation_config: + compression.type: lz4 log.retention.minutes: "10080" diff --git a/topics/ingest-metrics-dlq.yaml b/topics/ingest-metrics-dlq.yaml index 000b68eb..6da638bb 100644 --- a/topics/ingest-metrics-dlq.yaml +++ b/topics/ingest-metrics-dlq.yaml @@ -13,4 +13,5 @@ schemas: examples: - any/ topic_creation_config: + compression.type: lz4 log.retention.minutes: "10080" diff --git a/topics/ingest-metrics.yaml b/topics/ingest-metrics.yaml index d8c36d1a..3beb0623 100644 --- a/topics/ingest-metrics.yaml +++ b/topics/ingest-metrics.yaml @@ -14,3 +14,5 @@ schemas: resource: ingest-metrics.v1.schema.json examples: - ingest-metrics/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/ingest-performance-metrics.yaml b/topics/ingest-performance-metrics.yaml index ce2ba2f5..45eb6029 100644 --- a/topics/ingest-performance-metrics.yaml +++ b/topics/ingest-performance-metrics.yaml @@ -13,3 +13,5 @@ schemas: resource: ingest-metrics.v1.schema.json examples: - ingest-metrics/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/ingest-replay-events.yaml b/topics/ingest-replay-events.yaml index 9cf5c818..13113cec 100644 --- a/topics/ingest-replay-events.yaml +++ b/topics/ingest-replay-events.yaml @@ -15,5 +15,6 @@ schemas: examples: - ingest-replay-events/1/ topic_creation_config: + compression.type: lz4 message.timestamp.type: LogAppendTime max.message.bytes: "15000000" diff --git a/topics/ingest-replay-recordings.yaml b/topics/ingest-replay-recordings.yaml index 8f88aac1..f8aad7ae 100644 --- a/topics/ingest-replay-recordings.yaml +++ b/topics/ingest-replay-recordings.yaml @@ -13,3 +13,5 @@ schemas: resource: ingest-replay-recordings.v1.schema.json examples: - ingest-replay-recordings/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/metrics-subscription-results.yaml b/topics/metrics-subscription-results.yaml index f27726a4..662ed480 100644 --- a/topics/metrics-subscription-results.yaml +++ b/topics/metrics-subscription-results.yaml @@ -13,3 +13,5 @@ schemas: resource: subscription-results.v1.schema.json examples: - subscription-results/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/outcomes-billing.yaml b/topics/outcomes-billing.yaml index 84914338..7af66dd4 100644 --- a/topics/outcomes-billing.yaml +++ b/topics/outcomes-billing.yaml @@ -16,3 +16,5 @@ schemas: resource: outcomes.v1.schema.json examples: - outcomes/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/outcomes.yaml b/topics/outcomes.yaml index 01891c98..3318799e 100644 --- a/topics/outcomes.yaml +++ b/topics/outcomes.yaml @@ -16,3 +16,5 @@ schemas: resource: outcomes.v1.schema.json examples: - outcomes/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/processed-profiles.yaml b/topics/processed-profiles.yaml index d5790e49..8ca6626c 100644 --- a/topics/processed-profiles.yaml +++ b/topics/processed-profiles.yaml @@ -14,4 +14,5 @@ schemas: examples: - profile-metadata/1/ topic_creation_config: + compression.type: lz4 message.timestamp.type: LogAppendTime diff --git a/topics/profiles-call-tree.yaml b/topics/profiles-call-tree.yaml index 9c7a526e..ea2daa5f 100644 --- a/topics/profiles-call-tree.yaml +++ b/topics/profiles-call-tree.yaml @@ -14,3 +14,5 @@ schemas: resource: profile-functions.v1.schema.json examples: - profile-functions/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/sessions-subscription-results.yaml b/topics/sessions-subscription-results.yaml index 7f567a77..5ea17639 100644 --- a/topics/sessions-subscription-results.yaml +++ b/topics/sessions-subscription-results.yaml @@ -13,3 +13,5 @@ schemas: resource: subscription-results.v1.schema.json examples: - subscription-results/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/shared-resources-usage.yaml b/topics/shared-resources-usage.yaml index 9923a136..fdbdcb50 100644 --- a/topics/shared-resources-usage.yaml +++ b/topics/shared-resources-usage.yaml @@ -12,3 +12,5 @@ schemas: resource: shared-resources-usage.v1.schema.json examples: - shared-resources-usage/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/snuba-commit-log.yaml b/topics/snuba-commit-log.yaml index 344729e2..706fc4d9 100644 --- a/topics/snuba-commit-log.yaml +++ b/topics/snuba-commit-log.yaml @@ -15,5 +15,6 @@ schemas: examples: - snuba-commit-log/1/ topic_creation_config: + compression.type: lz4 cleanup.policy: compact,delete min.compaction.lag.ms: "3600000" diff --git a/topics/snuba-dead-letter-generic-metrics.yaml b/topics/snuba-dead-letter-generic-metrics.yaml index d4073707..5a29361a 100644 --- a/topics/snuba-dead-letter-generic-metrics.yaml +++ b/topics/snuba-dead-letter-generic-metrics.yaml @@ -13,4 +13,5 @@ schemas: examples: - any/ topic_creation_config: + compression.type: lz4 log.retention.minutes: "10080" diff --git a/topics/snuba-dead-letter-group-attributes.yaml b/topics/snuba-dead-letter-group-attributes.yaml index e8266f1c..34ce5898 100644 --- a/topics/snuba-dead-letter-group-attributes.yaml +++ b/topics/snuba-dead-letter-group-attributes.yaml @@ -13,4 +13,5 @@ schemas: examples: - any/ topic_creation_config: + compression.type: lz4 log.retention.minutes: "10080" diff --git a/topics/snuba-dead-letter-metrics.yaml b/topics/snuba-dead-letter-metrics.yaml index 0f25031f..771e5a73 100644 --- a/topics/snuba-dead-letter-metrics.yaml +++ b/topics/snuba-dead-letter-metrics.yaml @@ -13,4 +13,5 @@ schemas: examples: - any/ topic_creation_config: + compression.type: lz4 log.retention.minutes: "10080" diff --git a/topics/snuba-dead-letter-querylog.yaml b/topics/snuba-dead-letter-querylog.yaml index 6b02d0d2..5af99dd4 100644 --- a/topics/snuba-dead-letter-querylog.yaml +++ b/topics/snuba-dead-letter-querylog.yaml @@ -13,4 +13,5 @@ schemas: examples: - any/ topic_creation_config: + compression.type: lz4 log.retention.minutes: "10080" diff --git a/topics/snuba-dead-letter-replays.yaml b/topics/snuba-dead-letter-replays.yaml index f6fe23d4..eaab4d0c 100644 --- a/topics/snuba-dead-letter-replays.yaml +++ b/topics/snuba-dead-letter-replays.yaml @@ -13,4 +13,5 @@ schemas: examples: - any/ topic_creation_config: + compression.type: lz4 log.retention.minutes: "10080" diff --git a/topics/snuba-generic-events-commit-log.yaml b/topics/snuba-generic-events-commit-log.yaml index 1fe032f8..85865238 100644 --- a/topics/snuba-generic-events-commit-log.yaml +++ b/topics/snuba-generic-events-commit-log.yaml @@ -15,5 +15,6 @@ schemas: examples: - snuba-commit-log/1/ topic_creation_config: + compression.type: lz4 cleanup.policy: compact,delete min.compaction.lag.ms: "3600000" diff --git a/topics/snuba-generic-metrics-counters-commit-log.yaml b/topics/snuba-generic-metrics-counters-commit-log.yaml index f0163206..09ad6c4c 100644 --- a/topics/snuba-generic-metrics-counters-commit-log.yaml +++ b/topics/snuba-generic-metrics-counters-commit-log.yaml @@ -15,5 +15,6 @@ schemas: examples: - snuba-commit-log/1/ topic_creation_config: + compression.type: lz4 cleanup.policy: compact,delete min.compaction.lag.ms: "3600000" diff --git a/topics/snuba-generic-metrics-distributions-commit-log.yaml b/topics/snuba-generic-metrics-distributions-commit-log.yaml index 74d29dd0..2b9d6674 100644 --- a/topics/snuba-generic-metrics-distributions-commit-log.yaml +++ b/topics/snuba-generic-metrics-distributions-commit-log.yaml @@ -15,5 +15,6 @@ schemas: examples: - snuba-commit-log/1/ topic_creation_config: + compression.type: lz4 cleanup.policy: compact,delete min.compaction.lag.ms: "3600000" diff --git a/topics/snuba-generic-metrics-sets-commit-log.yaml b/topics/snuba-generic-metrics-sets-commit-log.yaml index 08a9f430..117a42c0 100644 --- a/topics/snuba-generic-metrics-sets-commit-log.yaml +++ b/topics/snuba-generic-metrics-sets-commit-log.yaml @@ -15,5 +15,6 @@ schemas: examples: - snuba-commit-log/1/ topic_creation_config: + compression.type: lz4 cleanup.policy: compact,delete min.compaction.lag.ms: "3600000" diff --git a/topics/snuba-generic-metrics.yaml b/topics/snuba-generic-metrics.yaml index 2bff1553..193b3435 100644 --- a/topics/snuba-generic-metrics.yaml +++ b/topics/snuba-generic-metrics.yaml @@ -14,4 +14,5 @@ schemas: examples: - snuba-generic-metrics/1/ topic_creation_config: + compression.type: lz4 message.timestamp.type: LogAppendTime diff --git a/topics/snuba-metrics-commit-log.yaml b/topics/snuba-metrics-commit-log.yaml index 43385742..e963f6e4 100644 --- a/topics/snuba-metrics-commit-log.yaml +++ b/topics/snuba-metrics-commit-log.yaml @@ -15,5 +15,6 @@ schemas: examples: - snuba-commit-log/1/ topic_creation_config: + compression.type: lz4 cleanup.policy: compact,delete min.compaction.lag.ms: "3600000" diff --git a/topics/snuba-metrics-summaries.yaml b/topics/snuba-metrics-summaries.yaml index 5e456f50..e9efbd40 100644 --- a/topics/snuba-metrics-summaries.yaml +++ b/topics/snuba-metrics-summaries.yaml @@ -12,3 +12,5 @@ schemas: resource: snuba-metrics-summaries.v1.schema.json examples: - snuba-metrics-summaries/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/snuba-metrics.yaml b/topics/snuba-metrics.yaml index 41edfd70..ec8f1748 100644 --- a/topics/snuba-metrics.yaml +++ b/topics/snuba-metrics.yaml @@ -14,4 +14,5 @@ schemas: examples: - snuba-metrics/1/ topic_creation_config: + compression.type: lz4 message.timestamp.type: LogAppendTime diff --git a/topics/snuba-queries.yaml b/topics/snuba-queries.yaml index f6330380..047d4c5c 100644 --- a/topics/snuba-queries.yaml +++ b/topics/snuba-queries.yaml @@ -13,4 +13,5 @@ schemas: examples: - snuba-queries/1/ topic_creation_config: + compression.type: lz4 max.message.bytes: "2000000" diff --git a/topics/snuba-spans.yaml b/topics/snuba-spans.yaml index 483a7d7f..9dd6fb75 100644 --- a/topics/snuba-spans.yaml +++ b/topics/snuba-spans.yaml @@ -13,3 +13,5 @@ schemas: resource: snuba-spans.v1.schema.json examples: - snuba-spans/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/snuba-transactions-commit-log.yaml b/topics/snuba-transactions-commit-log.yaml index 16f99e6c..6d610a0f 100644 --- a/topics/snuba-transactions-commit-log.yaml +++ b/topics/snuba-transactions-commit-log.yaml @@ -15,5 +15,6 @@ schemas: examples: - snuba-commit-log/1/ topic_creation_config: + compression.type: lz4 cleanup.policy: compact,delete min.compaction.lag.ms: "3600000" diff --git a/topics/transactions-subscription-results.yaml b/topics/transactions-subscription-results.yaml index 34e5acb7..05d45d63 100644 --- a/topics/transactions-subscription-results.yaml +++ b/topics/transactions-subscription-results.yaml @@ -13,3 +13,5 @@ schemas: resource: subscription-results.v1.schema.json examples: - subscription-results/1/ +topic_creation_config: + compression.type: lz4 diff --git a/topics/transactions.yaml b/topics/transactions.yaml index 49d309c0..ac80e9b4 100644 --- a/topics/transactions.yaml +++ b/topics/transactions.yaml @@ -15,5 +15,6 @@ schemas: examples: - transactions/1/ topic_creation_config: + compression.type: lz4 max.message.bytes: "25000000" message.timestamp.type: LogAppendTime