feat: Ensure compression.type is specified on all topics (#234)

getsentry · Mar 12, 2024 · 18932e4 · 18932e4
1 parent e74bfe4
commit 18932e4
Show file tree

Hide file tree

Showing 41 changed files with 64 additions and 7 deletions.
diff --git a/python/tests/test_sentry_kafka_schemas.py b/python/tests/test_sentry_kafka_schemas.py
@@ -3,15 +3,12 @@
 
 
 def test_get_topic() -> None:
-    # Topic with creation config
     topic_name = "snuba-queries"
     topic_data = get_topic(topic_name)
-    assert topic_data["topic_creation_config"] == {"max.message.bytes": "2000000"}
-
-    # Topic without creation config
-    topic_name_no_config = "outcomes"
-    topic_data = get_topic(topic_name_no_config)
-    assert topic_data["topic_creation_config"] == {}
+    assert topic_data["topic_creation_config"] == {
+        "compression.type": "lz4",
+        "max.message.bytes": "2000000",
+    }
 
 
 def test_get_schema() -> None:

diff --git a/python/tests/test_valid_topic_data.py b/python/tests/test_valid_topic_data.py
@@ -98,6 +98,13 @@ def test_all_topics() -> None:
             # Check description provided for topic
             assert topic_data["description"]
 
+            # Check every topic has an explicit, valid compression type
+            # Today we use lz4 everywhere, this list can be extended if needed
+            valid_types = ["lz4"]
+            assert (
+                topic_data["topic_creation_config"]["compression.type"] in valid_types
+            )
+
             # Check valid schema versions
             topic_schemas = topic_data["schemas"]
             for i, schema_raw in enumerate(topic_schemas):

diff --git a/topics/events-subscription-results.yaml b/topics/events-subscription-results.yaml
@@ -13,3 +13,5 @@ schemas:
     resource: subscription-results.v1.schema.json
     examples:
       - subscription-results/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/events.yaml b/topics/events.yaml
@@ -15,5 +15,6 @@ schemas:
     examples:
       - events/1/
 topic_creation_config:
+  compression.type: lz4
   max.message.bytes: "25000000"
   message.timestamp.type: LogAppendTime
diff --git a/topics/generic-events.yaml b/topics/generic-events.yaml
@@ -14,4 +14,5 @@ schemas:
     examples:
       - generic-events/1/
 topic_creation_config:
+  compression.type: lz4
   message.timestamp.type: LogAppendTime
diff --git a/topics/generic-metrics-subscription-results.yaml b/topics/generic-metrics-subscription-results.yaml
@@ -13,3 +13,5 @@ schemas:
     resource: subscription-results.v1.schema.json
     examples:
       - subscription-results/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/group-attributes.yaml b/topics/group-attributes.yaml
@@ -13,4 +13,5 @@ schemas:
     examples:
       - group-attributes/1/
 topic_creation_config:
+  compression.type: lz4
   message.timestamp.type: LogAppendTime
diff --git a/topics/ingest-events-dlq.yaml b/topics/ingest-events-dlq.yaml
@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"
diff --git a/topics/ingest-events.yaml b/topics/ingest-events.yaml
@@ -14,4 +14,5 @@ schemas:
     examples:
       - ingest-events/1/
 topic_creation_config:
+  compression.type: lz4
   max.message.bytes: "10000000"
diff --git a/topics/ingest-generic-metrics-dlq.yaml b/topics/ingest-generic-metrics-dlq.yaml
@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"
diff --git a/topics/ingest-metrics-dlq.yaml b/topics/ingest-metrics-dlq.yaml
@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"
diff --git a/topics/ingest-metrics.yaml b/topics/ingest-metrics.yaml
@@ -14,3 +14,5 @@ schemas:
     resource: ingest-metrics.v1.schema.json
     examples:
       - ingest-metrics/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/ingest-performance-metrics.yaml b/topics/ingest-performance-metrics.yaml
@@ -13,3 +13,5 @@ schemas:
     resource: ingest-metrics.v1.schema.json
     examples:
       - ingest-metrics/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/ingest-replay-events.yaml b/topics/ingest-replay-events.yaml
@@ -15,5 +15,6 @@ schemas:
     examples:
       - ingest-replay-events/1/
 topic_creation_config:
+  compression.type: lz4
   message.timestamp.type: LogAppendTime
   max.message.bytes: "15000000"
diff --git a/topics/ingest-replay-recordings.yaml b/topics/ingest-replay-recordings.yaml
@@ -13,3 +13,5 @@ schemas:
     resource: ingest-replay-recordings.v1.schema.json
     examples:
       - ingest-replay-recordings/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/metrics-subscription-results.yaml b/topics/metrics-subscription-results.yaml
@@ -13,3 +13,5 @@ schemas:
     resource: subscription-results.v1.schema.json
     examples:
       - subscription-results/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/outcomes-billing.yaml b/topics/outcomes-billing.yaml
@@ -16,3 +16,5 @@ schemas:
     resource: outcomes.v1.schema.json
     examples:
       - outcomes/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/outcomes.yaml b/topics/outcomes.yaml
@@ -16,3 +16,5 @@ schemas:
     resource: outcomes.v1.schema.json
     examples:
       - outcomes/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/processed-profiles.yaml b/topics/processed-profiles.yaml
@@ -14,4 +14,5 @@ schemas:
     examples:
       - profile-metadata/1/
 topic_creation_config:
+  compression.type: lz4
   message.timestamp.type: LogAppendTime
diff --git a/topics/profiles-call-tree.yaml b/topics/profiles-call-tree.yaml
@@ -14,3 +14,5 @@ schemas:
     resource: profile-functions.v1.schema.json
     examples:
       - profile-functions/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/sessions-subscription-results.yaml b/topics/sessions-subscription-results.yaml
@@ -13,3 +13,5 @@ schemas:
     resource: subscription-results.v1.schema.json
     examples:
       - subscription-results/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/shared-resources-usage.yaml b/topics/shared-resources-usage.yaml
@@ -12,3 +12,5 @@ schemas:
     resource: shared-resources-usage.v1.schema.json
     examples:
       - shared-resources-usage/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/snuba-commit-log.yaml b/topics/snuba-commit-log.yaml
@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"
diff --git a/topics/snuba-dead-letter-generic-metrics.yaml b/topics/snuba-dead-letter-generic-metrics.yaml
@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"
diff --git a/topics/snuba-dead-letter-group-attributes.yaml b/topics/snuba-dead-letter-group-attributes.yaml
@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"
diff --git a/topics/snuba-dead-letter-metrics.yaml b/topics/snuba-dead-letter-metrics.yaml
@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"
diff --git a/topics/snuba-dead-letter-querylog.yaml b/topics/snuba-dead-letter-querylog.yaml
@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"
diff --git a/topics/snuba-dead-letter-replays.yaml b/topics/snuba-dead-letter-replays.yaml
@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"
diff --git a/topics/snuba-generic-events-commit-log.yaml b/topics/snuba-generic-events-commit-log.yaml
@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"
diff --git a/topics/snuba-generic-metrics-counters-commit-log.yaml b/topics/snuba-generic-metrics-counters-commit-log.yaml
@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"
diff --git a/topics/snuba-generic-metrics-distributions-commit-log.yaml b/topics/snuba-generic-metrics-distributions-commit-log.yaml
@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"
diff --git a/topics/snuba-generic-metrics-sets-commit-log.yaml b/topics/snuba-generic-metrics-sets-commit-log.yaml
@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"
diff --git a/topics/snuba-generic-metrics.yaml b/topics/snuba-generic-metrics.yaml
@@ -14,4 +14,5 @@ schemas:
     examples:
       - snuba-generic-metrics/1/
 topic_creation_config:
+  compression.type: lz4
   message.timestamp.type: LogAppendTime
diff --git a/topics/snuba-metrics-commit-log.yaml b/topics/snuba-metrics-commit-log.yaml
@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"
diff --git a/topics/snuba-metrics-summaries.yaml b/topics/snuba-metrics-summaries.yaml
@@ -12,3 +12,5 @@ schemas:
     resource: snuba-metrics-summaries.v1.schema.json
     examples:
       - snuba-metrics-summaries/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/snuba-metrics.yaml b/topics/snuba-metrics.yaml
@@ -14,4 +14,5 @@ schemas:
     examples:
       - snuba-metrics/1/
 topic_creation_config:
+  compression.type: lz4
   message.timestamp.type: LogAppendTime
diff --git a/topics/snuba-queries.yaml b/topics/snuba-queries.yaml
@@ -13,4 +13,5 @@ schemas:
     examples:
       - snuba-queries/1/
 topic_creation_config:
+  compression.type: lz4
   max.message.bytes: "2000000"
diff --git a/topics/snuba-spans.yaml b/topics/snuba-spans.yaml
@@ -13,3 +13,5 @@ schemas:
     resource: snuba-spans.v1.schema.json
     examples:
       - snuba-spans/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/snuba-transactions-commit-log.yaml b/topics/snuba-transactions-commit-log.yaml
@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"
diff --git a/topics/transactions-subscription-results.yaml b/topics/transactions-subscription-results.yaml
@@ -13,3 +13,5 @@ schemas:
     resource: subscription-results.v1.schema.json
     examples:
       - subscription-results/1/
+topic_creation_config:
+  compression.type: lz4
diff --git a/topics/transactions.yaml b/topics/transactions.yaml
@@ -15,5 +15,6 @@ schemas:
     examples:
       - transactions/1/
 topic_creation_config:
+  compression.type: lz4
   max.message.bytes: "25000000"
   message.timestamp.type: LogAppendTime