feat: Ensure compression.type is specified on all topics

python/tests/test_sentry_kafka_schemas.py

@@ -3,15 +3,12 @@
 
 
 def test_get_topic() -> None:
-    # Topic with creation config
     topic_name = "snuba-queries"
     topic_data = get_topic(topic_name)
-    assert topic_data["topic_creation_config"] == {"max.message.bytes": "2000000"}
-
-    # Topic without creation config
-    topic_name_no_config = "outcomes"
-    topic_data = get_topic(topic_name_no_config)
-    assert topic_data["topic_creation_config"] == {}
+    assert topic_data["topic_creation_config"] == {
+        "compression.type": "lz4",
+        "max.message.bytes": "2000000",
+    }
 
 
 def test_get_schema() -> None:

python/tests/test_valid_topic_data.py

@@ -98,6 +98,13 @@ def test_all_topics() -> None:
             # Check description provided for topic
             assert topic_data["description"]
 
+            # Check every topic has an explicit, valid compression type
+            # Today we use lz4 everywhere, this list can be extended if needed
+            valid_types = ["lz4"]
+            assert (
+                topic_data["topic_creation_config"]["compression.type"] in valid_types
+            )
+
             # Check valid schema versions
             topic_schemas = topic_data["schemas"]
             for i, schema_raw in enumerate(topic_schemas):

topics/events-subscription-results.yaml

@@ -13,3 +13,5 @@ schemas:
     resource: subscription-results.v1.schema.json
     examples:
       - subscription-results/1/
+topic_creation_config:
+  compression.type: lz4

topics/events.yaml

@@ -15,5 +15,6 @@ schemas:
     examples:
       - events/1/
 topic_creation_config:
+  compression.type: lz4
   max.message.bytes: "25000000"
   message.timestamp.type: LogAppendTime

topics/generic-events.yaml

@@ -14,4 +14,5 @@ schemas:
     examples:
       - generic-events/1/
 topic_creation_config:
+  compression.type: lz4
   message.timestamp.type: LogAppendTime

topics/generic-metrics-subscription-results.yaml

@@ -13,3 +13,5 @@ schemas:
     resource: subscription-results.v1.schema.json
     examples:
       - subscription-results/1/
+topic_creation_config:
+  compression.type: lz4

topics/group-attributes.yaml

@@ -13,4 +13,5 @@ schemas:
     examples:
       - group-attributes/1/
 topic_creation_config:
+  compression.type: lz4
   message.timestamp.type: LogAppendTime

topics/ingest-events-dlq.yaml

@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"

topics/ingest-events.yaml

@@ -14,4 +14,5 @@ schemas:
     examples:
       - ingest-events/1/
 topic_creation_config:
+  compression.type: lz4
   max.message.bytes: "10000000"

topics/ingest-generic-metrics-dlq.yaml

@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"

topics/ingest-metrics-dlq.yaml

@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"

topics/ingest-metrics.yaml

@@ -14,3 +14,5 @@ schemas:
     resource: ingest-metrics.v1.schema.json
     examples:
       - ingest-metrics/1/
+topic_creation_config:
+  compression.type: lz4

topics/ingest-performance-metrics.yaml

@@ -13,3 +13,5 @@ schemas:
     resource: ingest-metrics.v1.schema.json
     examples:
       - ingest-metrics/1/
+topic_creation_config:
+  compression.type: lz4

topics/ingest-replay-events.yaml

@@ -15,5 +15,6 @@ schemas:
     examples:
       - ingest-replay-events/1/
 topic_creation_config:
+  compression.type: lz4
   message.timestamp.type: LogAppendTime
   max.message.bytes: "15000000"

topics/ingest-replay-recordings.yaml

@@ -13,3 +13,5 @@ schemas:
     resource: ingest-replay-recordings.v1.schema.json
     examples:
       - ingest-replay-recordings/1/
+topic_creation_config:
+  compression.type: lz4

topics/metrics-subscription-results.yaml

@@ -13,3 +13,5 @@ schemas:
     resource: subscription-results.v1.schema.json
     examples:
       - subscription-results/1/
+topic_creation_config:
+  compression.type: lz4

topics/outcomes-billing.yaml

@@ -16,3 +16,5 @@ schemas:
     resource: outcomes.v1.schema.json
     examples:
       - outcomes/1/
+topic_creation_config:
+  compression.type: lz4

topics/outcomes.yaml

@@ -16,3 +16,5 @@ schemas:
     resource: outcomes.v1.schema.json
     examples:
       - outcomes/1/
+topic_creation_config:
+  compression.type: lz4

topics/processed-profiles.yaml

@@ -14,4 +14,5 @@ schemas:
     examples:
       - profile-metadata/1/
 topic_creation_config:
+  compression.type: lz4
   message.timestamp.type: LogAppendTime

topics/profiles-call-tree.yaml

@@ -14,3 +14,5 @@ schemas:
     resource: profile-functions.v1.schema.json
     examples:
       - profile-functions/1/
+topic_creation_config:
+  compression.type: lz4

topics/sessions-subscription-results.yaml

@@ -13,3 +13,5 @@ schemas:
     resource: subscription-results.v1.schema.json
     examples:
       - subscription-results/1/
+topic_creation_config:
+  compression.type: lz4

topics/shared-resources-usage.yaml

@@ -12,3 +12,5 @@ schemas:
     resource: shared-resources-usage.v1.schema.json
     examples:
       - shared-resources-usage/1/
+topic_creation_config:
+  compression.type: lz4

topics/snuba-commit-log.yaml

@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"

topics/snuba-dead-letter-generic-metrics.yaml

@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"

topics/snuba-dead-letter-group-attributes.yaml

@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"

topics/snuba-dead-letter-metrics.yaml

@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"

topics/snuba-dead-letter-querylog.yaml

@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"

topics/snuba-dead-letter-replays.yaml

@@ -13,4 +13,5 @@ schemas:
     examples:
       - any/
 topic_creation_config:
+  compression.type: lz4
   log.retention.minutes: "10080"

topics/snuba-generic-events-commit-log.yaml

@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"

topics/snuba-generic-metrics-counters-commit-log.yaml

@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"

topics/snuba-generic-metrics-distributions-commit-log.yaml

@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"

topics/snuba-generic-metrics-sets-commit-log.yaml

@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"

topics/snuba-generic-metrics.yaml

@@ -14,4 +14,5 @@ schemas:
     examples:
       - snuba-generic-metrics/1/
 topic_creation_config:
+  compression.type: lz4
   message.timestamp.type: LogAppendTime

topics/snuba-metrics-commit-log.yaml

@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"

topics/snuba-metrics-summaries.yaml

@@ -12,3 +12,5 @@ schemas:
     resource: snuba-metrics-summaries.v1.schema.json
     examples:
       - snuba-metrics-summaries/1/
+topic_creation_config:
+  compression.type: lz4

topics/snuba-metrics.yaml

@@ -14,4 +14,5 @@ schemas:
     examples:
       - snuba-metrics/1/
 topic_creation_config:
+  compression.type: lz4
   message.timestamp.type: LogAppendTime

topics/snuba-queries.yaml

@@ -13,4 +13,5 @@ schemas:
     examples:
       - snuba-queries/1/
 topic_creation_config:
+  compression.type: lz4
   max.message.bytes: "2000000"

topics/snuba-spans.yaml

@@ -13,3 +13,5 @@ schemas:
     resource: snuba-spans.v1.schema.json
     examples:
       - snuba-spans/1/
+topic_creation_config:
+  compression.type: lz4

topics/snuba-transactions-commit-log.yaml

@@ -15,5 +15,6 @@ schemas:
     examples:
       - snuba-commit-log/1/
 topic_creation_config:
+  compression.type: lz4
   cleanup.policy: compact,delete
   min.compaction.lag.ms: "3600000"

topics/transactions-subscription-results.yaml

@@ -13,3 +13,5 @@ schemas:
     resource: subscription-results.v1.schema.json
     examples:
       - subscription-results/1/
+topic_creation_config:
+  compression.type: lz4

topics/transactions.yaml

@@ -15,5 +15,6 @@ schemas:
     examples:
       - transactions/1/
 topic_creation_config:
+  compression.type: lz4
   max.message.bytes: "25000000"
   message.timestamp.type: LogAppendTime