Experimental support for Apollo tracing over OTLP

.changesets/feat_timbotnik_experimental_studio_otlp_tracing.md

@@ -0,0 +1,41 @@
+### Add experimental support for sending traces to Studio via OTLP ([PR #4982](https://github.com/apollographql/router/pull/4982))
+
+As the ecosystem around OpenTelemetry (OTel) has been expanding rapidly, we are evaluating a migration of Apollo's internal
+tracing system to use an OTel-based protocol.
+
+In the short-term, benefits include:
+
+- A comprehensive way to visualize the Router execution path in Studio.
+- Additional spans that were previously not included in Studio traces, such as query parsing, planning, execution, and more.
+- Additional metadata such as subgraph fetch details, Router idle / busy timing, and more.
+
+Long-term, we see this as a strategic enhancement to consolidate these 2 disparate tracing systems.  
+This will pave the way for future enhancements to more easily plug into the Studio trace visualizer.
+
+#### Configuration
+
+This change adds a new configuration option `experimental_otlp_tracing_sampler`. This can be used to send
+a percentage of traces via OTLP instead of the native Apollo Usage Reporting protocol. Supported values:
+
+- `always_off` (default): send all traces via Apollo Usage Reporting protocol.
+- `always_on`: send all traces via OTLP.
+- `0.0 - 1.0`: the ratio of traces to send via OTLP (0.5 = 50 / 50).
+
+Note that this sampler is only applied _after_ the common tracing sampler, for example:
+
+#### Sample 1% of traces, send all traces via OTLP:
+
+```yaml
+telemetry:
+  apollo:
+    # Send all traces via OTLP
+    experimental_otlp_tracing_sampler: always_on
+
+  exporters:
+    tracing:
+      common:
+        # Sample traces at 1% of all traffic
+        sampler: 0.01
+```
+
+By [@timbotnik](https://github.com/timbotnik) in https://github.com/apollographql/router/pull/4982

.gitleaks.toml

@@ -14,6 +14,7 @@
 
         paths = [
             '''^apollo-router\/src\/.+\/testdata\/.+''',
+            '''^apollo-router\/tests\/snapshots\/apollo_otel_traces__.+\.snap$''',
         ]
 
 [[ rules ]]
@@ -97,4 +98,4 @@
         paths = [
             '''^apollo-router\/src\/.+\/testdata\/.+''',
             '''^examples/telemetry/jaeger-collector.router.yaml$'''
-        ]
+        ]

apollo-router/Cargo.toml

@@ -169,6 +169,7 @@ opentelemetry-otlp = { version = "0.13.0", default-features = false, features =
     "http-proto",
     "metrics",
     "reqwest-client",
+    "trace"
 ] }
 opentelemetry-semantic-conventions = "0.12.0"
 opentelemetry-zipkin = { version = "0.18.0", default-features = false, features = [

apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap

@@ -1579,6 +1579,14 @@ expression: "&schema"
           "description": "The Apollo Studio endpoint for exporting traces and metrics.",
           "type": "string"
         },
+        "experimental_otlp_tracing_protocol": {
+          "$ref": "#/definitions/Protocol",
+          "description": "#/definitions/Protocol"
+        },
+        "experimental_otlp_tracing_sampler": {
+          "$ref": "#/definitions/SamplerOption",
+          "description": "#/definitions/SamplerOption"
+        },
         "field_level_instrumentation_sampler": {
           "$ref": "#/definitions/SamplerOption",
           "description": "#/definitions/SamplerOption"

apollo-router/src/metrics/filter.rs

@@ -94,7 +94,7 @@ impl FilterMeterProvider {
             .delegate(delegate)
             .allow(
                 Regex::new(
-                    r"apollo\.(graphos\.cloud|router\.(operations?|lifecycle|config|schema|query|query_planning))(\..*|$)",
+                    r"apollo\.(graphos\.cloud|router\.(operations?|lifecycle|config|schema|query|query_planning|telemetry))(\..*|$)",
                 )
                 .expect("regex should have been valid"),
             )

apollo-router/src/plugins/telemetry/apollo.rs

@@ -3,6 +3,7 @@ use std::collections::HashMap;
 use std::fmt::Display;
 use std::num::NonZeroUsize;
 use std::ops::AddAssign;
+use std::sync::OnceLock;
 use std::time::SystemTime;
 
 use http::header::HeaderName;
@@ -12,10 +13,13 @@ use serde::ser::SerializeMap;
 use serde::Deserialize;
 use serde::Serialize;
 use url::Url;
+use uuid::Uuid;
 
+use super::config::Sampler;
 use super::metrics::apollo::studio::ContextualizedStats;
 use super::metrics::apollo::studio::SingleStats;
 use super::metrics::apollo::studio::SingleStatsReport;
+use super::otlp::Protocol;
 use super::tracing::apollo::TracesReport;
 use crate::plugin::serde::deserialize_header_name;
 use crate::plugin::serde::deserialize_vec_header_name;
@@ -34,6 +38,9 @@ pub(crate) const ENDPOINT_DEFAULT: &str =
 
 pub(crate) const OTLP_ENDPOINT_DEFAULT: &str = "https://usage-reporting.api.apollographql.com";
 
+// Random unique UUID for the Router. This doesn't actually identify the router, it just allows disambiguation between multiple routers with the same metadata.
+pub(crate) static ROUTER_ID: OnceLock<Uuid> = OnceLock::new();
+
 #[derive(Clone, Deserialize, JsonSchema, Debug)]
 #[serde(deny_unknown_fields, default)]
 pub(crate) struct Config {
@@ -69,6 +76,13 @@ pub(crate) struct Config {
     /// Field level instrumentation for subgraphs via ftv1. ftv1 tracing can cause performance issues as it is transmitted in band with subgraph responses.
     pub(crate) field_level_instrumentation_sampler: SamplerOption,
 
+    /// Percentage of traces to send via the OTel protocol when sending to Apollo Studio.
+    pub(crate) experimental_otlp_tracing_sampler: SamplerOption,
+
+    /// OTLP protocol used for OTel traces.
+    /// Note this only applies if OTel traces are enabled and is only intended for use in tests.
+    pub(crate) experimental_otlp_tracing_protocol: Protocol,
+
     /// To configure which request header names and values are included in trace data that's sent to Apollo Studio.
     pub(crate) send_headers: ForwardHeaders,
     /// To configure which GraphQL variable values are included in trace data that's sent to Apollo Studio
@@ -134,6 +148,10 @@ const fn default_field_level_instrumentation_sampler() -> SamplerOption {
     SamplerOption::TraceIdRatioBased(0.01)
 }
 
+const fn default_experimental_otlp_tracing_sampler() -> SamplerOption {
+    SamplerOption::Always(Sampler::AlwaysOff)
+}
+
 fn endpoint_default() -> Url {
     Url::parse(ENDPOINT_DEFAULT).expect("must be valid url")
 }
@@ -167,13 +185,15 @@ impl Default for Config {
         Self {
             endpoint: endpoint_default(),
             experimental_otlp_endpoint: otlp_endpoint_default(),
+            experimental_otlp_tracing_protocol: Protocol::default(),
             apollo_key: apollo_key(),
             apollo_graph_ref: apollo_graph_reference(),
             client_name_header: client_name_header_default(),
             client_version_header: client_version_header_default(),
             schema_id: "<no_schema_id>".to_string(),
             buffer_size: default_buffer_size(),
             field_level_instrumentation_sampler: default_field_level_instrumentation_sampler(),
+            experimental_otlp_tracing_sampler: default_experimental_otlp_tracing_sampler(),
             send_headers: ForwardHeaders::None,
             send_variable_values: ForwardValues::None,
             batch_processor: BatchProcessorConfig::default(),

apollo-router/src/plugins/telemetry/apollo_exporter.rs

@@ -36,7 +36,9 @@ use crate::plugins::telemetry::tracing::BatchProcessorConfig;
 
 const BACKOFF_INCREMENT: Duration = Duration::from_millis(50);
 const ROUTER_REPORT_TYPE_METRICS: &str = "metrics";
-const ROUTER_REPORT_TYPE_TRACES: &str = "traces";
+pub(crate) const ROUTER_REPORT_TYPE_TRACES: &str = "traces";
+const ROUTER_TRACING_PROTOCOL_APOLLO: &str = "apollo";
+pub(crate) const ROUTER_TRACING_PROTOCOL_OTLP: &str = "otlp";
 
 #[derive(thiserror::Error, Debug)]
 pub(crate) enum ApolloExportError {
@@ -301,7 +303,8 @@ impl ApolloExporter {
                             "apollo.router.telemetry.studio.reports",
                             "The number of reports submitted to Studio by the Router",
                             1,
-                            report.type = report_type
+                            report.type = report_type,
+                            report.protocol = ROUTER_TRACING_PROTOCOL_APOLLO
                         );
                         if has_traces && !self.strip_traces.load(Ordering::SeqCst) {
                             // If we had traces then maybe disable sending traces from this exporter based on the response.