Experimental support for Apollo tracing over OTLP

apollo-router/Cargo.toml

@@ -164,6 +164,7 @@ opentelemetry-otlp = { version = "0.13.0", default-features = false, features =
     "http-proto",
     "metrics",
     "reqwest-client",
+    "trace"
 ] }
 opentelemetry-semantic-conventions = "0.12.0"
 opentelemetry-zipkin = { version = "0.18.0", default-features = false, features = [

apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap

@@ -1,5 +1,6 @@
 ---
 source: apollo-router/src/configuration/tests.rs
+assertion_line: 31
 expression: "&schema"
 ---
 {
@@ -3213,6 +3214,32 @@ expression: "&schema"
               "default": "https://usage-reporting.api.apollographql.com/",
               "type": "string"
             },
+            "experimental_tracing_protocol": {
+              "description": "The protocol used for sending traces to Apollo Studio.",
+              "oneOf": [
+                {
+                  "description": "Use only the Apollo usage reporting protobuf over http",
+                  "type": "string",
+                  "enum": [
+                    "apollo"
+                  ]
+                },
+                {
+                  "description": "Use only OTLP over GRPC",
+                  "type": "string",
+                  "enum": [
+                    "otlp"
+                  ]
+                },
+                {
+                  "description": "Use both the Apollo usage reporting protobuf AND OTLP (note this is a testing mode and not intended for use in production)",
+                  "type": "string",
+                  "enum": [
+                    "apollo_and_otlp"
+                  ]
+                }
+              ]
+            },
             "field_level_instrumentation_sampler": {
               "description": "Field level instrumentation for subgraphs via ftv1. ftv1 tracing can cause performance issues as it is transmitted in band with subgraph responses.",
               "anyOf": [

apollo-router/src/plugins/telemetry/apollo.rs

@@ -3,6 +3,7 @@ use std::collections::HashMap;
 use std::fmt::Display;
 use std::num::NonZeroUsize;
 use std::ops::AddAssign;
+use std::sync::OnceLock;
 use std::time::SystemTime;
 
 use http::header::HeaderName;
@@ -12,6 +13,7 @@ use serde::ser::SerializeMap;
 use serde::Deserialize;
 use serde::Serialize;
 use url::Url;
+use uuid::Uuid;
 
 use super::metrics::apollo::studio::ContextualizedStats;
 use super::metrics::apollo::studio::SingleStats;
@@ -34,6 +36,9 @@ pub(crate) const ENDPOINT_DEFAULT: &str =
 
 pub(crate) const OTLP_ENDPOINT_DEFAULT: &str = "https://usage-reporting.api.apollographql.com";
 
+// Random unique UUID for the Router. This doesn't actually identify the router, it just allows disambiguation between multiple routers with the same metadata.
+pub(crate) static ROUTER_ID: OnceLock<Uuid> = OnceLock::new();
+
 #[derive(Clone, Deserialize, JsonSchema, Debug)]
 #[serde(deny_unknown_fields, default)]
 pub(crate) struct Config {
@@ -69,6 +74,9 @@ pub(crate) struct Config {
     /// Field level instrumentation for subgraphs via ftv1. ftv1 tracing can cause performance issues as it is transmitted in band with subgraph responses.
     pub(crate) field_level_instrumentation_sampler: SamplerOption,
 
+    /// The protocol used for sending traces to Apollo Studio.
+    pub(crate) experimental_tracing_protocol: ApolloTracingProtocol,
+
     /// To configure which request header names and values are included in trace data that's sent to Apollo Studio.
     pub(crate) send_headers: ForwardHeaders,
     /// To configure which GraphQL variable values are included in trace data that's sent to Apollo Studio
@@ -174,6 +182,7 @@ impl Default for Config {
             schema_id: "<no_schema_id>".to_string(),
             buffer_size: default_buffer_size(),
             field_level_instrumentation_sampler: default_field_level_instrumentation_sampler(),
+            experimental_tracing_protocol: ApolloTracingProtocol::Apollo,
             send_headers: ForwardHeaders::None,
             send_variable_values: ForwardValues::None,
             batch_processor: BatchProcessorConfig::default(),
@@ -182,6 +191,18 @@ impl Default for Config {
     }
 }
 
+#[derive(Copy, Clone, Debug, Deserialize, JsonSchema, PartialEq)]
+#[serde(deny_unknown_fields, rename_all = "snake_case")]
+pub(crate) enum ApolloTracingProtocol {
+    /// Use only the Apollo usage reporting protobuf over http
+    Apollo,
+    /// Use only OTLP over GRPC
+    Otlp,
+    /// Use both the Apollo usage reporting protobuf AND OTLP
+    /// (note this is a testing mode and not intended for use in production)
+    ApolloAndOtlp,
+}
+
 schemar_fn!(
     forward_headers_only,
     Vec<String>,

apollo-router/src/plugins/telemetry/apollo_otlp_exporter.rs

@@ -0,0 +1,140 @@
+use std::borrow::Cow;
+use std::sync::Arc;
+use std::sync::Mutex;
+
+use derivative::Derivative;
+use futures::future::BoxFuture;
+use opentelemetry::sdk::export::trace::ExportResult;
+use opentelemetry::sdk::export::trace::SpanData;
+use opentelemetry::sdk::export::trace::SpanExporter;
+use opentelemetry::sdk::trace::EvictedQueue;
+use opentelemetry::sdk::Resource;
+use opentelemetry::trace::SpanContext;
+use opentelemetry::trace::Status;
+use opentelemetry::trace::TraceFlags;
+use opentelemetry::trace::TraceState;
+use opentelemetry::InstrumentationLibrary;
+use opentelemetry::KeyValue;
+use opentelemetry_otlp::SpanExporterBuilder;
+use opentelemetry_otlp::WithExportConfig;
+use sys_info::hostname;
+use tonic::metadata::MetadataMap;
+use tonic::metadata::MetadataValue;
+use tower::BoxError;
+use url::Url;
+use uuid::Uuid;
+
+use super::tracing::apollo_telemetry::LightSpanData;
+use crate::plugins::telemetry::apollo::ROUTER_ID;
+use crate::plugins::telemetry::apollo_exporter::get_uname;
+use crate::plugins::telemetry::tracing::BatchProcessorConfig;
+use crate::plugins::telemetry::GLOBAL_TRACER_NAME;
+
+/// The Apollo Otlp exporter is a thin wrapper around the OTLP SpanExporter.
+#[derive(Clone, Derivative)]
+#[derivative(Debug)]
+pub(crate) struct ApolloOtlpExporter {
+    batch_config: BatchProcessorConfig,
+    endpoint: Url,
+    apollo_key: String,
+    resource_template: Resource,
+    intrumentation_library: InstrumentationLibrary,
+    #[derivative(Debug = "ignore")]
+    otlp_exporter: Arc<Mutex<opentelemetry_otlp::SpanExporter>>,
+}
+
+impl ApolloOtlpExporter {
+    pub(crate) fn new(
+        endpoint: &Url,
+        batch_config: &BatchProcessorConfig,
+        apollo_key: &str,
+        apollo_graph_ref: &str,
+        schema_id: &str,
+    ) -> Result<ApolloOtlpExporter, BoxError> {
+        tracing::debug!(endpoint = %endpoint, "creating Apollo OTLP traces exporter");
+
+        let mut metadata = MetadataMap::new();
+        metadata.insert("apollo.api.key", MetadataValue::try_from(apollo_key)?);
+
+        return Ok(Self {
+            endpoint: endpoint.clone(),
+            batch_config: batch_config.clone(),
+            apollo_key: apollo_key.to_string(),
+            resource_template: Resource::new([
+                KeyValue::new(
+                    "apollo.router.id",
+                    ROUTER_ID.get_or_init(Uuid::new_v4).to_string(),
+                ),
+                KeyValue::new("apollo.graph.ref", apollo_graph_ref.to_string()),
+                KeyValue::new("apollo.schema.id", schema_id.to_string()),
+                KeyValue::new(
+                    "apollo.user.agent",
+                    format!(
+                        "{}@{}",
+                        std::env!("CARGO_PKG_NAME"),
+                        std::env!("CARGO_PKG_VERSION")
+                    ),
+                ),
+                KeyValue::new("apollo.client.host", hostname()?),
+                KeyValue::new("apollo.client.uname", get_uname()?),
+            ]),
+            intrumentation_library: InstrumentationLibrary::new(
+                GLOBAL_TRACER_NAME,
+                Some(format!(
+                    "{}@{}",
+                    std::env!("CARGO_PKG_NAME"),
+                    std::env!("CARGO_PKG_VERSION")
+                )),
+                Option::<String>::None,
+                None,
+            ),
+            otlp_exporter: Arc::new(Mutex::new(
+                SpanExporterBuilder::from(
+                    opentelemetry_otlp::new_exporter()
+                        .tonic()
+                        .with_timeout(batch_config.max_export_timeout)
+                        .with_endpoint(endpoint.to_string())
+                        .with_metadata(metadata),
+                        // TBD(tim): figure out why compression seems to be turned off on our collector
+                        // .with_compression(opentelemetry_otlp::Compression::Gzip),
+                )
+                .build_span_exporter()?,
+            )),
+            // TBD(tim): do we need another batch processor for this?
+            // Seems like we've already set up a batcher earlier in the pipe but not quite sure.
+        });
+    }
+
+    pub(crate) fn prepare_for_export(&self, span: &LightSpanData) -> SpanData {
+        SpanData {
+            span_context: SpanContext::new(
+                span.trace_id,
+                span.span_id,
+                TraceFlags::default().with_sampled(true),
+                true,
+                TraceState::default(),
+            ),
+            parent_span_id: span.parent_span_id,
+            span_kind: span.span_kind.clone(),
+            name: span.name.clone(),
+            start_time: span.start_time,
+            end_time: span.end_time,
+            attributes: span.attributes.clone(),
+            events: EvictedQueue::new(0),
+            links: EvictedQueue::new(0),
+            status: Status::Unset,
+            resource: Cow::Owned(self.resource_template.to_owned()),
+            instrumentation_lib: self.intrumentation_library.clone(),
+        }
+    }
+
+    pub(crate) fn export(&self, spans: Vec<SpanData>) -> BoxFuture<'static, ExportResult> {
+        let mut exporter = self.otlp_exporter.lock().unwrap();
+        exporter.export(spans)
+    }
+
+    pub(crate) fn shutdown(&self) {
+        let mut exporter = self.otlp_exporter.lock().unwrap();
+        exporter.shutdown()
+    }
+}

apollo-router/src/plugins/telemetry/metrics/apollo.rs

@@ -1,7 +1,6 @@
 //! Apollo metrics
 use std::sync::atomic::AtomicBool;
 use std::sync::atomic::Ordering;
-use std::sync::OnceLock;
 use std::time::Duration;
 
 use opentelemetry::runtime;
@@ -17,6 +16,7 @@ use url::Url;
 use uuid::Uuid;
 
 use crate::plugins::telemetry::apollo::Config;
+use crate::plugins::telemetry::apollo::ROUTER_ID;
 use crate::plugins::telemetry::apollo_exporter::get_uname;
 use crate::plugins::telemetry::apollo_exporter::ApolloExporter;
 use crate::plugins::telemetry::config::MetricsCommon;
@@ -35,9 +35,6 @@ fn default_buckets() -> Vec<f64> {
     ]
 }
 
-// Random unique UUID for the Router. This doesn't actually identify the router, it just allows disambiguation between multiple routers with the same metadata.
-static ROUTER_ID: OnceLock<Uuid> = OnceLock::new();
-
 impl MetricsConfigurator for Config {
     fn enabled(&self) -> bool {
         self.apollo_key.is_some() && self.apollo_graph_ref.is_some()

apollo-router/src/plugins/telemetry/mod.rs

@@ -129,6 +129,7 @@ use crate::ListenAddr;
 
 pub(crate) mod apollo;
 pub(crate) mod apollo_exporter;
+pub(crate) mod apollo_otlp_exporter;
 pub(crate) mod config;
 pub(crate) mod config_new;
 pub(crate) mod dynamic_attribute;

apollo-router/src/plugins/telemetry/tracing/apollo.rs

@@ -26,6 +26,8 @@ impl TracingConfigurator for Config {
         tracing::debug!("configuring Apollo tracing");
         let exporter = apollo_telemetry::Exporter::builder()
             .endpoint(&self.endpoint)
+            .otlp_endpoint(&self.experimental_otlp_endpoint)
+            .apollo_tracing_protocol(self.experimental_tracing_protocol)
             .apollo_key(
                 self.apollo_key
                     .as_ref()