Router panic in query planning on 1.43.2

[machados]

Describe the bug

A very low TPS perf test causes the router to panic after upgrading from 1.38.0 to 1.43.2 last week:

{"timestamp":"2024-04-05T20:20:13.407796000Z","level":"WARN","message":"RUST_BACKTRACE=full detected. This is useful for diagnostics but will have a performance impact and may leak sensitive information","target":"apollo_router::executable","resource":{}}
{"timestamp":"2024-04-05T20:20:13.409159000Z","level":"INFO","message":"Apollo Router v1.43.2 // (c) Apollo Graph, Inc. // Licensed as ELv2 (https://go.apollo.dev/elv2)","target":"apollo_router::executable","resource":{}}
{"timestamp":"2024-04-05T20:20:13.409193000Z","level":"INFO","message":"Anonymous usage data collection is disabled.","target":"apollo_router::executable","resource":{}}
{"timestamp":"2024-04-05T20:20:13.742366000Z","level":"WARN","trace_id":"00000000000000000000000000000000","span_id":"0000000000000000","message":"telemetry.instrumentation.spans.mode is currently set to 'deprecated', either explicitly or via defaulting. Set telemetry.instrumentation.spans.mode explicitly in your router.yaml to 'spec_compliant' for log and span attributes that follow OpenTelemetry semantic conventions. This option will be defaulted to 'spec_compliant' in a future release and eventually removed altogether","target":"apollo_router::plugins::telemetry","spans":[{"name":"starting"},{"name":"plugins"},{"name":"plugin: apollo.telemetry"}],"resource":{}}
{"timestamp":"2024-04-05T20:20:13.944742000Z","level":"INFO","message":"Health check exposed at http://127.0.0.1:8088/health","target":"apollo_router::axum_factory::axum_http_server_factory","resource":{"process.executable.name":"router","service.name":"unknown_service:router","service.version":"1.43.2"}}
{"timestamp":"2024-04-05T20:20:13.946265000Z","level":"INFO","message":"GraphQL endpoint exposed at http://127.0.0.1:4000/ 🚀","target":"apollo_router::axum_factory::axum_http_server_factory","resource":{"process.executable.name":"router","service.name":"unknown_service:router","service.version":"1.43.2"}}
{"timestamp":"2024-04-05T20:20:34.267848000Z","level":"ERROR","message":"panicked at /Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/context/extensions/sync.rs:66:21:\nExtensionsGuard held for 25ms. This is probably a bug that will stall the Router and cause performance problems. Run with `RUST_BACKTRACE=1` environment variable to display a backtrace\nBacktrace [{ fn: \"std::backtrace_rs::backtrace::libunwind::trace\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/../../backtrace/src/backtrace/libunwind.rs\", line: 104 }, 
{ fn: \"std::backtrace_rs::backtrace::trace_unsynchronized\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/../../backtrace/src/backtrace/mod.rs\", line: 66 },
 { fn: \"std::backtrace::Backtrace::create\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/backtrace.rs\", line: 331 }, 
{ fn: \"apollo_router::executable::setup_panic_handler::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/executable.rs\", line: 714 }, 
{ fn: \"<alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/alloc/src/boxed.rs\", line: 2029 }, 
{ fn: \"std::panicking::rust_panic_with_hook\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs\", line: 783 },
 { fn: \"std::panicking::begin_panic_handler::{{closure}}\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs\", line: 657 }, 
{ fn: \"std::sys_common::backtrace::__rust_end_short_backtrace\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/sys_common/backtrace.rs\", line: 171 }, 
{ fn: \"rust_begin_unwind\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs\", line: 645 },
 { fn: \"core::panicking::panic_fmt\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/panicking.rs\", line: 72 }, 
{ fn: \"<apollo_router::context::extensions::sync::ExtensionsGuard as core::ops::drop::Drop>::drop\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/context/extensions/sync.rs\", line: 66 }, 
{ fn: \"core::ptr::drop_in_place<apollo_router::context::extensions::sync::ExtensionsGuard>\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/ptr/mod.rs\", line: 507 }, 
{ fn: \"apollo_router::query_planner::caching_query_planner::CachingQueryPlanner<T>::plan::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/query_planner/caching_query_planner.rs\", line: 442 },
 { fn: \"<apollo_router::query_planner::caching_query_planner::CachingQueryPlanner<T> as tower_service::Service<apollo_router::services::query_planner::CachingRequest>>::call::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/query_planner/caching_query_planner.rs\", line: 264 },
 { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<tracing_futures::Instrumented<T> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tracing-futures-0.2.5/src/lib.rs\", line: 283 }, { fn: \"apollo_router::services::supergraph::service::plan_query::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/services/supergraph/service.rs\", line: 619 }, { fn: \"apollo_router::services::supergraph::service::service_call::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/services/supergraph/service.rs\", line: 183 }, { fn: \"<F as futures_core::future::TryFuture>::try_poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-core-0.3.30/src/future.rs\", line: 82 }, { fn: \"<futures_util::future::try_future::into_future::IntoFuture<Fut> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/future/try_future/into_future.rs\", line: 34 }, { fn: \"<futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/future/future/map.rs\", line: 55 }, { fn: \"<futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/lib.rs\", line: 91 }, { fn: \"<futures_util::future::try_future::MapErr<Fut,F> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/lib.rs\", line: 91 }, { fn: \"<F as futures_core::future::TryFuture>::try_poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-core-0.3.30/src/future.rs\", line: 82 }, { fn: \"<futures_util::future::try_future::try_flatten_err::TryFlattenErr<Fut,<Fut as futures_core::future::TryFuture>::Error> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/future/try_future/try_flatten_err.rs\", line: 46 }, { fn: \"<futures_util::future::try_future::TryFlattenErr<Fut1,Fut2> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/lib.rs\", line: 91 }, { fn: \"<futures_util::future::try_future::OrElse<Fut1,Fut2,F> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/lib.rs\", line: 91 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<tower::util::either::Either<A,B> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tower-0.4.13/src/util/either.rs\", line: 71 }, { fn: \"<tower::util::oneshot::Oneshot<S,Req> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tower-0.4.13/src/util/oneshot.rs\", line: 97 }, { fn: \"<apollo_router::plugins::traffic_shaping::timeout::future::ResponseFuture<F> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/plugins/traffic_shaping/timeout/future.rs\", line: 43 }, { fn: \"<tower::util::oneshot::Oneshot<S,Req> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tower-0.4.13/src/util/oneshot.rs\", line: 97 }, { fn: \"<apollo_router::layers::async_checkpoint::OneShotAsyncCheckpointService<S,Fut,Request> as tower_service::Service<Request>>::call::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/layers/async_checkpoint.rs\", line: 139 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<apollo_router::plugins::telemetry::Telemetry as apollo_router::plugin::Plugin>::supergraph_service::{{closure}}::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/plugins/telemetry/mod.rs\", line: 514 }, { fn: \"<F as futures_core::future::TryFuture>::try_poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-core-0.3.30/src/future.rs\", line: 82 }, { fn: \"<futures_util::future::try_future::into_future::IntoFuture<Fut> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/future/try_future/into_future.rs\", line: 34 }, { fn: \"<futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/future/future/map.rs\", line: 55 }, { fn: \"<futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/lib.rs\", line: 91 }, { fn: \"<futures_util::future::try_future::MapOk<Fut,F> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/lib.rs\", line: 91 }, { fn: \"<tower::util::map_response::MapResponseFuture<F,N> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tower-0.4.13/src/macros.rs\", line: 38 }, { fn: \"<tracing::instrument::Instrumented<T> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tracing-0.1.40/src/instrument.rs\", line: 321 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<apollo_router::layers::map_first_graphql_response::MapFirstGraphqlResponseService<InnerService,Callback> as tower_service::Service<Request>>::call::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/layers/map_first_graphql_response.rs\", line: 70 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<tower::util::oneshot::Oneshot<S,Req> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tower-0.4.13/src/util/oneshot.rs\", line: 97 }, { fn: \"apollo_router::services::router::service::RouterService::process_supergraph_request::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/services/router/service.rs\", line: 250 }, { fn: \"<futures_util::future::maybe_done::MaybeDone<Fut> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/future/maybe_done.rs\", line: 95 }, { fn: \"<futures_util::future::join_all::JoinAll<F> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/future/join_all.rs\", line: 143 }, { fn: \"apollo_router::services::router::service::RouterService::call_inner::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/services/router/service.rs\", line: 434 }, { fn: \"<apollo_router::services::router::service::RouterService as tower_service::Service<apollo_router::services::router::Request>>::call::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/services/router/service.rs\", line: 222 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<apollo_router::plugins::telemetry::Telemetry as apollo_router::plugin::Plugin>::router_service::{{closure}}::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/apollo-router-1.43.2/src/plugins/telemetry/mod.rs\", line: 376 }, { fn: \"<tracing::instrument::Instrumented<T> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tracing-0.1.40/src/instrument.rs\", line: 321 }, { fn: \"<tower::util::either::Either<A,B> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tower-0.4.13/src/util/either.rs\", line: 70 }, { fn: \"<F as futures_core::future::TryFuture>::try_poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-core-0.3.30/src/future.rs\", line: 82 }, { fn: \"<futures_util::future::try_future::into_future::IntoFuture<Fut> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/future/try_future/into_future.rs\", line: 34 }, { fn: \"<futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/future/future/map.rs\", line: 55 }, { fn: \"<futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/lib.rs\", line: 91 }, { fn: \"<futures_util::future::try_future::MapOk<Fut,F> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.30/src/lib.rs\", line: 91 }, { fn: \"<tower::util::map_response::MapResponseFuture<F,N> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tower-0.4.13/src/macros.rs\", line: 38 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<core::pin::Pin<P> as core::future::future::Future>::poll\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/future/future.rs\", line: 124 }, { fn: \"<tower::util::oneshot::Oneshot<S,Req> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tower-0.4.13/src/util/oneshot.rs\", line: 97 }, { fn: \"<tracing::instrument::WithDispatch<T> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tracing-0.1.40/src/instrument.rs\", line: 388 }, { fn: \"<tracing::instrument::Instrumented<T> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tracing-0.1.40/src/instrument.rs\", line: 321 }, { fn: \"tokio::runtime::task::core::Core<T,S>::poll::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/core.rs\", line: 328 }, { fn: \"tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/loom/std/unsafe_cell.rs\", line: 16 }, { fn: \"tokio::runtime::task::core::Core<T,S>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/core.rs\", line: 317 }, { fn: \"tokio::runtime::task::harness::poll_future::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/harness.rs\", line: 485 }, { fn: \"<core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/panic/unwind_safe.rs\", line: 272 }, { fn: \"std::panicking::try::do_call\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs\", line: 552 }, { fn: \"___rust_try\" }, { fn: \"std::panicking::try\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs\", line: 516 }, { fn: \"std::panic::catch_unwind\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panic.rs\", line: 142 }, { fn: \"tokio::runtime::task::harness::poll_future\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/harness.rs\", line: 473 }, { fn: \"tokio::runtime::task::harness::Harness<T,S>::poll_inner\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/harness.rs\", line: 208 }, { fn: \"tokio::runtime::task::harness::Harness<T,S>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/harness.rs\", line: 153 }, { fn: \"tokio::runtime::task::raw::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/raw.rs\", line: 271 }, { fn: \"tokio::runtime::task::raw::RawTask::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/raw.rs\", line: 201 }, { fn: \"tokio::runtime::task::LocalNotified<S>::run\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/mod.rs\", line: 427 }, { fn: \"tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs\", line: 576 }, { fn: \"tokio::runtime::coop::with_budget\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/coop.rs\", line: 107 }, { fn: \"tokio::runtime::coop::budget\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/coop.rs\", line: 73 }, { fn: \"tokio::runtime::scheduler::multi_thread::worker::Context::run_task\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs\", line: 575 }, { fn: \"tokio::runtime::scheduler::multi_thread::worker::Context::run\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs\", line: 526 }, { fn: \"tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs\", line: 491 }, { fn: \"tokio::runtime::context::scoped::Scoped<T>::set\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/context/scoped.rs\", line: 40 }, { fn: \"tokio::runtime::context::set_scheduler::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/context.rs\", line: 176 }, { fn: \"std::thread::local::LocalKey<T>::try_with\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/thread/local.rs\", line: 270 }, { fn: \"std::thread::local::LocalKey<T>::with\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/thread/local.rs\", line: 246 }, { fn: \"tokio::runtime::context::set_scheduler\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/context.rs\", line: 176 }, { fn: \"tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs\", line: 486 }, { fn: \"tokio::runtime::context::runtime::enter_runtime\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/context/runtime.rs\", line: 65 }, { fn: \"tokio::runtime::scheduler::multi_thread::worker::run\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs\", line: 478 }, { fn: \"tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs\", line: 447 }, { fn: \"<tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/blocking/task.rs\", line: 42 }, { fn: \"tokio::runtime::task::core::Core<T,S>::poll::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/core.rs\", line: 328 }, { fn: \"tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/loom/std/unsafe_cell.rs\", line: 16 }, { fn: \"tokio::runtime::task::core::Core<T,S>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/core.rs\", line: 317 }, { fn: \"tokio::runtime::task::harness::poll_future::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/harness.rs\", line: 485 }, { fn: \"<core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/panic/unwind_safe.rs\", line: 272 }, { fn: \"std::panicking::try::do_call\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs\", line: 552 }, { fn: \"___rust_try\" }, { fn: \"std::panicking::try\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs\", line: 516 }, { fn: \"std::panic::catch_unwind\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panic.rs\", line: 142 }, { fn: \"tokio::runtime::task::harness::poll_future\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/harness.rs\", line: 473 }, { fn: \"tokio::runtime::task::harness::Harness<T,S>::poll_inner\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/harness.rs\", line: 208 }, { fn: \"tokio::runtime::task::harness::Harness<T,S>::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/harness.rs\", line: 153 }, { fn: \"tokio::runtime::task::raw::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/raw.rs\", line: 271 }, { fn: \"tokio::runtime::task::raw::RawTask::poll\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/raw.rs\", line: 201 }, { fn: \"tokio::runtime::task::UnownedTask<S>::run\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/task/mod.rs\", line: 464 }, { fn: \"tokio::runtime::blocking::pool::Task::run\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/blocking/pool.rs\", line: 159 }, { fn: \"tokio::runtime::blocking::pool::Inner::run\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/blocking/pool.rs\", line: 513 }, { fn: \"tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\", file: \"/Users/a/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.37.0/src/runtime/blocking/pool.rs\", line: 471 }, { fn: \"std::sys_common::backtrace::__rust_begin_short_backtrace\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/sys_common/backtrace.rs\", line: 155 }, { fn: \"std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/thread/mod.rs\", line: 529 }, { fn: \"<core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/panic/unwind_safe.rs\", line: 272 }, { fn: \"std::panicking::try::do_call\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs\", line: 552 }, { fn: \"___rust_try\" }, { fn: \"std::panicking::try\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs\", line: 516 }, { fn: \"std::panic::catch_unwind\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panic.rs\", line: 142 }, { fn: \"std::thread::Builder::spawn_unchecked_::{{closure}}\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/thread/mod.rs\", line: 528 }, { fn: \"core::ops::function::FnOnce::call_once{{vtable.shim}}\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/ops/function.rs\", line: 250 }, { fn: \"<alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/alloc/src/boxed.rs\", line: 2015 }, { fn: \"<alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/alloc/src/boxed.rs\", line: 2015 }, { fn: \"std::sys::unix::thread::Thread::new::thread_start\", file: \"/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/sys/unix/thread.rs\", line: 108 }, { fn: \"__pthread_joiner_wake\" }]","target":"apollo_router::executable","spans":[{"http.request.method":"POST","http.method":"POST","http.route":"/","otel.kind":"SERVER","http.flavor":"HTTP/1.1","name":"request"},{"http.request.method":"POST","http.method":"POST","trace_id":"","http.route":"/","http.flavor":"HTTP/1.1","otel.kind":"INTERNAL","http.request.method":"POST","http.route":"/","url.path":"/","client.name":"","client.version":"","name":"router"},{"graphql.document":"   \n    query {\n        todoList {\n            id\n            text\n            done\n        }\n    }","otel.kind":"INTERNAL","name":"supergraph"},{"otel.kind":"INTERNAL","name":"query_planning"}],"resource":{"process.executable.name":"router","service.name":"unknown_service:router","service.version":"1.43.2"}}

I'm using the plugin development scaffolding. The crash happens with our without my plugin enabled.

This happens just a few seconds after the perf test starts. The initial requests are processed but then it panics.

To Reproduce
Steps to reproduce the behavior:

I'm using rust 1.76.0 toolchain and running in MacOS 14.4.1.

Router configuration (I have tried also commenting the query_planning options and it still panics):

supergraph:
  listen: 127.0.0.1:4000
  introspection: true
  query_planning:
    warmed_up_queries: 100
    cache:
      in_memory:
        limit: 512
cors:
  allow_any_origin: true
homepage:
  enabled: false
health_check:
  listen: 127.0.0.1:8088
headers:
  all:
    request:
      - propagate:
          matching: .*
include_subgraph_errors:
  all: true
subscription:
  enabled: false

Supergraph:

schema
  @core(feature: "https://specs.apollo.dev/core/v0.2"),
  @core(feature: "https://specs.apollo.dev/join/v0.1", for: EXECUTION)
  @core(feature: "https://specs.apollo.dev/inaccessible/v0.1", for: SECURITY)
{
  query: Query
}

directive @core(as: String, feature: String!, for: core__Purpose) repeatable on SCHEMA

directive @join__field(graph: join__Graph, requires: join__FieldSet, provides: join__FieldSet) on FIELD_DEFINITION

directive @join__type(graph: join__Graph!, key: join__FieldSet) repeatable on OBJECT | INTERFACE

directive @join__owner(graph: join__Graph!) on OBJECT | INTERFACE

directive @join__graph(name: String!, url: String!) on ENUM_VALUE

directive @tag(name: String!) repeatable on FIELD_DEFINITION | INTERFACE | OBJECT | UNION

directive @inaccessible on OBJECT | FIELD_DEFINITION | INTERFACE | UNION

enum core__Purpose {
  """
  `EXECUTION` features provide metadata necessary to for operation execution.
  """
  EXECUTION

  """
  `SECURITY` features provide metadata necessary to securely resolve fields.
  """
  SECURITY
}

scalar join__FieldSet

enum join__Graph {
  ACCOUNTS @join__graph(name: "accounts", url: "http://127.0.0.1:8000")
  TODO @join__graph(name: "todo", url: "http://127.0.0.1:8000/todo/graphql")
}

type Query {
  me: User @join__field(graph: ACCOUNTS)
  todoList: [Todo] @join__field(graph: TODO)
}

type User
  @join__owner(graph: ACCOUNTS)
  @join__type(graph: ACCOUNTS, key: "id")
{
  id: ID! @join__field(graph: ACCOUNTS)
  name: String @join__field(graph: ACCOUNTS)
  username: String @join__field(graph: ACCOUNTS)
}

type Todo
  @join__owner(graph: TODO)
  @join__type(graph: TODO, key: "id")
{
  id: ID! @join__field(graph: TODO)
  text: String @join__field(graph: TODO)
  done: Boolean @join__field(graph: TODO)
}

Start command:

cargo run --package gql-gateway --bin router \
    -- \
    --anonymous-telemetry-disabled \
    --config router.yaml \
    --supergraph ./supergraph-todo-local.graphql \
    --hot-reload \
    --apollo-uplink-endpoints ''

The query is:

query {
        todoList {
            id
            text
            done
        }
    }

I run the load from a k6 script.

Expected behavior
The router should not panic.

Output
See above.

Desktop (please complete the following information):

• OS: MacOS
• Version: 14.4.1

Additional context
NA

[machados]

UPDATE: if I add --release to the cargo run command I don't get the panic:

cargo run --release --package gql-gateway --bin router \
    -- \
    --anonymous-telemetry-disabled \
    --config router.yaml \
    --supergraph ./supergraph-todo-local.graphql \
    --hot-reload \
    --apollo-uplink-endpoints ''

[machados]

I guess this is expected since the panic happens because there is a call to panic. This didn't happen before though and the message indicates that this might be due to a bug.