assertion failed: self.num_remote_reset_streams > 0
jeromegn opened this issue · comments
We are running h2
v0.3.18.
Here's the backtrace from Sentry:
assertion failed: self.num_remote_reset_streams > 0
Thread 0 Crashed:
0 std 0x55c96d2f6c5c std::sys_common::backtrace::__rust_end_short_backtrace (backtrace.rs:137)
1 <unknown> 0x55c96d2f8142 rust_begin_unwind (panicking.rs:575)
2 core 0x55c96d31dc13 core::panicking::panic_fmt (panicking.rs:64)
3 core 0x55c96d31dced core::panicking::panic (panicking.rs:111)
4 <unknown> 0x55c96cf4afff h2[c00baf226614d5c]::proto::streams::counts::Counts::dec_num_remote_reset_streams (counts.rs:137)
5 <unknown> 0x55c96c61f83e [inlined] h2[c00baf226614d5c]::proto::streams::streams::Streams<T>::next_incoming::{closure#0} (streams.rs:146)
6 <unknown> 0x55c96c61f83e [inlined] core[cf0a937d1125fa47]::option::Option<T>::map:: (option.rs:969)
7 <unknown> 0x55c96c61f83e h2[c00baf226614d5c]::proto::streams::streams::Streams<T>::next_incoming (streams.rs:133)
8 <unknown> 0x55c96c245f64 [inlined] <h2[c00baf226614d5c]::proto::connection::Connection<hyper[39e87571ecf802b8]::common::io::rewind::Rewind<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]... (connection.rs:552)
9 <unknown> 0x55c96c245f64 <h2[c00baf226614d5c]::server::Connection<hyper[39e87571ecf802b8]::common::io::rewind::Rewind<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Bo... (server.rs:426)
10 <unknown> 0x55c96c43f29d [inlined] <hyper[39e87571ecf802b8]::proto::h2::server::Serving<hyper[39e87571ecf802b8]::common::io::rewind::Rewind<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa... (server.rs:284)
11 <unknown> 0x55c96c43f29d <hyper[39e87571ecf802b8]::proto::h2::server::Server<hyper[39e87571ecf802b8]::common::io::rewind::Rewind<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]... (server.rs:218)
12 <unknown> 0x55c96c252c35 [inlined] <hyper[39e87571ecf802b8]::server::conn::ProtoServer<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<fp_tcp_idle_timeout[691300eef0d93111]::... (conn.rs:988)
13 <unknown> 0x55c96c252c35 <hyper[39e87571ecf802b8]::server::conn::upgrades::UpgradeableConnection<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<fp_tcp_idle_timeout... (conn.rs:1046)
14 <unknown> 0x55c96c2e9186 [inlined] <&mut hyper[39e87571ecf802b8]::server::conn::upgrades::UpgradeableConnection<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<fp_tcp_idle_ti... (future.rs:112)
15 <unknown> 0x55c96c2e9186 [inlined] <tracing[f7f30f32cc34498a]::instrument::Instrumented<&mut hyper[39e87571ecf802b8]::server::conn::upgrades::UpgradeableConnection<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pi... (instrument.rs:272)
16 <unknown> 0x55c96c2e9186 <opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Instrumented<&mut hyper[39e87571ecf802b8]::server::conn::upgrades::UpgradeableConnection<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_met... (context.rs:373)
17 <unknown> 0x55c96c2871d6 [inlined] <tripwire[24f4131df546c329]::preempt::PreemptibleFuture<&mut tripwire[24f4131df546c329]::tripwire::Tripwire, opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Instrumented<&mut hyper[39e87571ecf802b8... (preempt.rs:56)
18 <unknown> 0x55c96c2871d6 [inlined] <fp_http[4f776ff130c93902]::service::HttpService as tower_service[51fcaf158879a154]::Service<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Bo... (service.rs:129)
19 <unknown> 0x55c96c2871d6 <tracing[f7f30f32cc34498a]::instrument::Instrumented<<fp_http[4f776ff130c93902]::service::HttpService as tower_service[51fcaf158879a154]::Service<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1... (instrument.rs:272)
20 <unknown> 0x55c96c24461f [inlined] <core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::future::future::Future<Output = core[cf0a937d1125fa47]::result::Result<(), fp_errors[4e01595fafbed632]::proxy::ProxyError>> + core[cf0a937d1125fa47]::marke... (future.rs:124)
21 <unknown> 0x55c96c24461f <fp_tls[2bc31ed72f4ace2]::layers::tls::TlsLayeredService<fp_http[4f776ff130c93902]::service::HttpService> as tower_service[51fcaf158879a154]::Service<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::b... (tls.rs:109)
22 <unknown> 0x55c96c44645c [inlined] <core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::future::future::Future<Output = core[cf0a937d1125fa47]::result::Result<(), fp_errors[4e01595fafbed632]::proxy::ProxyError>> + core[cf0a937d1125fa47]::marke... (future.rs:124)
23 <unknown> 0x55c96c44645c [inlined] <tower[92830710c1249629]::limit::concurrency::future::ResponseFuture<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::future::future::Future<Output = core[cf0a937d1125fa47]::result::Result<(), fp_errors[4e... (future.rs:39)
24 <unknown> 0x55c96c44645c <fp_layers[fd136ed4dda1018c]::load_shed::ResponseFuture<tower[92830710c1249629]::limit::concurrency::future::ResponseFuture<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::future::future::Future<Output = ... (load_shed.rs:152)
25 <unknown> 0x55c96c30636b [inlined] <core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::future::future::Future<Output = core[cf0a937d1125fa47]::result::Result<(), alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::error::Error + c... (future.rs:124)
26 <unknown> 0x55c96c30636b <tower[92830710c1249629]::util::oneshot::Oneshot<tower[92830710c1249629]::util::boxed::sync::BoxService<fp_tcp[27e50990d6be481a]::permitted::PermittedTcpStream, (), alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::error::Error + core[cf0a... (oneshot.rs:97)
27 <unknown> 0x55c96c27c958 [inlined] fp_edge[71fb797d61ca5443]::handle_edge_incoming::{closure#0}::{closure#0} (lib.rs:104)
28 <unknown> 0x55c96c27c958 tracing[f7f30f32cc34498a]::instrument::Instrumented<T>::poll (instrument.rs:272)
29 <unknown> 0x55c96c276c3c [inlined] opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<T>::poll (context.rs:373)
30 <unknown> 0x55c96c276c3c [inlined] <spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Instrumented<fp_edge[71fb797d61ca5443]::handle_edge_incoming::{closure#0}::{closure#0}>>> as core[cf0a937d1125fa... (lib.rs:103)
31 <unknown> 0x55c96c276c3c <tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Instrumented<fp_edge[71fb797d61ca5443]::handle_edge_incoming... (instrument.rs:272)
32 <unknown> 0x55c96c333c0c [inlined] <tokio[1aa1d45b4aef9d67]::runtime::task::core::Core<tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Instrumen... (core.rs:223)
33 <unknown> 0x55c96c333c0c [inlined] <tokio[1aa1d45b4aef9d67]::loom::std::unsafe_cell::UnsafeCell<tokio[1aa1d45b4aef9d67]::runtime::task::core::Stage<tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context:... (unsafe_cell.rs:14)
34 <unknown> 0x55c96c333c0c <tokio[1aa1d45b4aef9d67]::runtime::task::core::Core<tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Instrumen... (core.rs:212)
35 <unknown> 0x55c96c4240a9 [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::harness::poll_future:: (harness.rs:476)
36 <unknown> 0x55c96c4240a9 [inlined] <core[cf0a937d1125fa47]::panic::unwind_safe::AssertUnwindSafe<tokio[1aa1d45b4aef9d67]::runtime::task::harness::poll_future<tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace... (unwind_safe.rs:271)
37 <unknown> 0x55c96c4240a9 [inlined] std[645413f22396c11]::panicking::try::do_call:: (panicking.rs:483)
38 <unknown> 0x55c96c4240a9 [inlined] std[645413f22396c11]::panicking::try:: (panicking.rs:447)
39 <unknown> 0x55c96c4240a9 [inlined] std[645413f22396c11]::panic::catch_unwind:: (panic.rs:137)
40 <unknown> 0x55c96c4240a9 [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::harness::poll_future:: (harness.rs:464)
41 <unknown> 0x55c96c4240a9 [inlined] <tokio[1aa1d45b4aef9d67]::runtime::task::harness::Harness<tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Ins... (harness.rs:198)
42 <unknown> 0x55c96c4240a9 <tokio[1aa1d45b4aef9d67]::runtime::task::harness::Harness<tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Ins... (harness.rs:152)
43 <unknown> 0x55c96cfd4c0c [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::raw::RawTask::poll (raw.rs:200)
44 <unknown> 0x55c96cfd4c0c [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::LocalNotified<T>::run (mod.rs:394)
45 <unknown> 0x55c96cfd4c0c [inlined] tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::Context::run_task::{closure#0} (worker.rs:464)
46 <unknown> 0x55c96cfd4c0c [inlined] tokio[1aa1d45b4aef9d67]::runtime::coop::with_budget:: (coop.rs:107)
47 <unknown> 0x55c96cfd4c0c [inlined] tokio[1aa1d45b4aef9d67]::runtime::coop::budget:: (coop.rs:73)
48 <unknown> 0x55c96cfd4c0c tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::Context::run_task (worker.rs:463)
49 <unknown> 0x55c96cfd4691 tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::Context::run (worker.rs:433)
50 <unknown> 0x55c96cfe0f19 [inlined] tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::run::{closure#0} (worker.rs:406)
51 <unknown> 0x55c96cfe0f19 tokio[1aa1d45b4aef9d67]::macros::scoped_tls::ScopedKey<T>::set::<T> (scoped_tls.rs:61)
52 <unknown> 0x55c96cfd3a64 tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::run (worker.rs:403)
53 <unknown> 0x55c96c2a611b [inlined] tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_place::<T>::{closure#0}::{closure#0} (worker.rs:343)
54 <unknown> 0x55c96c2a611b [inlined] <tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_place<estate_corrosion[4f14d3214435355a]::instances::read::{closure#0}::{closure#0}, core[cf0a937d1125fa47]::resu... (task.rs:42)
55 <unknown> 0x55c96c2a611b [inlined] <tokio[1aa1d45b4aef9d67]::runtime::task::core::Core<tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_place<estate_corrosion[4f14d3214435355a]::instances::read::{cl... (core.rs:223)
56 <unknown> 0x55c96c2a611b <tokio[1aa1d45b4aef9d67]::loom::std::unsafe_cell::UnsafeCell<tokio[1aa1d45b4aef9d67]::runtime::task::core::Stage<tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_p... (unsafe_cell.rs:14)
57 <unknown> 0x55c96c334e5c <tokio[1aa1d45b4aef9d67]::runtime::task::core::Core<tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_place<estate_corrosion[4f14d3214435355a]::instances::read::{cl... (core.rs:212)
58 <unknown> 0x55c96c426c99 [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::harness::poll_future:: (harness.rs:476)
59 <unknown> 0x55c96c426c99 [inlined] <core[cf0a937d1125fa47]::panic::unwind_safe::AssertUnwindSafe<tokio[1aa1d45b4aef9d67]::runtime::task::harness::poll_future<tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::... (unwind_safe.rs:271)
60 <unknown> 0x55c96c426c99 [inlined] std[645413f22396c11]::panicking::try::do_call:: (panicking.rs:483)
61 <unknown> 0x55c96c426c99 [inlined] std[645413f22396c11]::panicking::try:: (panicking.rs:447)
62 <unknown> 0x55c96c426c99 [inlined] std[645413f22396c11]::panic::catch_unwind:: (panic.rs:137)
63 <unknown> 0x55c96c426c99 [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::harness::poll_future:: (harness.rs:464)
64 <unknown> 0x55c96c426c99 [inlined] <tokio[1aa1d45b4aef9d67]::runtime::task::harness::Harness<tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_place<estate_corrosion[4f14d3214435355a]::instances::rea... (harness.rs:198)
65 <unknown> 0x55c96c426c99 <tokio[1aa1d45b4aef9d67]::runtime::task::harness::Harness<tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_place<estate_corrosion[4f14d3214435355a]::instances::rea... (harness.rs:152)
66 <unknown> 0x55c96cfc5abc [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::raw::RawTask::poll (raw.rs:200)
67 <unknown> 0x55c96cfc5abc [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::UnownedTask<T>::run (mod.rs:431)
68 <unknown> 0x55c96cfc5abc [inlined] tokio[1aa1d45b4aef9d67]::runtime::blocking::pool::Task::run (pool.rs:159)
69 <unknown> 0x55c96cfc5abc tokio[1aa1d45b4aef9d67]::runtime::blocking::pool::Inner::run (pool.rs:511)
70 <unknown> 0x55c96cfc6dfd [inlined] tokio[1aa1d45b4aef9d67]::runtime::blocking::pool::Spawner::spawn_thread::{closure#0} (pool.rs:469)
71 <unknown> 0x55c96cfc6dfd std[645413f22396c11]::sys_common::backtrace::__rust_begin_short_backtrace::<T> (backtrace.rs:121)
72 <unknown> 0x55c96cfd1347 [inlined] std[645413f22396c11]::thread::Builder::spawn_unchecked_::<T>::{closure#1}::{closure#0} (mod.rs:550)
73 <unknown> 0x55c96cfd1347 [inlined] <core[cf0a937d1125fa47]::panic::unwind_safe::AssertUnwindSafe<<std[645413f22396c11]::thread::Builder>::spawn_unchecked_<<tokio[1aa1d45b4aef9d67]::runtime::blocking::pool::Spawner>::spawn_thread::{closure#0}, ()>::{closure#1}::{closure#0}> as core[cf0a93... (unwind_safe.rs:271)
74 <unknown> 0x55c96cfd1347 [inlined] std[645413f22396c11]::panicking::try::do_call:: (panicking.rs:483)
75 <unknown> 0x55c96cfd1347 [inlined] std[645413f22396c11]::panicking::try:: (panicking.rs:447)
76 <unknown> 0x55c96cfd1347 [inlined] std[645413f22396c11]::panic::catch_unwind:: (panic.rs:137)
77 <unknown> 0x55c96cfd1347 [inlined] std[645413f22396c11]::thread::Builder::spawn_unchecked_::<T>::{closure#1} (mod.rs:549)
78 <unknown> 0x55c96cfd1347 std[645413f22396c11]::thread::Builder::spawn_unchecked_<T>::{closure#1}::call_once::{shim:vtable#0} (function.rs:507)
79 alloc 0x55c96d2fdb63 [inlined] alloc::boxed::Box<T>::call_once (boxed.rs:2000)
80 alloc 0x55c96d2fdb63 [inlined] alloc::boxed::Box<T>::call_once (boxed.rs:2000)
81 std 0x55c96d2fdb63 std::sys::unix::thread::Thread::new::thread_start (thread.rs:108)
82 <unknown> 0x7fcc4412f6db start_thread (pthread_create.c:463)
83 <unknown> 0x7fcc438b671f __GI___clone (clone.S:95)
84 <unknown> 0x0 <unknown>
There are 3 subsequent PoisonError
unwrap panics, but I assume they were caused by this assertion. If not I can also provide their backtrace.
I believe this was fixed in #682, released in v0.3.19. (If not, we can re-open.)