segfault in rpc integration test
dotnwat opened this issue · comments
hitting a segfault in master running integration test sudo build/src/integration_tests/smf_rpc_integration_test
full gdb dump below
Also note that this doesn't seem to be deterministic. Outside of GDB the run succeeded once, inside GDB I need to set disable-randomization off in order to get reliable reproduction of the fault.
narrowing it down a bit, with this patch the segfault is gone
diff --git a/src/core/rpc_server.cc b/src/core/rpc_server.cc
index c43ac5b..a857cde 100644
--- a/src/core/rpc_server.cc
+++ b/src/core/rpc_server.cc
@@ -132,7 +132,7 @@ rpc_server::start() {
seastar::future<>
rpc_server::stop() {
LOG_INFO("Stopped seastar::accept() calls");
- listener_->abort_accept();
+ //listener_->abort_accept();
std::for_each(open_connections_.begin(), open_connections_.end(),
[](auto &client_conn) {
Thread 16 (Thread 0x7f85067ff700 (LWP 18825)):
#0 0x00000000005a8e74 in seastar::basic_sstring<char, unsigned int, 15u, true>::compare(seastar::basic_sstring<char, unsigned int, 15u, true> const&) const ()
#1 0x00000000005abf98 in bool std::__lexicographical_compare_impl<std::_Rb_tree_const_iterator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > >, std::_Rb_tree_const_iterator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > >, __gnu_cxx::__ops::_Iter_less_iter>(std::_Rb_tree_const_iterator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > >, std::_Rb_tree_const_iterator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > >, std::_Rb_tree_const_iterator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > >, std::_Rb_tree_const_iterator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > >, __gnu_cxx::__ops::_Iter_less_iter) ()
#2 0x00000000005a661f in std::_Rb_tree<std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > >, std::pair<std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > > const, seastar::shared_ptr<seastar::metrics::impl::registered_metric> >, std::_Select1st<std::pair<std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > > const, seastar::shared_ptr<seastar::metrics::impl::registered_metric> > >, std::less<std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > > >, std::allocator<std::pair<std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > > const, seastar::shared_ptr<seastar::metrics::impl::registered_metric> > > >::_M_lower_bound(std::_Rb_tree_node<std::pair<std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > > const, seastar::shared_ptr<seastar::metrics::impl::registered_metric> > >*, std::_Rb_tree_node_base*, std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > > const&) [clone .isra.607] ()
#3 0x00000000005abffa in std::_Rb_tree<std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > >, std::pair<std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > > const, seastar::shared_ptr<seastar::metrics::impl::registered_metric> >, std::_Select1st<std::pair<std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > > const, seastar::shared_ptr<seastar::metrics::impl::registered_metric> > >, std::less<std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > > >, std::allocator<std::pair<std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > > const, seastar::shared_ptr<seastar::metrics::impl::registered_metric> > > >::find(std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > > const&) ()
#4 0x00000000005a6d42 in seastar::metrics::impl::impl::remove_registration(seastar::metrics::impl::metric_id const&) ()
#5 0x00000000005a6ead in seastar::metrics::impl::unregister_metric(seastar::metrics::impl::metric_id const&) ()
#6 0x00000000005a6f07 in seastar::metrics::impl::metric_groups_impl::~metric_groups_impl() ()
#7 0x00000000005a6f69 in seastar::metrics::impl::metric_groups_impl::~metric_groups_impl() ()
#8 0x0000000000570b33 in smf::rpc_server::~rpc_server() ()
#9 0x00000000004eb7e5 in seastar::shared_ptr_count_for<smf::rpc_server>::~shared_ptr_count_for() ()
#10 0x00000000005268d7 in seastar::smp_message_queue::async_work_item<seastar::sharded<smf::rpc_server>::stop()::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}>::process()::{lambda()#1}::operator()() const ()
#11 0x0000000000526cf8 in seastar::smp_message_queue::async_work_item<seastar::sharded<smf::rpc_server>::stop()::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}>::process() ()
#12 0x00000000005f332f in unsigned long seastar::smp_message_queue::process_queue<2ul, seastar::smp_message_queue::process_incoming()::{lambda(seastar::smp_message_queue::work_item*)#1}>(seastar::smp_message_queue::lf_queue&, seastar::smp_message_queue::process_incoming()::{lambda(seastar::smp_message_queue::work_item*)#1}) [clone .isra.4073] ()
#13 0x00000000005f3415 in seastar::smp::poll_queues() ()
#14 0x000000000063f386 in seastar::reactor::smp_pollfn::poll() ()
#15 0x00000000005c2ad5 in std::_Function_handler<bool (), seastar::reactor::run()::{lambda()#6}>::_M_invoke(std::_Any_data const&) ()
#16 0x0000000000616672 in seastar::reactor::run() ()
#17 0x000000000061a5fe in seastar::smp::configure(boost::program_options::variables_map)::{lambda()#3}::operator()() const ()
#18 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#19 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#20 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 15 (Thread 0x7f85033ff700 (LWP 18834)):
#0 0x00007f8509099df4 in read () from /lib64/libpthread.so.0
#1 0x00000000005c8e67 in seastar::thread_pool::work(seastar::basic_sstring<char, unsigned int, 15u, true>) ()
#2 0x00000000005c9112 in std::_Function_handler<void (), seastar::thread_pool::thread_pool(seastar::reactor*, seastar::basic_sstring<char, unsigned int, 15u, true>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
#3 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#4 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#5 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 14 (Thread 0x7f85079ff700 (LWP 18822)):
#0 0x00007f8508422cfb in shutdown () from /lib64/libc.so.6
#1 0x00000000006c29c5 in seastar::net::posix_connected_socket_impl<(seastar::transport)6>::shutdown_input() ()
#2 0x0000000000572d1d in smf::rpc_server::stop() ()
#3 0x0000000000526816 in seastar::smp_message_queue::async_work_item<seastar::sharded<smf::rpc_server>::stop()::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}>::process()::{lambda()#1}::operator()() const ()
#4 0x0000000000526cf8 in seastar::smp_message_queue::async_work_item<seastar::sharded<smf::rpc_server>::stop()::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}>::process() ()
#5 0x00000000005f332f in unsigned long seastar::smp_message_queue::process_queue<2ul, seastar::smp_message_queue::process_incoming()::{lambda(seastar::smp_message_queue::work_item*)#1}>(seastar::smp_message_queue::lf_queue&, seastar::smp_message_queue::process_incoming()::{lambda(seastar::smp_message_queue::work_item*)#1}) [clone .isra.4073] ()
#6 0x00000000005f3415 in seastar::smp::poll_queues() ()
#7 0x000000000063f386 in seastar::reactor::smp_pollfn::poll() ()
#8 0x00000000005c2ad5 in std::_Function_handler<bool (), seastar::reactor::run()::{lambda()#6}>::_M_invoke(std::_Any_data const&) ()
#9 0x0000000000616672 in seastar::reactor::run() ()
#10 0x000000000061a5fe in seastar::smp::configure(boost::program_options::variables_map)::{lambda()#3}::operator()() const ()
#11 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#12 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#13 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 13 (Thread 0x7f8502dff700 (LWP 18833)):
#0 0x00007f8509099df4 in read () from /lib64/libpthread.so.0
#1 0x00000000005c8e67 in seastar::thread_pool::work(seastar::basic_sstring<char, unsigned int, 15u, true>) ()
#2 0x00000000005c9112 in std::_Function_handler<void (), seastar::thread_pool::thread_pool(seastar::reactor*, seastar::basic_sstring<char, unsigned int, 15u, true>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
#3 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#4 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#5 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 12 (Thread 0x7f8503bff700 (LWP 18832)):
#0 0x00007f8509099df4 in read () from /lib64/libpthread.so.0
#1 0x00000000005c8e67 in seastar::thread_pool::work(seastar::basic_sstring<char, unsigned int, 15u, true>) ()
#2 0x00000000005c9112 in std::_Function_handler<void (), seastar::thread_pool::thread_pool(seastar::reactor*, seastar::basic_sstring<char, unsigned int, 15u, true>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
#3 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#4 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#5 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7f85047ff700 (LWP 18830)):
#0 0x00007f8509099df4 in read () from /lib64/libpthread.so.0
#1 0x00000000005c8e67 in seastar::thread_pool::work(seastar::basic_sstring<char, unsigned int, 15u, true>) ()
#2 0x00000000005c9112 in std::_Function_handler<void (), seastar::thread_pool::thread_pool(seastar::reactor*, seastar::basic_sstring<char, unsigned int, 15u, true>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
#3 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#4 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#5 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7f85061ff700 (LWP 18826)):
#0 0x00000000005ab9a6 in bool std::binary_search<char const*, char>(char const*, char const*, char const&) ()
#1 0x00000000005aba6d in char* std::__find_if<char*, __gnu_cxx::__ops::_Iter_pred<boost::algorithm::detail::is_any_ofF<char> > >(char*, char*, __gnu_cxx::__ops::_Iter_pred<boost::algorithm::detail::is_any_ofF<char> >, std::random_access_iterator_tag) ()
#2 0x00000000005abc50 in char* std::__remove_if<char*, __gnu_cxx::__ops::_Iter_pred<boost::algorithm::detail::is_any_ofF<char> > >(char*, char*, __gnu_cxx::__ops::_Iter_pred<boost::algorithm::detail::is_any_ofF<char> >) ()
#3 0x00000000005a697d in seastar::metrics::impl::metric_id::full_name() const ()
#4 0x00000000005a6d0c in seastar::metrics::impl::impl::remove_registration(seastar::metrics::impl::metric_id const&) ()
#5 0x00000000005a6ead in seastar::metrics::impl::unregister_metric(seastar::metrics::impl::metric_id const&) ()
#6 0x00000000005a6f07 in seastar::metrics::impl::metric_groups_impl::~metric_groups_impl() ()
#7 0x00000000005a6f69 in seastar::metrics::impl::metric_groups_impl::~metric_groups_impl() ()
#8 0x0000000000570b33 in smf::rpc_server::~rpc_server() ()
#9 0x00000000004eb7e5 in seastar::shared_ptr_count_for<smf::rpc_server>::~shared_ptr_count_for() ()
#10 0x00000000005268d7 in seastar::smp_message_queue::async_work_item<seastar::sharded<smf::rpc_server>::stop()::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}>::process()::{lambda()#1}::operator()() const ()
#11 0x0000000000526cf8 in seastar::smp_message_queue::async_work_item<seastar::sharded<smf::rpc_server>::stop()::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}>::process() ()
#12 0x00000000005f332f in unsigned long seastar::smp_message_queue::process_queue<2ul, seastar::smp_message_queue::process_incoming()::{lambda(seastar::smp_message_queue::work_item*)#1}>(seastar::smp_message_queue::lf_queue&, seastar::smp_message_queue::process_incoming()::{lambda(seastar::smp_message_queue::work_item*)#1}) [clone .isra.4073] ()
#13 0x00000000005f3415 in seastar::smp::poll_queues() ()
#14 0x000000000063f386 in seastar::reactor::smp_pollfn::poll() ()
#15 0x00000000005c2ad5 in std::_Function_handler<bool (), seastar::reactor::run()::{lambda()#6}>::_M_invoke(std::_Any_data const&) ()
#16 0x0000000000616672 in seastar::reactor::run() ()
#17 0x000000000061a5fe in seastar::smp::configure(boost::program_options::variables_map)::{lambda()#3}::operator()() const ()
#18 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#19 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#20 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7f85027ff700 (LWP 18835)):
#0 0x00007f8509099df4 in read () from /lib64/libpthread.so.0
#1 0x00000000005c8e67 in seastar::thread_pool::work(seastar::basic_sstring<char, unsigned int, 15u, true>) ()
#2 0x00000000005c9112 in std::_Function_handler<void (), seastar::thread_pool::thread_pool(seastar::reactor*, seastar::basic_sstring<char, unsigned int, 15u, true>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
#3 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#4 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#5 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7f8506dff700 (LWP 18824)):
#0 0x0000000000638e1e in seastar::reactor::add_task(std::unique_ptr<seastar::task, std::default_delete<seastar::task> >&&) ()
#1 0x00000000004ee29b in void seastar::promise<>::set_value<>() ()
#2 0x000000000056e534 in seastar::internal::do_until_state<smf::rpc_server::handle_client_connection(seastar::lw_shared_ptr<smf::rpc_server_connection>)::{lambda()#1}, smf::rpc_server::handle_client_connection(seastar::lw_shared_ptr<smf::rpc_server_connection>)::{lambda()#2}>::run_and_dispose() ()
#3 0x00000000005c51c1 in seastar::reactor::run_tasks(seastar::reactor::task_queue&) ()
#4 0x00000000005c54b0 in seastar::reactor::run_some_tasks() [clone .part.2977] ()
#5 0x0000000000616636 in seastar::reactor::run() ()
#6 0x000000000061a5fe in seastar::smp::configure(boost::program_options::variables_map)::{lambda()#3}::operator()() const ()
#7 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#8 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#9 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7f8501fff700 (LWP 18836)):
#0 0x00007f8509099df4 in read () from /lib64/libpthread.so.0
#1 0x00000000005c8e67 in seastar::thread_pool::work(seastar::basic_sstring<char, unsigned int, 15u, true>) ()
#2 0x00000000005c9112 in std::_Function_handler<void (), seastar::thread_pool::thread_pool(seastar::reactor*, seastar::basic_sstring<char, unsigned int, 15u, true>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
#3 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#4 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#5 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7f85041ff700 (LWP 18831)):
#0 0x00007f8509099df4 in read () from /lib64/libpthread.so.0
#1 0x00000000005c8e67 in seastar::thread_pool::work(seastar::basic_sstring<char, unsigned int, 15u, true>) ()
#2 0x00000000005c9112 in std::_Function_handler<void (), seastar::thread_pool::thread_pool(seastar::reactor*, seastar::basic_sstring<char, unsigned int, 15u, true>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
#3 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#4 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#5 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7f85055ff700 (LWP 18828)):
#0 0x00007f8508422cfb in shutdown () from /lib64/libc.so.6
#1 0x00000000006c29c5 in seastar::net::posix_connected_socket_impl<(seastar::transport)6>::shutdown_input() ()
#2 0x0000000000572d1d in smf::rpc_server::stop() ()
#3 0x0000000000526816 in seastar::smp_message_queue::async_work_item<seastar::sharded<smf::rpc_server>::stop()::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}>::process()::{lambda()#1}::operator()() const ()
#4 0x0000000000526cf8 in seastar::smp_message_queue::async_work_item<seastar::sharded<smf::rpc_server>::stop()::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}>::process() ()
#5 0x00000000005f332f in unsigned long seastar::smp_message_queue::process_queue<2ul, seastar::smp_message_queue::process_incoming()::{lambda(seastar::smp_message_queue::work_item*)#1}>(seastar::smp_message_queue::lf_queue&, seastar::smp_message_queue::process_incoming()::{lambda(seastar::smp_message_queue::work_item*)#1}) [clone .isra.4073] ()
#6 0x00000000005f3415 in seastar::smp::poll_queues() ()
#7 0x000000000063f386 in seastar::reactor::smp_pollfn::poll() ()
#8 0x00000000005c2ad5 in std::_Function_handler<bool (), seastar::reactor::run()::{lambda()#6}>::_M_invoke(std::_Any_data const&) ()
#9 0x0000000000616672 in seastar::reactor::run() ()
#10 0x000000000061a5fe in seastar::smp::configure(boost::program_options::variables_map)::{lambda()#3}::operator()() const ()
#11 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#12 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#13 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f8505bff700 (LWP 18827)):
#0 0x00007f850841befd in syscall () from /lib64/libc.so.6
#1 0x0000000000714269 in seastar::internal::io_submit(unsigned long, long, seastar::internal::linux_abi::iocb**) ()
#2 0x000000000063ea85 in seastar::reactor_backend_aio::io_poll_poller::poll() ()
#3 0x00000000005c2ad5 in std::_Function_handler<bool (), seastar::reactor::run()::{lambda()#6}>::_M_invoke(std::_Any_data const&) ()
#4 0x0000000000616672 in seastar::reactor::run() ()
#5 0x000000000061a5fe in seastar::smp::configure(boost::program_options::variables_map)::{lambda()#3}::operator()() const ()
#6 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#7 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#8 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f8504fff700 (LWP 18829)):
#0 0x00007f8509099df4 in read () from /lib64/libpthread.so.0
#1 0x00000000005c8e67 in seastar::thread_pool::work(seastar::basic_sstring<char, unsigned int, 15u, true>) ()
#2 0x00000000005c9112 in std::_Function_handler<void (), seastar::thread_pool::thread_pool(seastar::reactor*, seastar::basic_sstring<char, unsigned int, 15u, true>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
#3 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#4 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#5 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f85073ff700 (LWP 18823)):
#0 0x00000000005752b4 in seastar::internal::lw_shared_ptr_accessors_no_esft<smf::rpc_server_connection>::dispose(seastar::lw_shared_ptr_counter_base*) ()
#1 0x000000000056e8c5 in seastar::continuation<seastar::future<>::then_wrapped<seastar::future<>::handle_exception<smf::rpc_server::handle_client_connection(seastar::lw_shared_ptr<smf::rpc_server_connection>)::{lambda(auto:1)#3}>(smf::rpc_server::handle_client_connection(seastar::lw_shared_ptr<smf::rpc_server_connection>)::{lambda(auto:1)#3}&&)::{lambda(smf::rpc_server::handle_client_connection(seastar::lw_shared_ptr<smf::rpc_server_connection>)::{lambda(auto:1)#3})#1}, seastar::future<> >(smf::rpc_server::handle_client_connection(seastar::lw_shared_ptr<smf::rpc_server_connection>)::{lambda(auto:1)#3})::{lambda(smf::rpc_server::handle_client_connection(seastar::lw_shared_ptr<smf::rpc_server_connection>)::{lambda(auto:1)#3})#1}>::run_and_dispose() ()
#2 0x00000000005c51c1 in seastar::reactor::run_tasks(seastar::reactor::task_queue&) ()
#3 0x00000000005c54b0 in seastar::reactor::run_some_tasks() [clone .part.2977] ()
#4 0x0000000000616636 in seastar::reactor::run() ()
#5 0x000000000061a5fe in seastar::smp::configure(boost::program_options::variables_map)::{lambda()#3}::operator()() const ()
#6 0x00000000005acafe in seastar::posix_thread::start_routine(void*) ()
#7 0x00007f850909058e in start_thread () from /lib64/libpthread.so.0
#8 0x00007f85084216a3 in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f8507d2bf40 (LWP 18820)):
#0 0x00007f85084fa23f in ?? () from /lib64/libgcc_s.so.1
#1 0x00007f85084fbec8 in _Unwind_Backtrace () from /lib64/libgcc_s.so.1
#2 0x00007f850842f966 in backtrace () from /lib64/libc.so.6
#3 0x000000000062d563 in void seastar::backtrace<seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}>(seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}&&) ()
#4 0x00000000005c9230 in seastar::print_with_backtrace(seastar::backtrace_buffer&) ()
#5 0x00000000005c930d in seastar::print_with_backtrace(char const*) ()
#6 0x00000000005c9364 in seastar::install_oneshot_signal_handler<11, &seastar::sigsegv_action>()::{lambda(int, siginfo_t*, void*)#1}::_FUN(int, siginfo_t*, void*) ()
#7 <signal handler called>
#8 0x0000000800000001 in ?? ()
#9 0x00000000005c51c1 in seastar::reactor::run_tasks(seastar::reactor::task_queue&) ()
#10 0x00000000005c54b0 in seastar::reactor::run_some_tasks() [clone .part.2977] ()
#11 0x0000000000616636 in seastar::reactor::run() ()
#12 0x0000000000581d62 in seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) ()
#13 0x00000000005826f7 in seastar::app_template::run(int, char**, std::function<seastar::future<int> ()>&&) ()
#14 0x00000000004db1c7 in main ()
quit
Interesting. This is how the seastar RPC works too.
are you saying the way seastar rpc works is that it crashes?
haha, no, i'm saying is likely a bug w/ upstream. could be worth stepping into it (listener::abort())
I fixed this in fixed in 4e68ff6