Random out of range panic when enabling qlog
dbaldassi opened this issue Ā· comments
Hi,
I'm getting an out of range panic for slice when enabling qlog.
I'm enabling qlog like this (in C/C++) :
_qlog_filename = fmt::format("{}.qlog", fmt::join(scid, ""));
auto qfile = fmt::format("{}/{}", _qlog_dir, _qlog_filename);
quiche_conn_set_qlog_path(_conn, qfile.c_str(), "quiche-client qlog", "quiche-client qlog");
I'm getting this backtrace from rust :
thread '<unnamed>' panicked at 'range end index 8191 out of range for slice of length 113', /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/alloc/src/vec/mod.rs:1993:36
stack backtrace:
0: 0x5569d5894a51 - std::backtrace_rs::backtrace::libunwind::trace::h66dc1c6acf794faa
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
1: 0x5569d5894a51 - std::backtrace_rs::backtrace::trace_unsynchronized::ha80d20099a67f790
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
2: 0x5569d5894a51 - std::sys_common::backtrace::_print_fmt::h7b959d43f35f16d4
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:65:5
3: 0x5569d5894a51 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::hdaa196410d9ee0b9
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:44:22
4: 0x5569d58f8e8f - core::fmt::rt::Argument::fmt::h0ddfbbe8be3f80d0
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/fmt/rt.rs:138:9
5: 0x5569d58f8e8f - core::fmt::write::h66b3c629f3d623e4
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/fmt/mod.rs:1094:21
6: 0x5569d5888be7 - std::io::Write::write_fmt::hb6d80fba4115e0c2
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/io/mod.rs:1714:15
7: 0x5569d5894865 - std::sys_common::backtrace::_print::h1a49cfb0cf3cce17
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:47:5
8: 0x5569d5894865 - std::sys_common::backtrace::print::hca95c2d0055e42a2
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:34:9
9: 0x5569d5897643 - std::panicking::default_hook::{{closure}}::hc03c01c56bca600c
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:269:22
10: 0x5569d58973d4 - std::panicking::default_hook::hb2cb5315b6634f1c
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:288:9
11: 0x5569d5897c89 - std::panicking::rust_panic_with_hook::h75cd912a39a34e8a
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:705:13
12: 0x5569d5897b87 - std::panicking::begin_panic_handler::{{closure}}::h1498b46f7849e167
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:597:13
13: 0x5569d5894eb6 - std::sys_common::backtrace::__rust_end_short_backtrace::hd36a39b27b98086b
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:151:18
14: 0x5569d58978d2 - rust_begin_unwind
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:593:5
15: 0x5569d5229193 - core::panicking::panic_fmt::h98ef273141454c23
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/panicking.rs:67:14
16: 0x5569d5229822 - core::slice::index::slice_end_index_len_fail_rt::h35a42e1fc28e1612
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/slice/index.rs:76:5
17: 0x5569d5229822 - core::slice::index::slice_end_index_len_fail::h26b3aa12eff5f17d
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/slice/index.rs:68:9
18: 0x5569d5881b44 - core::slice::index::range::h87d33c6fbd37daa5
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/slice/index.rs:723:9
19: 0x5569d5881b44 - alloc::vec::Vec<T,A>::drain::h052645066e0a8d1a
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/alloc/src/vec/mod.rs:1993:36
20: 0x5569d5881b44 - <std::io::buffered::bufwriter::BufWriter<W>::flush_buf::BufGuard as core::ops::drop::Drop>::drop::h4f7b9382e42eeb6f
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/io/buffered/bufwriter.rs:221:21
21: 0x5569d57aef80 - core::ptr::drop_in_place<std::io::buffered::bufwriter::BufWriter<W>::flush_buf::BufGuard>::hc37e431a110ee42c
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/ptr/mod.rs:497:1
22: 0x5569d57aef80 - std::io::buffered::bufwriter::BufWriter<W>::flush_buf::h01a99e0994172378
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/io/buffered/bufwriter.rs:245:5
23: 0x5569d5220dc9 - std::io::buffered::bufwriter::BufWriter<W>::write_all_cold::hccfa405a7cfe951d
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/io/buffered/bufwriter.rs:401:13
24: 0x5569d582e8ee - std::io::impls::<impl std::io::Write for &mut W>::write_all::h24a9685e13a6ab63
at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/io/impls.rs:78:9
25: 0x5569d582e8ee - serde_json::ser::Formatter::write_string_fragment::h7eef4bfae65d9e1d
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.107/src/ser.rs:1746:9
26: 0x5569d582e8ee - serde_json::ser::format_escaped_str_contents::h538293b1ace97776
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.107/src/ser.rs:2094:5
27: 0x5569d582e8ee - serde_json::ser::format_escaped_str::h42e74cfbbe9d961a
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.107/src/ser.rs:2057:10
28: 0x5569d5995d4f - <&mut serde_json::ser::Serializer<W,F> as serde::ser::Serializer>::serialize_str::h7b56c94773d0e081
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.107/src/ser.rs:187:9
29: 0x5569d5995d4f - <serde_json::ser::MapKeySerializer<W,F> as serde::ser::Serializer>::serialize_str::h03ab041ff4ea362f
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.107/src/ser.rs:801:18
30: 0x5569d5995d4f - serde::ser::impls::<impl serde::ser::Serialize for str>::serialize::h2d5f71c779239a41
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde-1.0.188/src/ser/impls.rs:46:9
31: 0x5569d5995d4f - <serde_json::ser::Compound<W,F> as serde::ser::SerializeMap>::serialize_key::h2329c84fbac82a37
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.107/src/ser.rs:628:22
32: 0x5569d5995d4f - serde::ser::SerializeMap::serialize_entry::h45d76de02bbf45a9
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde-1.0.188/src/ser/mod.rs:1810:14
33: 0x5569d5995d4f - <serde_json::ser::Compound<W,F> as serde::ser::SerializeStruct>::serialize_field::hb8c71a339e81ed2e
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.107/src/ser.rs:693:37
34: 0x5569d5995d4f - qlog::events::quic::_::<impl serde::ser::Serialize for qlog::events::quic::PacketHeader>::serialize::h181a74c2a4adba36
at /home/user/Documents/quic_implementation/quiche/qlog/src/events/quic.rs:66:17
35: 0x5569d5999fce - <serde_json::ser::Compound<W,F> as serde::ser::SerializeMap>::serialize_value::had903f1fcc33113e
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.107/src/ser.rs:652:22
36: 0x5569d5999fce - serde::ser::SerializeMap::serialize_entry::h7337973cd1176863
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde-1.0.188/src/ser/mod.rs:1811:9
37: 0x5569d5999fce - <serde_json::ser::Compound<W,F> as serde::ser::SerializeStruct>::serialize_field::h3eb67e10513ae6de
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.107/src/ser.rs:693:37
38: 0x5569d5999fce - qlog::events::quic::_::<impl serde::ser::Serialize for qlog::events::quic::PacketReceived>::serialize::hfdb7f4e98361ea66
at /home/user/Documents/quic_implementation/quiche/qlog/src/events/quic.rs:607:10
39: 0x5569d5829b10 - qlog::events::_::<impl serde::ser::Serialize for qlog::events::EventData>::serialize::h58d41c89a14b799a
at /home/user/Documents/quic_implementation/quiche/qlog/src/events/mod.rs:456:10
40: 0x5569d5829b10 - serde::ser::impls::<impl serde::ser::Serialize for &T>::serialize::hbf09e97cc8eb573b
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde-1.0.188/src/ser/impls.rs:462:1
41: 0x5569d5829b10 - qlog::events::_::<impl serde::ser::Serialize for qlog::events::Event>::serialize::hcb8459c349dddb42
at /home/user/Documents/quic_implementation/quiche/qlog/src/events/mod.rs:67:10
42: 0x5569d5843beb - serde_json::ser::to_writer::h684590643c4e3d4c
at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.107/src/ser.rs:2145:5
43: 0x5569d5843beb - qlog::streamer::QlogStreamer::add_event::ha03e663e29793bb0
at /home/user/Documents/quic_implementation/quiche/qlog/src/streamer.rs:214:9
44: 0x5569d5843b12 - qlog::streamer::QlogStreamer::add_event_data_with_instant::hf12bd92f2d036089
at /home/user/Documents/quic_implementation/quiche/qlog/src/streamer.rs:200:9
45: 0x5569d5920a58 - quiche::Connection::recv_single::hcfd791daa7d2eea2
at /home/user/Documents/quic_implementation/quiche/quiche/src/lib.rs:2769:13
46: 0x5569d5920a58 - quiche::Connection::recv::h0ea8cee14424735a
at /home/user/Documents/quic_implementation/quiche/quiche/src/lib.rs:2128:30
47: 0x5569d57b5025 - quiche_conn_recv
at /home/user/Documents/quic_implementation/quiche/quiche/src/ffi.rs:726:11
48: 0x5569d52d18f1 - recv_cb
49: 0x7fc83a0ed773 - ev_invoke_pending
50: 0x7fc83a0f1041 - ev_run
51: 0x5569d52d158e - _ZN12QuicheServer5startEv
52: 0x5569d523597e - _ZN9OutTunnel3runEv
53: 0x7fc839f77253 - <unknown>
54: 0x7fc839c20ac3 - <unknown>
55: 0x7fc839cb1bf4 - __clone
If, I do not enable qlog, the error does not show, and if qlog is enabled, the error shows at random during the session.
My recv code is almost the same as the C client example :
static uint8_t buf[65535];
struct conn_io *conn_io = (struct conn_io*)w->data;
while (1) {
struct sockaddr_storage peer_addr;
socklen_t peer_addr_len = sizeof(peer_addr);
memset(&peer_addr, 0, peer_addr_len);
ssize_t read = recvfrom(conn_io->sock, buf, sizeof(buf), 0,
(struct sockaddr *) &peer_addr,
&peer_addr_len);
if (read < 0) {
if ((errno == EWOULDBLOCK) || (errno == EAGAIN)) break;
perror("failed to read");
return;
}
quiche_recv_info recv_info = {
(struct sockaddr *) &peer_addr,
peer_addr_len,
(struct sockaddr *) &conn_io->local_addr,
conn_io->local_addr_len,
};
ssize_t done = quiche_conn_recv(conn_io->conn, buf, read, &recv_info);
if (done < 0) {
fprintf(stderr, "failed to process packet\n");
continue;
}
}
Nothing springs to mind why this might be happening. With quiche_conn_set_qlog_path() the writing is all handled internally, which rules out possible weirdness caused by a C++ writer.
Are you able to reproduce this alongside a decrypted PCAP (using SSLKEYLOGFILE)? It would be interesting to see what packet is triggering the panic.
So that is the moment of the crash, there is no more packets going out of the client after this point
So I guess this ACK is the last packet received by the client before panicking :
However it seems that wireshark has trouble with decrypting the packets coming out of the client :
Well, it seems I have fixed the issue by putting mutex around quic_conn_send* and quic_conn_recv* functions
Closing as root cause found