segment fault happen when benchmark thread set over 1 thread
guoanwu opened this issue · comments
[root@NC03 build]# ./bench -fill=1 -value_size=4096 -threads=2 -path=/mnt/pmem0/kvdk -space=2748779069 -num=4000000 -max_write_threads=1 -type=string -populate=1
to fill 4000000 uniform keys
[LOG] time 0 ms: Initializing PMEM size 2748779069 in file /mnt/pmem0/kvdk/data
[LOG] time 380 ms: Map pmem space done
[LOG] time 1163 ms: In restoring: iterated 0 records
[LOG] time 1163 ms: Populating PMEM space ...
[LOG] time 2697 ms: Populating done
init 2 write threads
init 0 read threads
------- ops in seconds -----------
time (ms), read ops, not found, write ops, total read, total write
Segmentation fault (core dumped)
compile the debug version and use the gdb to see the offset of the thread cache entry is overflow:
Thread 20 "bench" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7ffafd7f8700 (LWP 4346)]
0x00007ffff7e613a5 in memmove_mov_sse2_clwb () from /lib64/libpmem.so.1
Missing separate debuginfos, use: dnf debuginfo-install gflags-2.2.2-5.fc32.x86_64 libgcc-10.3.1-1.fc32.x86_64 libpmem-1.8-2.fc32.x86_64 libstdc++-10.3.1-1.fc32.x86_64
(gdb) bt
#0 0x00007ffff7e613a5 in memmove_mov_sse2_clwb () from /lib64/libpmem.so.1
#1 0x00007ffff7e49da6 in memmove_nodrain_sse2_clwb () from /lib64/libpmem.so.1
#2 0x00007ffff7e496cd in pmem_memcpy_persist () from /lib64/libpmem.so.1
#3 0x00007ffff7f8560a in kvdk::PMEMAllocator::Allocate (this=0x43bca0, size=4128) at /home/dennis/kvdk/engine/pmem_allocator.cpp:334
#4 0x00007ffff7f4e472 in kvdk::KVEngine::HashSetImpl (this=0x43ba90, key=..., value=..., dt=1, batch_hint=0x0) at /home/dennis/kvdk/engine/kv_engine.cpp:806
#5 0x00007ffff7f4e97c in kvdk::KVEngine::Set (this=0x43ba90, key="I\023\000\000\000\000\000",
value="0yVu77DE49c8Rj9wD4D7pCWGv9DBdsi96M6QHZD3M64pnOX9FB7c8bLqj3si5DiyjMtX497gkFeLpiePIP8L5Xg5bw7cALT9gGYqjRSKz4SIsCYOjg5VLp94IdMZ3saQB1X475fOxuF3Vas5u04aIxKff25DzR05401cg2QClimC6FsL53o182S5plURj25TrkJBAZ0Y"...) at /home/dennis/kvdk/engine/kv_engine.cpp:871
#6 0x000000000041282d in DBWrite (id=1) at /home/dennis/kvdk/benchmark/bench.cpp:200
#7 0x000000000041bfda in std::__invoke_impl<void, void ()(int), int> (__f=@0x43bf70: 0x4126ab <DBWrite(int)>) at /usr/include/c++/10/bits/invoke.h:60
#8 0x000000000041bf35 in std::__invoke<void ()(int), int> (__fn=@0x43bf70: 0x4126ab <DBWrite(int)>) at /usr/include/c++/10/bits/invoke.h:95
#9 0x000000000041bea5 in std::thread::_Invoker<std::tuple<void ()(int), int> >::_M_invoke<0ul, 1ul> (this=0x43bf68) at /usr/include/c++/10/thread:264
#10 0x000000000041be44 in std::thread::_Invoker<std::tuple<void ()(int), int> >::operator() (this=0x43bf68) at /usr/include/c++/10/thread:271
#11 0x000000000041bd80 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(int), int> > >::M_run (this=0x43bf60)
at /usr/include/c++/10/thread:215
#12 0x00007ffff7cfd994 in execute_native_thread_routine () from /lib64/libstdc++.so.6
#13 0x00007ffff7e8e432 in start_thread () from /lib64/libpthread.so.0
#14 0x00007ffff79fa6d3 in clone () from /lib64/libc.so.6
(gdb) f 3
#3 0x00007ffff7f8560a in kvdk::PMEMAllocator::Allocate (this=0x43bca0, size=4128) at /home/dennis/kvdk/engine/pmem_allocator.cpp:334
334 pmem_memcpy_persist(
(gdb) list
329 // Padding remaining space
330 auto extra_space = thread_cache.free_entry.size - b_size;
331 // TODO optimize, do not write PMEM
332 if (extra_space >= FREE_SPACE_PADDING_BLOCK) {
333 DataHeader header(0, extra_space);
334 pmem_memcpy_persist(
335 offset2addr(thread_cache.free_entry.space_entry.offset + b_size),
336 &header, sizeof(DataHeader));
337 } else {
338 b_size = thread_cache.free_entry.size;
(gdb) p header
$1 = {checksum = 0, b_size = 4152360919}
(gdb) p thread_cache
$2 = (kvdk::PMEMAllocator::ThreadCache &) @0x43bc60: {segment_offset = 4294968816, segment_usable_blocks = 49, free_entry = {space_entry = {
offset = 140737352581152, hash_entry_reference = 0x43bc40, hash_entry_mutex = 0x7ffff7800000}, size = 4152360984}, freelist = {
offsets = std::vector of length -178956965, capacity 5863877454623 = {}
(gdb) p thread_cache.free_entry.space_entry.offset
$3 = 140737352581152
(gdb) p /x thread_cache.free_entry.space_entry.offset
$4 = 0x7ffff7e84020
The offset looks like a address, please take a look to see if there is something wrong.
Hi, max_write_threads is max current write thread of kvdk, it should larger than write threads of benchmark (in your case, it's 2)
In fact, it is recommanded to set max_write_threads to the number of cores (or hyper threads) for performance.