x86_64: Unhandled user fault: addr=0 ip=7efcd10002b4 access=r-- sig=11 err=bad address
chenchun opened this issue · comments
Description
Presto sql failed and pods kept crash when I run tpcds test against presto runsc ptrace container with hostnet. Whereas if I change to netstack, the sql runs ok.
Running sample-queries-tpcds/query2.sql, presto output:
Query 20220316_075854_00007_ievhn failed: Encountered too many errors talking to a worker node. The node may have crashed or be under too much load. This is probably a transient issue, so please retry your query in a few minutes. (http://172.16.0.187:8080/v1/task/20220316_075854_00007_ievhn.2.15/results/13/0 - 100 failures, failure duration 300.23s, total failed request time 302.41s)
Lots of presto worker pod dies frequently when sql starts running.
runsc throws "Unhandled user fault".
D0316 15:56:09.379086 104310 transport_flipcall.go:127] send [channel @0xc0002a6600] Twalkgetattr{FID: 543, NewFID: 598, Names: [jfr.jar]}
D0316 15:56:09.379138 104310 transport_flipcall.go:238] recv [channel @0xc0002a6600] Rlerror{Error: 2}
D0316 15:56:09.379464 104310 transport_flipcall.go:127] send [channel @0xc0002a6600] Twalkgetattr{FID: 538, NewFID: 598, Names: [classes]}
D0316 15:56:09.379525 104310 transport_flipcall.go:238] recv [channel @0xc0002a6600] Rlerror{Error: 2}
D0316 15:56:09.379650 104310 transport_flipcall.go:127] send [channel @0xc0002a6600] Twalkgetattr{FID: 543, NewFID: 598, Names: [meta-index]}
D0316 15:56:09.379743 104310 transport_flipcall.go:238] recv [channel @0xc0002a6600] Rwalkgetattr{Valid: AttrMask{with: Mode NLink UID GID RDev ATime MTime CTime Size Blocks}, Attr: Attr{Mode: 0o100644, UID: 0, GID: 0, NLink
: 1, RDev: 0, Size: 1922, BlockSize: 4096, Blocks: 8, ATime: {Sec: 1540231523, NanoSec: 0}, MTime: {Sec: 1540231523, NanoSec: 0}, CTime: {Sec: 1647414483, NanoSec: 458739639}, BTime: {Sec: 0, NanoSec: 0}, Gen: 0, DataVersion:
0}, QIDs: [QID{Type: 0, Version: 0, Path: 1452687}]}
D0316 15:56:09.379757 104310 transport_flipcall.go:127] send [channel @0xc0002a6600] Twalk{FID: 598, NewFID: 599, Names: []}
D0316 15:56:09.379820 104310 transport_flipcall.go:238] recv [channel @0xc0002a6600] Rwalk{QIDs: []}
D0316 15:56:09.379831 104310 transport_flipcall.go:127] send [channel @0xc0002a6600] Tlopen{FID: 599, Flags: ReadOnly}
D0316 15:56:09.379888 104310 transport_flipcall.go:238] recv [channel @0xc0002a6600] Rlopen{QID: QID{Type: 0, Version: 0, Path: 1452687}, IoUnit: 0, File: FD: 275}
D0316 15:56:09.382509 104310 task_run.go:295] [ 4: 8] Unhandled user fault: addr=0 ip=7efcd10002b4 access=r-- sig=11 err=bad address
D0316 15:56:09.382541 104310 task_log.go:87] [ 4: 8] Registers:
D0316 15:56:09.382557 104310 task_log.go:94] [ 4: 8] Cs = 0000000000000033
D0316 15:56:09.382563 104310 task_log.go:94] [ 4: 8] Ds = 0000000000000000
D0316 15:56:09.382569 104310 task_log.go:94] [ 4: 8] Eflags = 0000000000210246
D0316 15:56:09.382575 104310 task_log.go:94] [ 4: 8] Es = 0000000000000000
D0316 15:56:09.382587 104310 task_log.go:94] [ 4: 8] Fs = 0000000000000000
D0316 15:56:09.382592 104310 task_log.go:94] [ 4: 8] Fs_base = 00007efce9d5b700
D0316 15:56:09.382598 104310 task_log.go:94] [ 4: 8] Gs = 0000000000000000
D0316 15:56:09.382603 104310 task_log.go:94] [ 4: 8] Gs_base = 0000000000000000
D0316 15:56:09.382609 104310 task_log.go:94] [ 4: 8] Orig_rax = ffffffffffffffff
D0316 15:56:09.382614 104310 task_log.go:94] [ 4: 8] R10 = 00007efcd1000364
D0316 15:56:09.382619 104310 task_log.go:94] [ 4: 8] R11 = 0000000000000000
D0316 15:56:09.382625 104310 task_log.go:94] [ 4: 8] R12 = 00007efce9d5aa30
D0316 15:56:09.382630 104310 task_log.go:94] [ 4: 8] R13 = 00007efce8119a18
D0316 15:56:09.382635 104310 task_log.go:94] [ 4: 8] R14 = 00007efce000cf50
D0316 15:56:09.382640 104310 task_log.go:94] [ 4: 8] R15 = 00007efce000b950
D0316 15:56:09.382646 104310 task_log.go:94] [ 4: 8] R8 = 0000000cffffffff
D0316 15:56:09.382653 104310 task_log.go:94] [ 4: 8] R9 = 000000000000000c
D0316 15:56:09.382658 104310 task_log.go:94] [ 4: 8] Rax = 0000000000000006
D0316 15:56:09.382663 104310 task_log.go:94] [ 4: 8] Rbp = 00007efce8149400
D0316 15:56:09.382669 104310 task_log.go:94] [ 4: 8] Rbx = 00000000030c0800
D0316 15:56:09.382674 104310 task_log.go:94] [ 4: 8] Rcx = 00000000cafebabe
D0316 15:56:09.382679 104310 task_log.go:94] [ 4: 8] Rdi = 00007efce8149400
D0316 15:56:09.382684 104310 task_log.go:94] [ 4: 8] Rdx = 0000000000000000
D0316 15:56:09.382691 104310 task_log.go:94] [ 4: 8] Rip = 00007efcd10002b4
D0316 15:56:09.382696 104310 task_log.go:94] [ 4: 8] Rsi = 0000000000000000
D0316 15:56:09.382702 104310 task_log.go:94] [ 4: 8] Rsp = 00007efce9d5a7d8
D0316 15:56:09.382707 104310 task_log.go:94] [ 4: 8] Ss = 000000000000002b
D0316 15:56:09.382712 104310 task_log.go:111] [ 4: 8] Stack:
D0316 15:56:09.382718 104310 task_log.go:128] [ 4: 8] 7efce9d5a7d0: 21 01 00 2c 00 00 00 00 46 02 00 00 00 00 00 00
D0316 15:56:09.382725 104310 task_log.go:128] [ 4: 8] 7efce9d5a7e0: 60 01 00 d1 fc 7e 00 00 bc 93 14 e8 fc 7e 00 00
D0316 15:56:09.382730 104310 task_log.go:128] [ 4: 8] 7efce9d5a7f0: 00 aa d5 e9 fc 7e 00 00 1d d7 c7 e7 fc 7e 00 00
D0316 15:56:09.382736 104310 task_log.go:128] [ 4: 8] 7efce9d5a800: 05 00 00 00 00 00 00 00 ff ff ff ff ff ff ff ff
D0316 15:56:09.382741 104310 task_log.go:128] [ 4: 8] 7efce9d5a810: ff ff ff ff f0 00 00 00 b8 d8 06 e8 fc 7e 00 00
D0316 15:56:09.382747 104310 task_log.go:128] [ 4: 8] 7efce9d5a820: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382752 104310 task_log.go:128] [ 4: 8] 7efce9d5a830: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382758 104310 task_log.go:128] [ 4: 8] 7efce9d5a840: 05 00 00 00 00 00 00 00 ff ff ff ff ff ff ff ff
D0316 15:56:09.382764 104310 task_log.go:128] [ 4: 8] 7efce9d5a850: ff ff ff ff 10 00 00 00 b8 d8 06 e8 fc 7e 00 00
D0316 15:56:09.382770 104310 task_log.go:128] [ 4: 8] 7efce9d5a860: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382775 104310 task_log.go:128] [ 4: 8] 7efce9d5a870: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382781 104310 task_log.go:128] [ 4: 8] 7efce9d5a880: 05 00 00 00 00 00 00 00 ff ff ff ff ff ff ff ff
D0316 15:56:09.382787 104310 task_log.go:128] [ 4: 8] 7efce9d5a890: ff ff ff ff 20 00 00 00 b8 d8 06 e8 fc 7e 00 00
D0316 15:56:09.382792 104310 task_log.go:128] [ 4: 8] 7efce9d5a8a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382797 104310 task_log.go:128] [ 4: 8] 7efce9d5a8b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382803 104310 task_log.go:128] [ 4: 8] 7efce9d5a8c0: 05 00 00 00 00 00 00 00 ff ff ff ff ff ff ff ff
D0316 15:56:09.382809 104310 task_log.go:128] [ 4: 8] 7efce9d5a8d0: ff ff ff ff 60 00 00 00 b8 d8 06 e8 fc 7e 00 00
D0316 15:56:09.382826 104310 task_log.go:128] [ 4: 8] 7efce9d5a8e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382832 104310 task_log.go:128] [ 4: 8] 7efce9d5a8f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382837 104310 task_log.go:128] [ 4: 8] 7efce9d5a900: 05 00 00 00 00 00 00 00 ff ff ff ff ff ff ff ff
D0316 15:56:09.382843 104310 task_log.go:128] [ 4: 8] 7efce9d5a910: ff ff ff ff 50 00 00 00 b8 d8 06 e8 fc 7e 00 00
D0316 15:56:09.382848 104310 task_log.go:128] [ 4: 8] 7efce9d5a920: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382854 104310 task_log.go:128] [ 4: 8] 7efce9d5a930: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382867 104310 task_log.go:128] [ 4: 8] 7efce9d5a940: 05 00 00 00 00 00 00 00 ff ff ff ff ff ff ff ff
D0316 15:56:09.382873 104310 task_log.go:128] [ 4: 8] 7efce9d5a950: ff ff ff ff 40 00 00 00 b8 d8 06 e8 fc 7e 00 00
D0316 15:56:09.382878 104310 task_log.go:128] [ 4: 8] 7efce9d5a960: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382884 104310 task_log.go:128] [ 4: 8] 7efce9d5a970: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382890 104310 task_log.go:128] [ 4: 8] 7efce9d5a980: 05 00 00 00 00 00 00 00 ff ff ff ff ff ff ff ff
D0316 15:56:09.382895 104310 task_log.go:128] [ 4: 8] 7efce9d5a990: ff ff ff ff 00 00 00 00 b8 d8 06 e8 fc 7e 00 00
D0316 15:56:09.382901 104310 task_log.go:128] [ 4: 8] 7efce9d5a9a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382906 104310 task_log.go:128] [ 4: 8] 7efce9d5a9b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382913 104310 task_log.go:128] [ 4: 8] 7efce9d5a9c0: 50 b9 00 e0 fc 7e 00 00 00 04 87 1b 8d 48 a9 a2
D0316 15:56:09.382918 104310 task_log.go:128] [ 4: 8] 7efce9d5a9d0: 00 aa d5 e9 fc 7e 00 00 60 aa d5 e9 fc 7e 00 00
D0316 15:56:09.382924 104310 task_log.go:128] [ 4: 8] 7efce9d5a9e0: 30 aa d5 e9 fc 7e 00 00 40 b9 00 e0 fc 7e 00 00
D0316 15:56:09.382930 104310 task_log.go:128] [ 4: 8] 7efce9d5a9f0: 50 cf 00 e0 fc 7e 00 00 50 b9 00 e0 fc 7e 00 00
D0316 15:56:09.382936 104310 task_log.go:128] [ 4: 8] 7efce9d5aa00: 20 ac d5 e9 fc 7e 00 00 06 eb c7 e7 fc 7e 00 00
D0316 15:56:09.382941 104310 task_log.go:128] [ 4: 8] 7efce9d5aa10: 90 aa d5 e9 fc 7e 00 00 d8 03 00 00 00 00 00 00
D0316 15:56:09.382947 104310 task_log.go:128] [ 4: 8] 7efce9d5aa20: d0 18 0f e8 fc 7e 00 00 28 bd 00 e0 fc 7e 00 00
D0316 15:56:09.382952 104310 task_log.go:128] [ 4: 8] 7efce9d5aa30: d0 18 0f e8 fc 7e 00 00 50 b9 00 e0 fc 7e 00 00
D0316 15:56:09.382958 104310 task_log.go:128] [ 4: 8] 7efce9d5aa40: 70 b4 01 e0 fc 7e 00 00 70 b4 01 e0 fc 7e 00 00
D0316 15:56:09.382964 104310 task_log.go:128] [ 4: 8] 7efce9d5aa50: 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00
D0316 15:56:09.382971 104310 task_log.go:128] [ 4: 8] 7efce9d5aa60: 21 2c cb e7 fc 7e 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382977 104310 task_log.go:128] [ 4: 8] 7efce9d5aa70: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382982 104310 task_log.go:128] [ 4: 8] 7efce9d5aa80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382988 104310 task_log.go:128] [ 4: 8] 7efce9d5aa90: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.382993 104310 task_log.go:128] [ 4: 8] 7efce9d5aaa0: 00 00 00 00 00 00 00 00 00 00 00 00 7c 00 00 00
D0316 15:56:09.382999 104310 task_log.go:128] [ 4: 8] 7efce9d5aab0: 60 aa d5 e9 fc 7e 00 00 60 01 00 d1 fc 7e 00 00
D0316 15:56:09.383004 104310 task_log.go:128] [ 4: 8] 7efce9d5aac0: 00 00 00 00 00 00 00 00 7b 03 00 d1 fc 7e 00 00
D0316 15:56:09.383010 104310 task_log.go:128] [ 4: 8] 7efce9d5aad0: b8 03 00 d1 fc 7e 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.383015 104310 task_log.go:128] [ 4: 8] 7efce9d5aae0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.383021 104310 task_log.go:128] [ 4: 8] 7efce9d5aaf0: 60 01 00 d1 fc 7e 00 00 00 00 01 00 00 00 00 00
D0316 15:56:09.383029 104310 task_log.go:128] [ 4: 8] 7efce9d5ab00: 60 aa d5 e9 fc 7e 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.383035 104310 task_log.go:128] [ 4: 8] 7efce9d5ab10: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.383042 104310 task_log.go:128] [ 4: 8] 7efce9d5ab20: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.383047 104310 task_log.go:128] [ 4: 8] 7efce9d5ab30: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.383053 104310 task_log.go:128] [ 4: 8] 7efce9d5ab40: 00 00 00 00 00 00 00 00 00 00 02 e7 fc 7e 00 00
D0316 15:56:09.383058 104310 task_log.go:128] [ 4: 8] 7efce9d5ab50: 60 aa d5 e9 fc 7e 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.383063 104310 task_log.go:128] [ 4: 8] 7efce9d5ab60: 00 00 00 00 00 00 00 00 60 01 00 d1 fc 7e 00 00
D0316 15:56:09.383069 104310 task_log.go:128] [ 4: 8] 7efce9d5ab70: 58 02 00 00 fc 7e 00 00 88 ab d5 e9 fc 7e 00 00
D0316 15:56:09.383074 104310 task_log.go:128] [ 4: 8] 7efce9d5ab80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.383080 104310 task_log.go:128] [ 4: 8] 7efce9d5ab90: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.383086 104310 task_log.go:128] [ 4: 8] 7efce9d5aba0: 00 00 00 00 00 00 00 00 00 04 87 1b 8d 48 a9 a2
D0316 15:56:09.383091 104310 task_log.go:128] [ 4: 8] 7efce9d5abb0: 00 00 00 00 00 00 00 00 b0 02 ef 27 61 55 00 00
D0316 15:56:09.383097 104310 task_log.go:128] [ 4: 8] 7efce9d5abc0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
D0316 15:56:09.383102 104310 task_log.go:149] [ 4: 8] Code:
D0316 15:56:09.383107 104310 task_log.go:167] [ 4: 8] 7efcd1000270: 18 81 f9 00 00 00 18 75 57 33 c9 0f 01 d0 48 8d
D0316 15:56:09.383113 104310 task_log.go:167] [ 4: 8] 7efcd1000280: b5 f0 00 00 00 89 06 89 56 04 83 e0 06 83 f8 06
D0316 15:56:09.383119 104310 task_log.go:167] [ 4: 8] 7efcd1000290: 75 3e b9 be ba fe ca c5 f9 6e c1 c5 f9 70 c0 00
D0316 15:56:09.383124 104310 task_log.go:167] [ 4: 8] 7efcd10002a0: c4 e3 7d 18 c0 01 c5 fe 6f f8 c5 7e 6f c0 c5 7e
D0316 15:56:09.383130 104310 task_log.go:167] [ 4: 8] 7efcd10002b0: 6f f8 33 f6 8b 06 48 8d b5 f8 00 00 00 c5 fe 7f
D0316 15:56:09.383135 104310 task_log.go:167] [ 4: 8] 7efcd10002c0: 06 c5 fe 7f 7e 20 c5 7e 7f 46 40 c5 7e 7f 7e 60
D0316 15:56:09.383140 104310 task_log.go:167] [ 4: 8] 7efcd10002d0: b8 07 00 00 00 3b 45 00 7f 0d 33 c9 0f a2 48 8d
D0316 15:56:09.383146 104310 task_log.go:167] [ 4: 8] 7efcd10002e0: 75 30 89 06 89 5e 04 b8 00 00 00 80 0f a2 81 f8
D0316 15:56:09.383151 104310 task_log.go:71] [ 4: 8] Mappings:
VMAs:
556127cec000-556127ced000 r-xp 00000000 00:0f 712 /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
556127eec000-556127eed000 r--p 00000000 00:0f 712 /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
556127eed000-556127eee000 rw-p 00001000 00:0f 712 /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
556127eee000-556127f0f000 rw-p 00000000 00:00 0 [heap]
7efcd1000000-7efcd1270000 rwxp 00000000 00:00 0
7efcd1270000-7efce0000000 ---p 00000000 00:00 0
7efce0000000-7efce0021000 rw-p 00000000 00:00 0
7efce0021000-7efce4000000 ---p 00000000 00:00 0
7efce4200000-7efce420a000 rw-p 00000000 00:00 0
7efce420a000-7efce45c0000 ---p 00000000 00:00 0
7efce4600000-7efce4608000 r-xp 00000000 00:0f 891 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libzip.so
7efce4608000-7efce4807000 ---p 00008000 00:0f 891 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libzip.so
7efce4807000-7efce4808000 r--p 00007000 00:0f 891 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libzip.so
7efce4808000-7efce4809000 rw-p 00008000 00:0f 891 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libzip.so
7efce4a00000-7efce4a0a000 r-xp 00000000 00:0f 888 /lib/x86_64-linux-gnu/libnss_files-2.24.so
7efce4a0a000-7efce4c0a000 ---p 0000a000 00:0f 888 /lib/x86_64-linux-gnu/libnss_files-2.24.so
7efce4c0a000-7efce4c0b000 r--p 0000a000 00:0f 888 /lib/x86_64-linux-gnu/libnss_files-2.24.so
7efce4c0b000-7efce4c0c000 rw-p 0000b000 00:0f 888 /lib/x86_64-linux-gnu/libnss_files-2.24.so
7efce4c0c000-7efce4c12000 rw-p 00000000 00:00 0
7efce4e00000-7efce4e0b000 r-xp 00000000 00:0f 886 /lib/x86_64-linux-gnu/libnss_nis-2.24.so
7efce4e0b000-7efce500a000 ---p 0000b000 00:0f 886 /lib/x86_64-linux-gnu/libnss_nis-2.24.so
7efce500a000-7efce500b000 r--p 0000a000 00:0f 886 /lib/x86_64-linux-gnu/libnss_nis-2.24.so
7efce500b000-7efce500c000 rw-p 0000b000 00:0f 886 /lib/x86_64-linux-gnu/libnss_nis-2.24.so
7efce5200000-7efce5214000 r-xp 00000000 00:0f 884 /lib/x86_64-linux-gnu/libnsl-2.24.so
7efce5214000-7efce5414000 ---p 00014000 00:0f 884 /lib/x86_64-linux-gnu/libnsl-2.24.so
7efce5414000-7efce5415000 r--p 00014000 00:0f 884 /lib/x86_64-linux-gnu/libnsl-2.24.so
7efce5415000-7efce5416000 rw-p 00015000 00:0f 884 /lib/x86_64-linux-gnu/libnsl-2.24.so
7efce5416000-7efce5418000 rw-p 00000000 00:00 0
7efce5600000-7efce5607000 r-xp 00000000 00:0f 882 /lib/x86_64-linux-gnu/libnss_compat-2.24.so
7efce5607000-7efce5806000 ---p 00007000 00:0f 882 /lib/x86_64-linux-gnu/libnss_compat-2.24.so
7efce5806000-7efce5807000 r--p 00006000 00:0f 882 /lib/x86_64-linux-gnu/libnss_compat-2.24.so
7efce5807000-7efce5808000 rw-p 00007000 00:0f 882 /lib/x86_64-linux-gnu/libnss_compat-2.24.so
7efce5a00000-7efce5a2d000 r-xp 00000000 00:0f 717 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjava.so
7efce5a2d000-7efce5c2d000 ---p 0002d000 00:0f 717 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjava.so
7efce5c2d000-7efce5c2e000 r--p 0002d000 00:0f 717 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjava.so
7efce5c2e000-7efce5c30000 rw-p 0002e000 00:0f 717 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjava.so
7efce5e00000-7efce5e0f000 r-xp 00000000 00:0f 876 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libverify.so
7efce5e0f000-7efce600f000 ---p 0000f000 00:0f 876 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libverify.so
7efce600f000-7efce6011000 r--p 0000f000 00:0f 876 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libverify.so
7efce6011000-7efce6012000 rw-p 00011000 00:0f 876 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libverify.so
7efce6200000-7efce6207000 r-xp 00000000 00:0f 875 /lib/x86_64-linux-gnu/librt-2.24.so
7efce6207000-7efce6406000 ---p 00007000 00:0f 875 /lib/x86_64-linux-gnu/librt-2.24.so
7efce6406000-7efce6407000 r--p 00006000 00:0f 875 /lib/x86_64-linux-gnu/librt-2.24.so
7efce6407000-7efce6408000 rw-p 00007000 00:0f 875 /lib/x86_64-linux-gnu/librt-2.24.so
7efce6600000-7efce6616000 r-xp 00000000 00:0f 725 /lib/x86_64-linux-gnu/libgcc_s.so.1
7efce6616000-7efce6815000 ---p 00016000 00:0f 725 /lib/x86_64-linux-gnu/libgcc_s.so.1
7efce6815000-7efce6816000 r--p 00015000 00:0f 725 /lib/x86_64-linux-gnu/libgcc_s.so.1
7efce6816000-7efce6817000 rw-p 00016000 00:0f 725 /lib/x86_64-linux-gnu/libgcc_s.so.1
7efce6a00000-7efce6b03000 r-xp 00000000 00:0f 46 /lib/x86_64-linux-gnu/libm-2.24.so
7efce6b03000-7efce6d02000 ---p 00103000 00:0f 46 /lib/x86_64-linux-gnu/libm-2.24.so
7efce6d02000-7efce6d03000 r--p 00102000 00:0f 46 /lib/x86_64-linux-gnu/libm-2.24.so
7efce6d03000-7efce6d04000 rw-p 00103000 00:0f 46 /lib/x86_64-linux-gnu/libm-2.24.so
7efce6e00000-7efce6f72000 r-xp 00000000 00:0f 724 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.22
7efce6f72000-7efce7172000 ---p 00172000 00:0f 724 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.22
7efce7172000-7efce717c000 r--p 00172000 00:0f 724 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.22
7efce717c000-7efce717e000 rw-p 0017c000 00:0f 724 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.22
7efce717e000-7efce7182000 rw-p 00000000 00:00 0
7efce7200000-7efce7e5f000 r-xp 00000000 00:0f 722 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so
7efce7e5f000-7efce805e000 ---p 00c5f000 00:0f 722 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so
7efce805e000-7efce80f2000 r--p 00c5e000 00:0f 722 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so
7efce80f2000-7efce811b000 rw-p 00cf2000 00:0f 722 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so
7efce811b000-7efce814b000 rw-p 00000000 00:00 0
7efce8200000-7efce8395000 r-xp 00000000 00:0f 48 /lib/x86_64-linux-gnu/libc-2.24.so
7efce8395000-7efce8595000 ---p 00195000 00:0f 48 /lib/x86_64-linux-gnu/libc-2.24.so
7efce8595000-7efce8599000 r--p 00195000 00:0f 48 /lib/x86_64-linux-gnu/libc-2.24.so
7efce8599000-7efce859b000 rw-p 00199000 00:0f 48 /lib/x86_64-linux-gnu/libc-2.24.so
7efce859b000-7efce859f000 rw-p 00000000 00:00 0
7efce8600000-7efce8603000 r-xp 00000000 00:0f 38 /lib/x86_64-linux-gnu/libdl-2.24.so
7efce8603000-7efce8802000 ---p 00003000 00:0f 38 /lib/x86_64-linux-gnu/libdl-2.24.so
7efce8802000-7efce8803000 r--p 00002000 00:0f 38 /lib/x86_64-linux-gnu/libdl-2.24.so
7efce8803000-7efce8804000 rw-p 00003000 00:0f 38 /lib/x86_64-linux-gnu/libdl-2.24.so
7efce8a00000-7efce8a0d000 r-xp 00000000 00:0f 716 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/jli/libjli.so
7efce8a0d000-7efce8c0d000 ---p 0000d000 00:0f 716 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/jli/libjli.so
7efce8c0d000-7efce8c0e000 r--p 0000d000 00:0f 716 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/jli/libjli.so
7efce8c0e000-7efce8c0f000 rw-p 0000e000 00:0f 716 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/jli/libjli.so
7efce8e00000-7efce8e19000 r-xp 00000000 00:0f 44 /lib/x86_64-linux-gnu/libz.so.1.2.8
7efce8e19000-7efce9018000 ---p 00019000 00:0f 44 /lib/x86_64-linux-gnu/libz.so.1.2.8
7efce9018000-7efce9019000 r--p 00018000 00:0f 44 /lib/x86_64-linux-gnu/libz.so.1.2.8
7efce9019000-7efce901a000 rw-p 00019000 00:0f 44 /lib/x86_64-linux-gnu/libz.so.1.2.8
7efce9200000-7efce9218000 r-xp 00000000 00:0f 36 /lib/x86_64-linux-gnu/libpthread-2.24.so
7efce9218000-7efce9417000 ---p 00018000 00:0f 36 /lib/x86_64-linux-gnu/libpthread-2.24.so
7efce9417000-7efce9418000 r--p 00017000 00:0f 36 /lib/x86_64-linux-gnu/libpthread-2.24.so
7efce9418000-7efce9419000 rw-p 00018000 00:0f 36 /lib/x86_64-linux-gnu/libpthread-2.24.so
7efce9419000-7efce941d000 rw-p 00000000 00:00 0
7efce9600000-7efce9601000 r-xp 00000000 00:0f 704 /presto/bin/procname/Linux-x86_64/libprocname.so
7efce9601000-7efce9800000 ---p 00001000 00:0f 704 /presto/bin/procname/Linux-x86_64/libprocname.so
7efce9800000-7efce9801000 rw-p 00000000 00:0f 704 /presto/bin/procname/Linux-x86_64/libprocname.so
7efce9a00000-7efce9a23000 r-xp 00000000 00:0f 33 /lib/x86_64-linux-gnu/ld-2.24.so
7efce9c23000-7efce9c24000 r--p 00023000 00:0f 33 /lib/x86_64-linux-gnu/ld-2.24.so
7efce9c24000-7efce9c25000 rw-p 00024000 00:0f 33 /lib/x86_64-linux-gnu/ld-2.24.so
7efce9c25000-7efce9c26000 rw-p 00000000 00:00 0
7efce9c53000-7efce9c5b000 rw-s 00000000 00:0f 890 /tmp/hsperfdata_presto/3
7efce9c5b000-7efce9c5f000 ---p 00000000 00:00 0
7efce9c5f000-7efce9d5e000 rw-p 00000000 00:00 0
7efce9d61000-7efce9d62000 rw-p 00000000 00:00 0
7efce9d62000-7efce9d63000 r--p 00000000 00:00 0
7efce9d63000-7efce9d65000 rw-p 00000000 00:00 0
7efce9d65000-7efce9d66000 r--p 00000000 00:00 0 [vvar]
7efce9d66000-7efce9d68000 r-xp 00000000 00:00 0
7fb88ba1d000-7fb88c21d000 rw-p 00000000 00:00 0 [stack]
PMAs:
556127cec000-556127ced000 r-xp 00450000 *pgalloc.MemoryFile
556127eec000-556127eed000 r--p 005df000 *pgalloc.MemoryFile
556127eed000-556127eee000 rw-p 005e0000 *pgalloc.MemoryFile
556127eee000-556127f0f000 rw-p 00860000 *pgalloc.MemoryFile
7efcd1000000-7efcd1200000 rwxp 01200000 *pgalloc.MemoryFile
7efce0000000-7efce0021000 rw-p 009be000 *pgalloc.MemoryFile
7efce4200000-7efce420a000 rw-p 00896000 *pgalloc.MemoryFile
7efce4600000-7efce4608000 r-xs 00000000 *gofer.dentryPlatformFile
7efce4807000-7efce4808000 r--p 00894000 *pgalloc.MemoryFile
7efce4808000-7efce4809000 rw-p 00895000 *pgalloc.MemoryFile
7efce4a00000-7efce4a0a000 r-xs 00000000 *gofer.dentryPlatformFile
7efce4c0a000-7efce4c0b000 r--p 00889000 *pgalloc.MemoryFile
7efce4c0b000-7efce4c12000 rw-p 0088a000 *pgalloc.MemoryFile
7efce4e00000-7efce4e0b000 r-xs 00000000 *gofer.dentryPlatformFile
7efce500a000-7efce500b000 r--p 00887000 *pgalloc.MemoryFile
7efce500b000-7efce500c000 rw-p 00888000 *pgalloc.MemoryFile
7efce5200000-7efce5214000 r-xs 00000000 *gofer.dentryPlatformFile
7efce5414000-7efce5415000 r--p 00883000 *pgalloc.MemoryFile
7efce5415000-7efce5418000 rw-p 00884000 *pgalloc.MemoryFile
7efce5600000-7efce5607000 r-xs 00000000 *gofer.dentryPlatformFile
7efce5806000-7efce5807000 r--p 00881000 *pgalloc.MemoryFile
7efce5807000-7efce5808000 rw-p 00882000 *pgalloc.MemoryFile
7efce5a00000-7efce5a2d000 r-xs 00000000 *gofer.dentryPlatformFile
7efce5c2d000-7efce5c2e000 r--p 006c3000 *pgalloc.MemoryFile
7efce5c2e000-7efce5c30000 rw-p 006c4000 *pgalloc.MemoryFile
7efce5e00000-7efce5e0f000 r-xs 00000000 *gofer.dentryPlatformFile
7efce600f000-7efce6011000 r--p 006be000 *pgalloc.MemoryFile
7efce6011000-7efce6012000 rw-p 006c0000 *pgalloc.MemoryFile
7efce6200000-7efce6207000 r-xs 00000000 *gofer.dentryPlatformFile
7efce6406000-7efce6407000 r--p 006bc000 *pgalloc.MemoryFile
7efce6407000-7efce6408000 rw-p 006bd000 *pgalloc.MemoryFile
7efce6600000-7efce6616000 r-xs 00000000 *gofer.dentryPlatformFile
7efce6815000-7efce6816000 r--p 006ba000 *pgalloc.MemoryFile
7efce6816000-7efce6817000 rw-p 006bb000 *pgalloc.MemoryFile
7efce6a00000-7efce6b03000 r-xs 00000000 *gofer.dentryPlatformFile
7efce6d02000-7efce6d03000 r--p 005e4000 *pgalloc.MemoryFile
7efce6d03000-7efce6d04000 rw-p 005e5000 *pgalloc.MemoryFile
7efce6e00000-7efce6f72000 r-xs 00000000 *gofer.dentryPlatformFile
7efce7172000-7efce717c000 r--p 00604000 *pgalloc.MemoryFile
7efce717c000-7efce7182000 rw-p 0060e000 *pgalloc.MemoryFile
7efce7200000-7efce7e5f000 r-xs 00000000 *gofer.dentryPlatformFile
7efce805e000-7efce80f2000 r--p 00901000 *pgalloc.MemoryFile
7efce80f2000-7efce811b000 rw-p 00995000 *pgalloc.MemoryFile
7efce811b000-7efce814b000 rw-p 00b3b000 *pgalloc.MemoryFile
7efce8200000-7efce8395000 r-xs 00000000 *gofer.dentryPlatformFile
7efce8595000-7efce8599000 r--p 00449000 *pgalloc.MemoryFile
7efce8599000-7efce859b000 rw-p 0044d000 *pgalloc.MemoryFile
7efce859b000-7efce859f000 rw-p 005d9000 *pgalloc.MemoryFile
7efce8600000-7efce8603000 r-xs 00000000 *gofer.dentryPlatformFile
7efce8802000-7efce8803000 r--p 00447000 *pgalloc.MemoryFile
7efce8803000-7efce8804000 rw-p 00448000 *pgalloc.MemoryFile
7efce8a00000-7efce8a0d000 r-xs 00000000 *gofer.dentryPlatformFile
7efce8c0d000-7efce8c0e000 r--p 00445000 *pgalloc.MemoryFile
7efce8c0e000-7efce8c0f000 rw-p 00446000 *pgalloc.MemoryFile
7efce8e00000-7efce8e19000 r-xs 00000000 *gofer.dentryPlatformFile
7efce9018000-7efce9019000 r--p 00443000 *pgalloc.MemoryFile
7efce9019000-7efce901a000 rw-p 00444000 *pgalloc.MemoryFile
7efce9200000-7efce9218000 r-xs 00000000 *gofer.dentryPlatformFile
7efce9417000-7efce9418000 r--p 003d9000 *pgalloc.MemoryFile
7efce9418000-7efce9419000 rw-p 003da000 *pgalloc.MemoryFile
7efce9419000-7efce941d000 rw-p 0043f000 *pgalloc.MemoryFile
7efce9600000-7efce9601000 r-xs 00000000 *gofer.dentryPlatformFile
7efce9800000-7efce9801000 rw-p 00225000 *pgalloc.MemoryFile
7efce9a00000-7efce9a22000 r-xs 00000000 *gofer.dentryPlatformFile
7efce9a22000-7efce9a23000 r-xp 005e1000 *pgalloc.MemoryFile
7efce9c23000-7efce9c24000 r--p 005e2000 *pgalloc.MemoryFile
7efce9c24000-7efce9c25000 rw-p 005e3000 *pgalloc.MemoryFile
7efce9c25000-7efce9c26000 rw-p 00224000 *pgalloc.MemoryFile
7efce9c53000-7efce9c5b000 rw-s 00000000 *gofer.dentryPlatformFile
7efce9c5b000-7efce9c5c000 ---p 0106b000 *pgalloc.MemoryFile
7efce9c5c000-7efce9c5f000 ---p 00891000 *pgalloc.MemoryFile
7efce9c5f000-7efce9d5c000 rw-p 0106f000 *pgalloc.MemoryFile
7efce9d5c000-7efce9d5e000 rw-p 005dd000 *pgalloc.MemoryFile
7efce9d61000-7efce9d62000 rw-p 0044f000 *pgalloc.MemoryFile
7efce9d62000-7efce9d63000 r--p 003db000 *pgalloc.MemoryFile
7efce9d63000-7efce9d65000 rw-p 00228000 *pgalloc.MemoryFile
7efce9d65000-7efce9d66000 r--s 00002000 *pgalloc.MemoryFile
7efce9d66000-7efce9d68000 r-xs 00000000 *pgalloc.MemoryFile
7fb88c200000-7fb88c21d000 rw-p 005e7000 *pgalloc.MemoryFile
D0316 15:56:09.383526 104310 task_log.go:73] [ 4: 8] FDTable:
fd:0 => name /dev/null
fd:1 => name host:[4]
fd:2 => name host:[4]
D0316 15:56:09.383557 104310 task_signals.go:468] [ 4: 8] Notified of signal 11
D0316 15:56:09.383569 104310 task_signals.go:221] [ 4: 8] Signal 11: delivering to handler
D0316 15:56:09.487657 104310 transport_flipcall.go:127] send [channel @0xc0002a6600] Twalk{FID: 594, NewFID: 600, Names: []}
D0316 15:56:09.487791 104310 transport_flipcall.go:238] recv [channel @0xc0002a6600] Rwalk{QIDs: []}
D0316 15:56:09.487805 104310 transport_flipcall.go:127] send [channel @0xc0002a6600] Tlopen{FID: 600, Flags: ReadOnly}
D0316 15:56:09.487908 104310 transport_flipcall.go:238] recv [channel @0xc0002a6600] Rlopen{QID: QID{Type: 0, Version: 0, Path: 1452671}, IoUnit: 0, File: FD: 276}
D0316 15:56:09.519926 104310 transport_flipcall.go:127] send [channel @0xc0002a6600] Twalkgetattr{FID: 494, NewFID: 601, Names: [.hotspot_compiler]}
D0316 15:56:09.520279 104310 transport_flipcall.go:238] recv [channel @0xc0002a6600] Rlerror{Error: 2}
D0316 15:56:09.588547 104310 transport_flipcall.go:127] send [channel @0xc0002a6600] Twalkgetattr{FID: 543, NewFID: 601, Names: [ext]}
D0316 15:56:09.588754 104310 transport_flipcall.go:238] recv [channel @0xc0002a6600] Rwalkgetattr{Valid: AttrMask{with: Mode NLink UID GID RDev ATime MTime CTime Size Blocks}, Attr: Attr{Mode: 0o40755, UID: 0, GID: 0, NLink: 2, RDev: 0, Size: 4096, BlockSize: 4096, Blocks: 8, ATime: {Sec: 1549441859, NanoSec: 0}, MTime: {Sec: 1549441859, NanoSec: 0}, CTime: {Sec: 1647414484, NanoSec: 946765054}, BTime: {Sec: 0, NanoSec: 0}, Gen: 0, DataVersion: 0}, QIDs: [QID{Type: 128, Version: 0, Path: 1710998}]}
Steps to reproduce
I run tpcds test follow by https://github.com/hortonworks/hive-testbench.
I installed presto by https://github.com/wiwdata/presto-chart.
runsc version
latest master
No response
docker version (if using docker)
# containerd --version
containerd github.com/containerd/containerd v1.4.3 269548fa27e0089a8b8278fc4fc781d7f65a939buname
No response
kubectl (if using Kubernetes)
No response
repo state (if built from source)
No response
runsc debug logs (if available)
No response
Could you run it with strace enabled and provide the strace logs here?
Probably found what cause this problem but do not have proper solution.. @hbhasker
What happened in this issue:
- thread is in sentry context to send package using write/send
- Read end of this socket is closed, thread gets EPIPE and signal SIGPIPE
- sentry delivers SIGPIPE to init process of this pod.
- If init process use default action for SIGPIPE, it exits with 141.
So we may have two method to solve it.
- Do not forward SIGPIPE to sandbox.
- Forward SIGPIPE to right process.
Solution 1 is not good enough because we change the behavior of process.
Solution 2 is better but I have no idea to do this.
The reproducing step is as following.
- compile the following code
gcc -lpthread server.c
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <time.h>
#include<pthread.h>
#include <signal.h>
const char *addr ="127.0.0.1";
int client(void)
{
int sockfd = 0, n = 0;
struct sockaddr_in serv_addr;
if((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
{
printf("\n Error : Could not create socket \n");
return 1;
}
memset(&serv_addr, '0', sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_port = htons(5000);
if(inet_pton(AF_INET, addr, &serv_addr.sin_addr)<=0)
{
printf("\n inet_pton error occured\n");
return 1;
}
if( connect(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0)
{
printf("\n Error : Connect Failed \n");
return 1;
}
sleep(1);
n = write(sockfd, "x", 1);
if (n < 0) {
perror("[client] failed to send");
}
printf("[client] sending for 2st time, expect broken pipe\n");
n = write(sockfd, "x", 1);
if (n < 0) {
perror("[client] failed to send");
}
return 0;
}
int server()
{
int listenfd = 0, connfd = 0;
struct sockaddr_in serv_addr;
listenfd = socket(AF_INET, SOCK_STREAM, 0);
memset(&serv_addr, '0', sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
serv_addr.sin_port = htons(5000);
bind(listenfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr));
listen(listenfd, 10);
connfd = accept(listenfd, (struct sockaddr*)NULL, NULL);
close(connfd);
printf("[server: fd closed\n");
}
void *thread_server(void *ptr) {
printf("server starting\n");
server();
}
void *thread_client(void *ptr) {
printf("client starting\n");
client();
}
void main(void) {
pthread_t thread1, thread2;
signal(SIGPIPE, SIG_IGN);
pthread_create(&thread1, NULL, *thread_server, NULL);
sleep(1);
pthread_create(&thread2, NULL, *thread_client, NULL);
pause();
}
- run a container with sleep as init and with network=host.
docker run -d --name=test-sigpipe --runtime=runsc-hostnet -v $(pwd):/tmp ubuntu sleep 10000
- exec into test-sigpipe and execute.
/tmp/a.out
docker exec -ti test-sigpipe bash
[root@dc0a494aab7d /]# /tmp/a.out
server starting
client starting
[server: fd closed
[client] sending for 2st time, expect broken pipe
- container exited with signal 13
$ docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
dc0a494aab7d ubuntu "sleep 10000" 2 minutes ago Exited (141) 2 minutes ago test-sigpipe
@nixprime probably has better suggestions on signal handling. Though unclear on why sentry is sending the signal to the sleep rather than then binary that is doing the write. Let me look at the signal handling and get back to you.