The kernel crash in tunnel creation
apssoft opened this issue · comments
We use the gtp5g revison 508c558.
When we send the large number of GTP-U packets (more than 10,000,000 pps), the GTP-U tunnel is create by the command `gtp5g-tunnel'. The kernel is crashed by the zero-divide.
The crash occurs at this point.
static struct gtp5g_pdr *pdr_find_by_gtp1u(struct gtp5g_dev *gtp, struct sk_buff *skb,
unsigned int hdrlen, u32 teid) {
struct iphdr *iph;
__be32 *target_addr;
struct hlist_head *head;
struct gtp5g_pdr *pdr;
struct gtp5g_pdi *pdi;
switch(ntohs(skb->protocol)) {
case ETH_P_IP:
break;
default:
return NULL;
}
if (!pskb_may_pull(skb, hdrlen + sizeof(struct iphdr)))
return NULL;
iph = (struct iphdr *)(skb->data + hdrlen);
target_addr = (gtp->role == GTP5G_ROLE_UPF ? &iph->saddr : &iph->daddr);
head = >p->i_teid_hash[u32_hashfn(teid) % gtp->hash_size]; <= crash here
hlist_for_each_entry_rcu(pdr, head, hlist_i_teid) {
pdi = pdr->pdi;
// GTP-U packet must check teid
if (!(pdi->f_teid && pdi->f_teid->teid == teid))
continue;
And the crash log and the backtrace are these.
KERNEL: /usr/lib/debug/boot/vmlinux-5.0.0-23-generic
DUMPFILE: /var/crash/202006251024/dump.202006251024 [PARTIAL DUMP]
CPUS: 4
DATE: Thu Jun 25 10:22:59 2020
UPTIME: 00:04:20
LOAD AVERAGE: 0.51, 0.42, 0.18
TASKS: 250
NODENAME: GTP-U
RELEASE: 5.0.0-23-generic
VERSION: #24~18.04.1-Ubuntu SMP Mon Jul 29 16:12:28 UTC 2019
MACHINE: x86_64 (2261 Mhz)
MEMORY: 4 GB
PANIC: "divide error: 0000 [#1] SMP PTI"
PID: 9
COMMAND: "ksoftirqd/0"
TASK: ffff9faebaff0000 [THREAD_INFO: ffff9faebaff0000]
CPU: 0
STATE: TASK_RUNNING (PANIC)
crash> bt -l
PID: 9 TASK: ffff9faebaff0000 CPU: 0 COMMAND: "ksoftirqd/0"
#0 [ffffbe9080687680] machine_kexec at ffffffff82e6b583
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/arch/x86/kernel/machine_kexec_64.c: 346
#1 [ffffbe90806876e0] __crash_kexec at ffffffff82f43742
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/kernel/kexec_core.c: 957
#2 [ffffbe90806877b0] crash_kexec at ffffffff82f445e1
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/include/linux/compiler.h: 219
#3 [ffffbe90806877d0] oops_end at ffffffff82e3379d
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/arch/x86/kernel/dumpstack.c: 334
#4 [ffffbe90806877f8] die at ffffffff82e33f82
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/arch/x86/kernel/dumpstack.c: 406
#5 [ffffbe9080687828] do_trap at ffffffff82e2fb8e
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/arch/x86/kernel/traps.c: 212
#6 [ffffbe9080687870] do_error_trap at ffffffff82e2fffc
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/arch/x86/kernel/traps.c: 278
#7 [ffffbe90806878b8] do_divide_error at ffffffff82e303f8
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/arch/x86/kernel/traps.c: 289
#8 [ffffbe90806878e0] divide_error at ffffffff83a00ba4
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/arch/x86/entry/entry_64.S: 970
[exception RIP: gtp5g_encap_recv+431]
RIP: ffffffffc065841f RSP: ffffbe9080687990 RFLAGS: 00010246
RAX: 0000000054f51ee2 RBX: ffff9fadeec80200 RCX: 00000000ef536f6a
RDX: 0000000000000000 RSI: 000000007c163994 RDI: 0000000000000000
RBP: ffffbe9080687a28 R8: 0000000000000000 R9: 0000000000000010
R10: ffff9fade135e940 R11: 0000000000000024 R12: ffff9fade2ce7940
R13: 0000000000000008 R14: ffff9fade135e924 R15: 00000000ea030000
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
/home/*****/gtp5g/gtp5g.c: 1370
#9 [ffffbe9080687a30] udp_queue_rcv_one_skb at ffffffff83755c84
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/ipv4/udp.c: 2012
#10 [ffffbe9080687a68] udp_queue_rcv_skb at ffffffff83755f3f
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/ipv4/udp.c: 2098
#11 [ffffbe9080687a90] udp_unicast_rcv_skb at ffffffff83756117
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/ipv4/udp.c: 2251
#12 [ffffbe9080687aa0] __udp4_lib_rcv at ffffffff83756d8a
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/ipv4/udp.c: 2312
#13 [ffffbe9080687b28] udp_rcv at ffffffff8375789a
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/ipv4/udp.c: 2483
#14 [ffffbe9080687b38] ip_protocol_deliver_rcu at ffffffff8371e415
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/ipv4/ip_input.c: 209
#15 [ffffbe9080687b60] ip_local_deliver_finish at ffffffff8371e5e5
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/ipv4/ip_input.c: 238
#16 [ffffbe9080687b70] ip_local_deliver at ffffffff8371e65f
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/ipv4/ip_input.c: 258
#17 [ffffbe9080687bc8] ip_rcv_finish at ffffffff8371dd94
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/ipv4/ip_input.c: 415
#18 [ffffbe9080687bf0] ip_rcv at ffffffff8371e736
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/ipv4/ip_input.c: 526
#19 [ffffbe9080687c50] __netif_receive_skb_one_core at ffffffff836bf197
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/core/dev.c: 4989
#20 [ffffbe9080687c80] __netif_receive_skb at ffffffff836bf1f8
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/core/dev.c: 5102
#21 [ffffbe9080687ca0] netif_receive_skb_internal at ffffffff836be3c5
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/core/dev.c: 5202
#22 [ffffbe9080687cd0] napi_gro_receive at ffffffff836c03b0
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/core/dev.c: 5681
#23 [ffffbe9080687cf8] bnx2x_rx_int at ffffffffc037889a [bnx2x]
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c: 1088
#24 [ffffbe9080687db0] bnx2x_poll at ffffffffc037a978 [bnx2x]
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c: 3228
#25 [ffffbe9080687de0] net_rx_action at ffffffff836bf9c0
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/net/core/dev.c: 6362
#26 [ffffbe9080687e60] __softirqentry_text_start at ffffffff83c000e4
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/kernel/softirq.c: 292
#27 [ffffbe9080687ec8] run_ksoftirqd at ffffffff82e9d4fb
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/arch/x86/include/asm/paravirt.h: 776
#28 [ffffbe9080687ed8] smpboot_thread_fn at ffffffff82ec05cc
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/kernel/smpboot.c: 164
#29 [ffffbe9080687f08] kthread at ffffffff82ebc521
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/kernel/kthread.c: 246
#30 [ffffbe9080687f50] ret_from_fork at ffffffff83a00215
/build/linux-hwe-zHO4ZF/linux-hwe-5.0.0/arch/x86/entry/entry_64.S: 358
crash>
[ 143.046091] divide error: 0000 [#1] SMP PTI
[ 143.046242] CPU: 0 PID: 9 Comm: ksoftirqd/0 Kdump: loaded Tainted: G OE 5.0.0-23-generic #24~18.04.1-Ubuntu
[ 143.046497] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
[ 143.046754] RIP: 0010:gtp5g_encap_recv+0x1af/0x7e0 [gtp5g]
[ 143.046880] Code: ce 31 ca c1 ce 07 29 f2 89 d6 31 d0 c1 c6 10 29 f0 89 c6 31 c1 c1 c6 04 29 f1 31 ca c1 c1 0e 29 ca 31 d0 c1 ca 08 29 d0 31 d2 <41> f7 74 24 24 89 d0 49 8b 54 24 38 48 8d 04 c2 4c 8b 28 4d 85 ed
[ 143.047265] RSP: 0018:ffffbe9080687990 EFLAGS: 00010246
[ 143.047311] RAX: 0000000054f51ee2 RBX: ffff9fadeec80200 RCX: 00000000ef536f6a
[ 143.047346] RDX: 0000000000000000 RSI: 000000007c163994 RDI: 0000000000000000
[ 143.047381] RBP: ffffbe9080687a28 R08: 0000000000000000 R09: 0000000000000010
[ 143.047415] R10: ffff9fade135e940 R11: 0000000000000024 R12: ffff9fade2ce7940
[ 143.047450] R13: 0000000000000008 R14: ffff9fade135e924 R15: 00000000ea030000
[ 143.047485] FS: 0000000000000000(0000) GS:ffff9faebba00000(0000) knlGS:0000000000000000
[ 143.047524] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 143.047553] CR2: 000055a55fdfb110 CR3: 000000006b044000 CR4: 00000000000006f0
[ 143.047650] Call Trace:
[ 143.047672] ? gtp5g_genl_add_far+0x2a0/0x2a0 [gtp5g]
[ 143.047700] udp_queue_rcv_one_skb+0x1d4/0x450
[ 143.047724] udp_queue_rcv_skb+0x3f/0x1a0
[ 143.047746] udp_unicast_rcv_skb+0x77/0x90
[ 143.047767] __udp4_lib_rcv+0x4fa/0xba0
[ 143.047790] ? get_page_from_freelist+0xc7f/0x1560
[ 143.047815] udp_rcv+0x1a/0x20
[ 143.047835] ip_protocol_deliver_rcu+0x25/0x1b0
[ 143.047859] ip_local_deliver_finish+0x45/0x50
[ 143.047884] ip_local_deliver+0x6f/0xf0
[ 143.047906] ? ip_rcv_finish_core.isra.19+0x72/0x390
[ 143.047932] ip_rcv_finish+0x84/0xa0
[ 143.047952] ip_rcv+0x56/0xd0
[ 143.047972] __netif_receive_skb_one_core+0x57/0x80
[ 143.047998] __netif_receive_skb+0x18/0x60
[ 143.049060] netif_receive_skb_internal+0x45/0xe0
[ 143.049817] napi_gro_receive+0x120/0x150
[ 143.050589] bnx2x_rx_int+0x89a/0x1820 [bnx2x]
[ 143.051338] ? try_to_wake_up+0x59/0x4c0
[ 143.052078] ? entry_SYSCALL_64+0x36/0x38
[ 143.052786] ? inc_ucount+0x3e/0x210
[ 143.053478] bnx2x_poll+0x1c8/0x260 [bnx2x]
[ 143.054138] net_rx_action+0x140/0x3a0
[ 143.054800] __do_softirq+0xe4/0x2f3
[ 143.055429] run_ksoftirqd+0x2b/0x40
[ 143.056028] smpboot_thread_fn+0xfc/0x170
[ 143.056610] kthread+0x121/0x140
[ 143.057178] ? sort_range+0x30/0x30
[ 143.057747] ? kthread_park+0x90/0x90
[ 143.058297] ret_from_fork+0x35/0x40
[ 143.058843] Modules linked in: gtp5g(OE) udp_tunnel vmw_vsock_vmci_transport vsock vmwgfx joydev input_leds serio_raw ttm vmw_balloon drm_kms_helper drm fb_sys_fops syscopyarea sysfillrect sysimgblt vmw_vmci mac_hid binfmt_misc sch_fq_codel ip_tables x_tables autofs4 bnx2x mdio ahci psmouse libcrc32c vmxnet3 libahci vmw_pvscsi i2c_piix4 pata_acpi hid_generic usbhid hid
Please, check with the latest source, and closing this issue.