idle loop is burning CPU
mvwieringen opened this issue · comments
What I can see from a test instance running under proxmox for Raspberry Pi 4 is that a arm instance of Illumos is burning all of the 1 cpu's that it gets assigned. (For now it doesn't allow me to assign more cpus as the boot code doesn't seem to be able to start the rest of the cpus and the system then panics.) So for now it runs with 1 CPU core and 2 Gb of RAM under KVM on a RPI 8 Gb. It runs must faster then when I try running the same image under qemu on a 2.3 Ghz E5-2697v4 with lots of RAM.
> ::cpuinfo
ID ADDR FLG NRUN BSPL PRI RNRN KRNRN SWITCH THREAD PROC
0 fffffffffe11b6a0 1b 0 0 59 no no t-5 ffff80019fd70040 mdb
> ::stacks
THREAD STATE SOBJ COUNT
ffff800044011c20 SLEEP CV 287
cv_wait+0x74
taskq_thread_wait+0xbc
taskq_thread+0x29c
thread_exit
ffff8000453c7c20 SLEEP CV 218
cv_timedwait_hires+0xbc
cv_reltimedwait+0x68
taskq_thread_wait+0x48
taskq_d_thread+0x104
thread_exit
ffff8001966833e0 SLEEP CV 47
cv_wait_sig_swap_core+0x168
cv_wait_sig_swap+0x10
cv_waituntil_sig+0xa0
lwp_park+0xf8
syslwp_park+0x7c
svc_handler+0x88
ffff800195844020 SLEEP SHUTTLE 29
shuttle_swtch+0x170
door_return+0x2fc
doorfs+0x180
svc_handler+0x88
ffff800194d90000 SLEEP SHUTTLE 26
shuttle_resume+0x210
door_return+0x2f0
doorfs+0x180
svc_handler+0x88
ffff8001957ca460 SLEEP CV 20
cv_timedwait_sig_hires+0x2a0
cv_waituntil_sig+0x124
nanosleep+0xfc
svc_handler+0x88
ffff800195844bc0 SLEEP CV 15
cv_wait_sig_swap_core+0x168
cv_wait_sig_swap+0x10
cv_timedwait_sig_hrtime+0x28
poll_common+0x460
pollsys+0xb4
svc_handler+0x88
ffff800044f30c20 SLEEP CV 14
cv_wait+0x74
evch_delivery_hold+0xd4
evch_delivery_thr+0x27c
thread_exit
ffff800196ade080 SLEEP CV 11
cv_wait_sig_swap_core+0x168
cv_wait_sig_swap+0x10
pause+0x28
ffff800044080c20 FREE <NONE> 9
<no consistent stack found>
ffff800194d8fc20 SLEEP CV 9
cv_timedwait_sig_hires+0x2a0
cv_waituntil_sig+0x124
lwp_park+0xf8
syslwp_park+0x7c
svc_handler+0x88
ffff800196e2cc00 SLEEP CV 6
cv_wait_sig_swap_core+0x168
cv_wait_sig_swap+0x10
sigsuspend+0xc0
svc_handler+0x88
ffff800196505080 SLEEP CV 5
cv_timedwait_sig_hires+0x2a0
cv_timedwait_sig_hrtime+0x1c
poll_common+0x460
pollsys+0xb4
svc_handler+0x88
ffff8001944e0040 SLEEP CV 5
cv_wait_sig_swap_core+0x168
cv_wait_sig_swap+0x10
cv_waituntil_sig+0xa0
sigtimedwait+0x118
svc_handler+0x88
ffff800045391c20 SLEEP CV 3
cv_wait+0x74
mac_soft_ring_worker+0xb0
thread_exit
ffff8000442cfc20 SLEEP CV 3
cv_wait+0x74
zthr_procedure+0xd4
thread_exit
ffff8001901447e0 SLEEP CV 3
cv_wait_sig+0x168
door_unref+0x118
doorfs+0x1b4
svc_handler+0x88
ffff80019b09f020 SLEEP CV 3
cv_wait_sig_swap_core+0x168
cv_wait_sig_swap+0x10
waitid+0x44c
waitsys+0x2c
svc_handler+0x88
ffff800045435c20 SLEEP CV 2
cv_wait+0x74
mac_srs_worker+0x160
thread_exit
ffff800045446c20 SLEEP CV 2
cv_wait+0x74
squeue_polling_thread+0xec
thread_exit
ffff8000453bac20 SLEEP CV 2
cv_wait+0x74
squeue_worker+0x110
thread_exit
ffff8001944e0800 SLEEP CV 2
cv_wait_sig+0x168
cte_get_event+0x88
ctfs_endpoint_ioctl+0x110
ctfs_bu_ioctl+0x30
fop_ioctl+0x60
ioctl+0x1b4
svc_handler+0x88
ffff8000440d1c20 FREE <NONE> 1
cpus
ffff80004455dc20 SLEEP CV 1
cv_timedwait_hires+0xbc
cv_reltimedwait+0x68
idm_wd_thread+0x1bc
thread_exit
ffff800045cc3c20 SLEEP CV 1
cv_timedwait_hires+0xbc
cv_reltimedwait+0x68
kcfpool_svc+0xe0
thread_exit
ffff8000442c9c20 SLEEP CV 1
cv_timedwait_hires+0xbc
cv_reltimedwait+0x68
kcfpoold+0x100
thread_exit
ffff800044f54c20 SLEEP CV 1
cv_timedwait_hires+0xbc
cv_reltimedwait+0x68
mod_uninstall_daemon+0x17c
thread_exit
ffff8000442c3c20 SLEEP CV 1
cv_timedwait_hires+0xbc
cv_reltimedwait+0x68
page_capture_thread+0xac
thread_exit
ffff800044f5ac20 SLEEP CV 1
cv_timedwait_hires+0xbc
cv_reltimedwait+0x68
seg_pasync_thread+0xdc
thread_exit
ffff8000445bec20 SLEEP CV 1
cv_timedwait_hires+0xbc
cv_reltimedwait+0x68
stmf_svc_timeout+0x324
stmf_svc+0x78
taskq_thread+0x2e8
thread_exit
ffff8000444c1c20 SLEEP CV 1
cv_timedwait_hires+0xbc
cv_timedwait+0x4c
dce_reclaim_worker+0x150
thread_exit
ffff8000442e7c20 SLEEP CV 1
cv_timedwait_hires+0xbc
cv_timedwait+0x4c
l2arc_feed_thread+0x114
thread_exit
ffff800044f00c20 SLEEP CV 1
cv_timedwait_hires+0xbc
cv_timedwait+0x4c
txg_thread_wait+0x54
txg_sync_thread+0x74
thread_exit
ffff800045418c20 SLEEP CV 1
cv_timedwait_hires+0xbc
cv_timedwait+0x4c
zone_status_timedwait+0x64
auto_do_unmount+0x144
thread_exit
ffff800044f06c20 SLEEP CV 1
cv_timedwait_hires+0xbc
cv_timedwait_sig_hires+0x194
cv_timedwait_sig_hrtime+0x1c
mmp_thread+0x210
thread_exit
ffff8000442e1c20 SLEEP CV 1
cv_timedwait_hires+0xbc
dbuf_evict_thread+0x16c
thread_exit
ffff8000442d5c20 SLEEP CV 1
cv_timedwait_hires+0xbc
zthr_procedure+0xa4
thread_exit
ffff80019652abc0 SLEEP CV 1
cv_timedwait_sig_hires+0x2a0
cv_reltimedwait_sig+0x6c
bd_check_state+0x9c
bd_ioctl+0x190
cdev_ioctl+0x28
spec_ioctl+0x4c
fop_ioctl+0x60
ioctl+0x1b4
svc_handler+0x88
ffff800044ed3c20 SLEEP CV 1
cv_wait+0x74
crypto_bufcall_service+0xc4
thread_exit
ffff800044077c20 SLEEP CV 1
cv_wait+0x74
dld_taskq_dispatch+0x118
thread_exit
ffff800044f4ec20 SLEEP CV 1
cv_wait+0x74
fsflush+0x4c8
thread_exit
ffff800044491c20 SLEEP CV 1
cv_wait+0x74
i_mac_notify_thread+0xa8
thread_exit
ffff8000446c8c20 SLEEP CV 1
cv_wait+0x74
ibcm_process_tlist+0x238
thread_exit
ffff8000444c7c20 SLEEP CV 1
cv_wait+0x74
ill_taskq_dispatch+0x124
thread_exit
ffff8000444afc20 SLEEP CV 1
cv_wait+0x74
ipsec_loader+0xac
thread_exit
ffff8000453a9c20 SLEEP CV 1
cv_wait+0x74
log_event_deliver+0x2ac
thread_exit
ffff800044f48c20 SLEEP CV 1
cv_wait+0x74
pageout+0x1f8
thread_exit
ffff800044f60c20 SLEEP CV 1
cv_wait+0x74
pageout_scanner+0x174
thread_exit
ffff8000442edc20 SLEEP CV 1
cv_wait+0x74
pm_dep_thread+0xa4
thread_exit
ffff800044f1ec20 SLEEP CV 1
cv_wait+0x74
progressbar_key_abort_thread+0x70
thread_exit
ffff800044023c20 SLEEP CV 1
cv_wait+0x74
scsi_hba_barrier_daemon+0x98
thread_exit
ffff800044029c20 SLEEP CV 1
cv_wait+0x74
scsi_lunchg1_daemon+0xac
thread_exit
ffff80004402fc20 SLEEP CV 1
cv_wait+0x74
scsi_lunchg2_daemon+0xb4
thread_exit
ffff800044497c20 SLEEP CV 1
cv_wait+0x74
softmac_taskq_dispatch+0x114
thread_exit
ffff800044784c20 SLEEP CV 1
cv_wait+0x74
spa_thread+0x144
thread_exit
ffff8000442b1c20 SLEEP CV 1
cv_wait+0x74
streams_bufcall_service+0x1bc
thread_exit
ffff8000442b7c20 SLEEP CV 1
cv_wait+0x74
streams_qbkgrnd_service+0x1fc
thread_exit
ffff8000442bdc20 SLEEP CV 1
cv_wait+0x74
streams_sqbkgrnd_service+0x104
thread_exit
ffff800044f2ac20 SLEEP CV 1
cv_wait+0x74
task_commit+0xfc
thread_exit
ffff80004400bc20 SLEEP CV 1
cv_wait+0x74
thread_reaper+0x150
thread_exit
ffff800044ef7c20 SLEEP CV 1
cv_wait+0x74
txg_thread_wait+0xd0
txg_quiesce_thread+0xf8
thread_exit
ffff800045284c20 SLEEP CV 1
cv_wait+0x74
ufs_thread_idle+0x1e0
thread_exit
ffff8000450b5c20 SLEEP CV 1
cv_wait+0x74
ufs_thread_run+0x8c
ufs_thread_hlock+0x11c
thread_exit
ffff80019653fbe0 SLEEP CV 1
cv_wait_sig+0x168
so_dequeue_msg+0x7cc
so_recvmsg+0x2b4
socket_recvmsg+0x2c
socket_vop_read+0x5c
fop_read+0x58
read+0x1ec
svc_handler+0x88
ffff800194d8f460 SLEEP CV 1
cv_wait_sig_swap_core+0x168
cv_wait_sig_swap+0x10
cv_waituntil_sig+0xa0
port_getn+0x394
portfs+0x284
svc_handler+0x88
ffff80019b09f400 SLEEP CV 1
cv_wait_sig_swap_core+0x168
cv_wait_sig_swap+0x10
fifo_read+0x104
fop_read+0x58
read+0x1ec
svc_handler+0x88
ffff800194d907c0 SLEEP SHUTTLE 1
shuttle_resume+0x210
door_call+0x228
doorfs+0xfc
svc_handler+0x88
ffff800044005c20 ONPROC <NONE> 1
idle+0x44
thread_exit
ffff80019fd70040 ONPROC <NONE> 1
mmio+0xb0
mmrw+0x244
mmread+0x14
cdev_read+0x28
spec_read+0x160
fop_read+0x58
pread+0x1f0
svc_handler+0x88
fffffffffe10c000 STOPPED <NONE> 1
sched+0x990
main+0x444
_start+0x2c
ffff800044f3cc20 STOPPED <NONE> 1
<no consistent stack found>
As seen from PiOS.
Tasks: 163 total, 1 running, 160 sleeping, 0 stopped, 2 zombie
%Cpu(s): 29.7 us, 4.7 sy, 0.0 ni, 65.1 id, 0.2 wa, 0.0 hi, 0.2 si, 0.0 st
MiB Mem : 7811.2 total, 71.7 free, 4469.2 used, 3270.4 buff/cache
MiB Swap: 1024.0 total, 1023.0 free, 1.0 used. 3180.3 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
589252 root 20 0 3333832 1.2g 39160 S 99.7 15.9 8:04.77 kvm
Thanks this checks out. The code to idle the CPUs when idle turns out to have been disabled in the original port, and not re-enabled yet. Ouch.
Ok installed the new disk image of Omnios and indeed now things are much more usable and use around 8-10% when "Idle" which is indeed much better and not keeping the CPU busy. So from now on I will run the image 24x7 on my "production" Rpi4
with Proxmox for ARM which also runs my OpenHAB VM. I even setup LDAP and that mostly works including autofs for autohome and autodirect for some NFS mounted source trees I use between all my Illumos (Omios, OpenIndiana) VMs. I have one small problem with zsh as non root user probably due to ASLR on ARM. e.g.
Apr 23 11:18:14 braich genunix: [ID 533030 kern.notice] NOTICE: zsh[100951] attempt to execute code on stack by uid 1150
When switching to bash things work fine.