QEMU crash when execute device_add and device_del alternately #13
Description
When execute device_add and device_del alternately, qemu crashes:
[root@localhost coredump]# virsh qemu-monitor-command testvm --hmp 'device_add vfio-user-pci,socket=/var/run/cntrl,id=testdisk0'
[root@localhost coredump]# virsh qemu-monitor-command testvm --hmp 'device_del testdisk0'
[root@localhost coredump]# virsh qemu-monitor-command testvm --hmp 'device_add vfio-user-pci,socket=/var/run/cntrl,id=testdisk0'
[root@localhost coredump]# virsh qemu-monitor-command testvm --hmp 'device_del testdisk0'
[root@localhost coredump]# virsh qemu-monitor-command testvm --hmp 'device_add vfio-user-pci,socket=/var/run/cntrl,id=testdisk0'
error: Unable to read from monitor: Connection reset by peer
### And the coredump stack as follows:
Thread 2 (Thread 0x7f587b15df40 (LWP 2700247)):
#0 0x00007f587c57edd2 in futex_abstimed_wait_cancelable (private=, abstime=0x7ffdf875b130, expected=0, futex_word=0x559f5d6da2f0) at ../sysdeps/unix/sysv/linux/futex-internal.h:205
spdk/spdk#1 __pthread_cond_wait_common (abstime=0x7ffdf875b130, mutex=0x559f5d006f20, cond=0x559f5d6da2c8) at pthread_cond_wait.c:539
spdk/spdk#2 __pthread_cond_timedwait (cond=cond@entry=0x559f5d6da2c8, mutex=mutex@entry=0x559f5d006f20, abstime=abstime@entry=0x7ffdf875b130) at pthread_cond_wait.c:667
spdk/spdk#3 0x0000559f58d86e41 in qemu_cond_timedwait_impl (cond=0x559f5d6da2c8, mutex=0x559f5d006f20, ms=1000, file=0x559f58e3e3d0 "/root/qemu-5.0/builddir/build/BUILD/qemu-5.0.0.4/hw/vfio/user.c", line=721) at util/qemu-thread-posix.c:188
spdk/spdk#4 0x0000559f58a781b7 in vfio_user_send_wait (proxy=0x559f5d006ea0, hdr=0x559f5bf69c50, fds=, rsize=, nobql=) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/hw/vfio/user.c:721
spdk/spdk#5 0x0000559f58a785f8 in vfio_user_set_irqs (irq=0x7ffdf875b250, proxy=0x559f5d006ea0) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/hw/vfio/user.c:1380
spdk/spdk#6 vfio_user_io_set_irqs (vbasedev=, irqs=0x7ffdf875b250) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/hw/vfio/user.c:1616
spdk/spdk#7 0x0000559f58a65c2f in vfio_unmask_single_irqindex (vbasedev=vbasedev@entry=0x559f5c0822f0, index=index@entry=0) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/hw/vfio/common.c:89
spdk/spdk#8 0x0000559f58a6b676 in vfio_intx_disable_kvm (vdev=vdev@entry=0x559f5c081a00) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/hw/vfio/pci.c:225
spdk/spdk#9 0x0000559f58a6bcea in vfio_intx_disable (vdev=0x559f5c081a00) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/hw/vfio/pci.c:339
spdk/spdk#10 vfio_disable_interrupts (vdev=vdev@entry=0x559f5c081a00) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/hw/vfio/pci.c:1249
spdk/spdk#11 0x0000559f58a6f519 in vfio_pci_pre_reset (vdev=vdev@entry=0x559f5c081a00) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/hw/vfio/pci.c:2161
spdk/spdk#12 0x0000559f58a7000b in vfio_user_pci_reset (dev=) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/hw/vfio/pci.c:3752
spdk/spdk#13 0x0000559f58b63b10 in resettable_phase_hold (obj=obj@entry=0x559f5c081a00, opaque=opaque@entry=0x0, type=type@entry=RESET_TYPE_COLD) at hw/core/resettable.c:182
spdk/spdk#14 0x0000559f58b64160 in resettable_assert_reset (obj=obj@entry=0x559f5c081a00, type=type@entry=RESET_TYPE_COLD) at hw/core/resettable.c:60
spdk/spdk#15 0x0000559f58b5fb1d in device_set_realized (obj=, value=, errp=0x7ffdf875b548) at hw/core/qdev.c:935
spdk/spdk#16 0x0000559f58ca18d7 in property_set_bool (obj=0x559f5c081a00, v=, name=, opaque=0x559f5ba6ed70, errp=0x7ffdf875b548) at qom/object.c:2238
spdk/spdk#17 0x0000559f58ca651f in object_property_set_qobject (obj=obj@entry=0x559f5c081a00, value=value@entry=0x559f5c991980, name=name@entry=0x559f58e75518 "realized", errp=errp@entry=0x7ffdf875b548) at qom/qom-qobject.c:26
spdk/spdk#18 0x0000559f58ca3cb5 in object_property_set_bool (obj=0x559f5c081a00, value=, name=0x559f58e75518 "realized", errp=0x7ffdf875b548) at qom/object.c:1390
spdk/spdk#19 0x0000559f58b203d6 in qdev_device_add (opts=opts@entry=0x559f5c7cb3b0, errp=errp@entry=0x7ffdf875b620) at qdev-monitor.c:680
spdk/spdk#20 0x0000559f58b20753 in qmp_device_add (qdict=, ret_data=ret_data@entry=0x0, errp=errp@entry=0x7ffdf875b650) at qdev-monitor.c:805
spdk/spdk#21 0x0000559f58b20a2d in hmp_device_add (mon=0x7ffdf875b6e0, qdict=) at qdev-monitor.c:905
spdk/spdk#22 0x0000559f58c472a8 in handle_hmp_command (mon=mon@entry=0x7ffdf875b6e0, cmdline=, cmdline@entry=0x559f5bbe5600 "device_add vfio-user-pci,socket=/var/run/vfiouser-disk/vmuuid_test-d810e767-5426-41f5-8229-bdcb0a43a840/cntrl,id=testdisk0") at monitor/hmp.c:1082
--Type for more, q to quit, c to continue without paging--
spdk/spdk#23 0x0000559f58aadf92 in qmp_human_monitor_command (command_line=0x559f5bbe5600 "device_add vfio-user-pci,socket=/var/run/vfiouser-disk/vmuuid_test-d810e767-5426-41f5-8229-bdcb0a43a840/cntrl,id=testdisk0", has_cpu_index=, cpu_index=0, errp=errp@entry=0x7ffdf875b7f8) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/monitor/misc.c:142
spdk/spdk#24 0x0000559f58c6ccb9 in qmp_marshal_human_monitor_command (args=, ret=0x7ffdf875b890, errp=0x7ffdf875b888) at qapi/qapi-commands-misc.c:783
spdk/spdk#25 0x0000559f58d37a70 in qmp_dispatch (cmds=0x559f59421aa0 <qmp_commands>, request=, allow_oob=) at qapi/qmp-dispatch.c:155
spdk/spdk#26 0x0000559f58c442c1 in monitor_qmp_dispatch (mon=0x559f5badf8c0, req=) at monitor/qmp.c:145
spdk/spdk#27 0x0000559f58c44aa0 in monitor_qmp_bh_dispatcher (data=) at monitor/qmp.c:234
spdk/spdk#28 0x0000559f58d80027 in aio_bh_call (bh=0x559f5ba428a0) at util/async.c:136
spdk/spdk#29 aio_bh_poll (ctx=ctx@entry=0x559f5bade000) at util/async.c:164
spdk/spdk#30 0x0000559f58d8372e in aio_dispatch (ctx=0x559f5bade000) at util/aio-posix.c:380
spdk/spdk#31 0x0000559f58d7ff0e in aio_ctx_dispatch (source=, callback=, user_data=) at util/async.c:306
spdk/spdk#32 0x00007f587d2f6184 in g_main_dispatch (context=0x559f5bae5b80) at ../glib/gmain.c:3325
spdk/spdk#33 g_main_context_dispatch (context=context@entry=0x559f5bae5b80) at ../glib/gmain.c:4043
spdk/spdk#34 0x0000559f58d8296a in glib_pollfds_poll () at util/main-loop.c:219
spdk/spdk#35 os_host_main_loop_wait (timeout=1000000000) at util/main-loop.c:242
spdk/spdk#36 main_loop_wait (nonblocking=nonblocking@entry=0) at util/main-loop.c:518
spdk/spdk#37 0x0000559f58ab4a61 in qemu_main_loop () at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/softmmu/vl.c:1710
spdk/spdk#38 0x0000559f589bc9be in main (argc=, argv=, envp=) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/softmmu/main.c:49
Thread 1 (Thread 0x7f587b15a700 (LWP 2700251)):
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
spdk/spdk#1 0x00007f587c3f2921 in __GI_abort () at abort.c:79
spdk/spdk#2 0x00007f587cc37f29 in tcmalloc::Log(tcmalloc::LogMode, char const*, int, tcmalloc::LogItem, tcmalloc::LogItem, tcmalloc::LogItem, tcmalloc::LogItem) () at /usr/lib64/libtcmalloc.so.4
spdk/spdk#3 0x00007f587cc2bf29 in () at /usr/lib64/libtcmalloc.so.4
spdk/spdk#4 0x00007f587d2fbfa9 in g_free (mem=0x559f5940eae8 <vfio_group_list>) at ../glib/gmem.c:199
spdk/spdk#5 0x0000559f58ca262c in object_property_free (data=0x559f5d5f8b18) at qom/object.c:278
spdk/spdk#6 0x00007f587d2e29bb in g_hash_table_remove_all_nodes (hash_table=hash_table@entry=0x559f5c176520, notify=notify@entry=1, destruction=destruction@entry=1) at ../glib/ghash.c:708
spdk/spdk#7 0x00007f587d2e3e1a in g_hash_table_remove_all_nodes (destruction=1, notify=1, hash_table=0x559f5c176520) at ../glib/ghash.c:1459
spdk/spdk#8 g_hash_table_unref (hash_table=0x559f5c176520) at ../glib/ghash.c:1463
spdk/spdk#9 0x0000559f58ca3069 in object_property_del_all (obj=0x559f5c94c800) at qom/object.c:614
spdk/spdk#10 object_finalize (data=0x559f5c94c800) at qom/object.c:667
spdk/spdk#11 object_unref (obj=obj@entry=0x559f5c94c800) at qom/object.c:1128
spdk/spdk#12 0x0000559f589c360b in phys_section_destroy (mr=0x559f5c94c800) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/exec.c:1497
spdk/spdk#13 phys_sections_free (map=0x559f5d5f9510) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/exec.c:1506
spdk/spdk#14 address_space_dispatch_free (d=0x559f5d5f9500) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/exec.c:2971
spdk/spdk#15 0x0000559f58a0ed69 in flatview_destroy (view=0x559f5ce75e40) at /usr/src/debug/qemu-kvm-5.0.0.4-1.2.ctl2.x86_64/memory.c:285
spdk/spdk#16 0x0000559f58d9910c in call_rcu_thread (opaque=) at util/rcu.c:283
spdk/spdk#17 0x0000559f58d86654 in qemu_thread_start (args=0x559f5ba8f020) at util/qemu-thread-posix.c:519
spdk/spdk#18 0x00007f587c578f2b in start_thread (arg=0x7f587b15a700) at pthread_create.c:486
spdk/spdk#19 0x00007f587c4b070f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
oracle qemu branch: vfio-user-dbfix
spdk branch: V22.01
Has anyone ever encountered a similar issue? thanks🙂