Uploaded image for project: 'RHEL'
  1. RHEL
  2. RHEL-94534

[virtiofs]qemu coredumped during migration while the virtiofsd daemon is down[rhel10.1]

Linking RHIVOS CVEs to...Migration: Automation ...SWIFT: POC ConversionSync from "Extern...XMLWordPrintable

    • qemu-kvm-10.1.0-1.el10
    • No
    • Moderate
    • 1
    • rhel-virt-storage
    • ssg_virtualization
    • 2
    • False
    • False
    • Hide

      None

      Show
      None
    • None
    • Patch Under review backlog
    • Pass
    • Automated
    • Unspecified
    • Unspecified
    • Unspecified
    • 10.1.0
    • None

      What were you trying to do that didn't work?

      Try to migrate a VM with virtiofs block device and kill the virtiofsd daemon before migration, the qemu-kvm coredumped.

      What is the impact of this issue to you?

      Please provide the package NVR for which the bug is seen:

      qemu-kvm-10.0.0-4.el10

      virtiofsd-1.13.0-1.el10.x86_64

      kernel-6.12.0-83.el10.

      How reproducible is this bug?:

      100%

      Steps to reproduce

      1.Mount the nfs shared dir to both source and target nodes for the migration VM

       

      (source node)#mount -t nfs 10.72.140.12:/home/timao/nfs_share /home/timao/virtiofsd/share
      (target node)#mount -t nfs 10.72.140.12:/home/timao/nfs_share /home/timao/virtiofsd/repoduce/share
      

       

      2.Start virtiofsd daemon on the source node and start the VM

       

      #/usr/libexec/virtiofsd --shared-dir /home/timao/virtiofsd/share --socket-path /home/timao/virtiofsd/virtiofsd.sock &
      #/usr/libexec/qemu-kvm \
      -S  \
      -name 'avocado-vt-vm1'  \
      -sandbox on,elevateprivileges=deny,obsolete=deny,resourcecontrol=deny \
      -blockdev '{"node-name": "file_ovmf_code", "driver": "file", "filename": "/usr/share/OVMF/OVMF_CODE.secboot.fd", "auto-read-only": true, "discard": "unmap"}' \
      -blockdev '{"node-name": "drive_ovmf_code", "driver": "raw", "read-only": true, "file": "file_ovmf_code"}' \
      -blockdev '{"node-name": "file_ovmf_vars", "driver": "file", "filename": "/root/avocado/data/avocado-vt/avocado-vt-vm1_rhel960-64-virtio-scsi-ovmf_qcow2_filesystem_VARS.raw", "auto-read-only": true, "discard": "unmap"}' \
      -blockdev '{"node-name": "drive_ovmf_vars", "driver": "raw", "read-only": false, "file": "file_ovmf_vars"}' \
      -machine q35,pflash0=drive_ovmf_code,pflash1=drive_ovmf_vars \
      -device '{"id": "pcie-root-port-0", "driver": "pcie-root-port", "multifunction": true, "bus": "pcie.0", "addr": "0x1", "chassis": 1}' \
      -device '{"id": "pcie-pci-bridge-0", "driver": "pcie-pci-bridge", "addr": "0x0", "bus": "pcie-root-port-0"}'  \
      -nodefaults \
      -device '{"driver": "VGA", "bus": "pcie.0", "addr": "0x2"}' \
      -m 8192 \
      -object '{"size": 8589934592, "share": true, "id": "mem-mem1", "qom-type": "memory-backend-memfd"}'  \
      -smp 48,maxcpus=48,cores=24,threads=1,dies=1,sockets=2  \
      -numa node,memdev=mem-mem1,nodeid=0  \
      -cpu 'Icelake-Server-v2',+kvm_pv_unhalt \
      -chardev socket,path=/var/tmp/avocado_z_kvxqan/monitor-qmpmonitor1-20250529-045340-AFg74M9u,server=on,id=qmp_id_qmpmonitor1,wait=off  \
      -mon chardev=qmp_id_qmpmonitor1,mode=control \
      -chardev socket,path=/var/tmp/avocado_z_kvxqan/monitor-catch_monitor-20250529-045340-AFg74M9u,server=on,id=qmp_id_catch_monitor,wait=off  \
      -mon chardev=qmp_id_catch_monitor,mode=control \
      -device '{"ioport": 1285, "driver": "pvpanic", "id": "idznA8Ae"}' \
      -chardev socket,path=/var/tmp/avocado_z_kvxqan/serial-serial0-20250529-045340-AFg74M9u,server=on,id=chardev_serial0,wait=off \
      -device '{"id": "serial0", "driver": "isa-serial", "chardev": "chardev_serial0"}'  \
      -chardev socket,id=seabioslog_id_20250529-045340-AFg74M9u,path=/var/tmp/avocado_z_kvxqan/seabios-20250529-045340-AFg74M9u,server=on,wait=off \
      -device isa-debugcon,chardev=seabioslog_id_20250529-045340-AFg74M9u,iobase=0x402 \
      -device '{"id": "pcie-root-port-1", "port": 1, "driver": "pcie-root-port", "addr": "0x1.0x1", "bus": "pcie.0", "chassis": 2}' \
      -device '{"driver": "qemu-xhci", "id": "usb1", "bus": "pcie-root-port-1", "addr": "0x0"}' \
      -device '{"driver": "usb-tablet", "id": "usb-tablet1", "bus": "usb1.0", "port": "1"}' \
      -device '{"id": "pcie-root-port-2", "port": 2, "driver": "pcie-root-port", "addr": "0x1.0x2", "bus": "pcie.0", "chassis": 3}' \
      -device '{"id": "virtio_scsi_pci0", "driver": "virtio-scsi-pci", "bus": "pcie-root-port-2", "addr": "0x0"}' \
      -blockdev '{"node-name": "file_image1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/timao/virtiofsd/share/rhel960-64-virtio-scsi-ovmf.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_image1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_image1"}' \
      -device '{"driver": "scsi-hd", "id": "image1", "drive": "drive_image1", "write-cache": "on"}' \
      -chardev socket,id=char_virtiofs_fs,path=/home/timao/virtiofsd/virtiofsd.sock \
      -device '{"id": "pcie-root-port-3", "port": 3, "driver": "pcie-root-port", "addr": "0x1.0x3", "bus": "pcie.0", "chassis": 4}' \
      -device '{"id": "vufs_virtiofs_fs", "chardev": "char_virtiofs_fs", "tag": "myfs", "queue-size": 1024, "driver": "vhost-user-fs-pci", "bus": "pcie-root-port-3", "addr": "0x0"}' \
      -device '{"id": "pcie-root-port-4", "port": 4, "driver": "pcie-root-port", "addr": "0x1.0x4", "bus": "pcie.0", "chassis": 5}' \
      -device '{"driver": "virtio-net-pci", "mac": "9a:93:d9:de:2e:86", "id": "id1hBnNv", "netdev": "id5s0vaW", "bus": "pcie-root-port-4", "addr": "0x0"}' \
      -netdev  '{"id": "id5s0vaW", "type": "tap", "vhost": true}'  \
      -vnc :0  \
      -rtc base=utc,clock=host,driftfix=slew  \
      -boot menu=off,order=cdn,once=c,strict=off \
      -enable-kvm \
      -monitor stdio
      

      3.In guest mount the virtiofs and do fio

       

      (guest)# mount -t virtiofs myfs /mnt/
      (guest)# fio --name=fio --filename=/mnt/test --ioengine=libaio --rw=rw --bs=1k --size=200M --numjobs=4 --runtime=1800 --time_based
      

       

      4.In target node, start the virtiofsd daemon and start the vm

       

      # /usr/libexec/virtiofsd --shared-dir /home/timao/virtiofsd/repoduce/share --socket-path /home/timao/virtiofsd/tgtvirtiofsd.sock &
      #/usr/libexec/qemu-kvm \
      -S  \
      -name 'avocado-vt-vm1'  \
      -sandbox on,elevateprivileges=deny,obsolete=deny,resourcecontrol=deny \
      -blockdev '{"node-name": "file_ovmf_code", "driver": "file", "filename": "/usr/share/OVMF/OVMF_CODE.secboot.fd", "auto-read-only": true, "discard": "unmap"}' \
      -blockdev '{"node-name": "drive_ovmf_code", "driver": "raw", "read-only": true, "file": "file_ovmf_code"}' \
      -blockdev '{"node-name": "file_ovmf_vars", "driver": "file", "filename": "/root/avocado/data/avocado-vt/avocado-vt-vm1_rhel960-64-virtio-scsi-ovmf_qcow2_filesystem_VARS.raw", "auto-read-only": true, "discard": "unmap"}' \
      -blockdev '{"node-name": "drive_ovmf_vars", "driver": "raw", "read-only": false, "file": "file_ovmf_vars"}' \
      -machine q35,pflash0=drive_ovmf_code,pflash1=drive_ovmf_vars \
      -device '{"id": "pcie-root-port-0", "driver": "pcie-root-port", "multifunction": true, "bus": "pcie.0", "addr": "0x1", "chassis": 1}' \
      -device '{"id": "pcie-pci-bridge-0", "driver": "pcie-pci-bridge", "addr": "0x0", "bus": "pcie-root-port-0"}'  \
      -nodefaults \
      -device '{"driver": "VGA", "bus": "pcie.0", "addr": "0x2"}' \
      -m 8192 \
      -object '{"size": 8589934592, "share": true, "id": "mem-mem1", "qom-type": "memory-backend-memfd"}'  \
      -smp 48,maxcpus=48,cores=24,threads=1,dies=1,sockets=2  \
      -numa node,memdev=mem-mem1,nodeid=0  \
      -cpu 'Icelake-Server-v2',+kvm_pv_unhalt \
      -chardev socket,path=/var/tmp/avocado_z_kvxqan/monitor-qmpmonitor1-20250529-045340-AFg74M9u,server=on,id=qmp_id_qmpmonitor1,wait=off  \
      -mon chardev=qmp_id_qmpmonitor1,mode=control \
      -chardev socket,path=/var/tmp/avocado_z_kvxqan/monitor-catch_monitor-20250529-045340-AFg74M9u,server=on,id=qmp_id_catch_monitor,wait=off  \
      -mon chardev=qmp_id_catch_monitor,mode=control \
      -device '{"ioport": 1285, "driver": "pvpanic", "id": "idznA8Ae"}' \
      -chardev socket,path=/var/tmp/avocado_z_kvxqan/serial-serial0-20250529-045340-AFg74M9u,server=on,id=chardev_serial0,wait=off \
      -device '{"id": "serial0", "driver": "isa-serial", "chardev": "chardev_serial0"}'  \
      -chardev socket,id=seabioslog_id_20250529-045340-AFg74M9u,path=/var/tmp/avocado_z_kvxqan/seabios-20250529-045340-AFg74M9u,server=on,wait=off \
      -device isa-debugcon,chardev=seabioslog_id_20250529-045340-AFg74M9u,iobase=0x402 \
      -device '{"id": "pcie-root-port-1", "port": 1, "driver": "pcie-root-port", "addr": "0x1.0x1", "bus": "pcie.0", "chassis": 2}' \
      -device '{"driver": "qemu-xhci", "id": "usb1", "bus": "pcie-root-port-1", "addr": "0x0"}' \
      -device '{"driver": "usb-tablet", "id": "usb-tablet1", "bus": "usb1.0", "port": "1"}' \
      -device '{"id": "pcie-root-port-2", "port": 2, "driver": "pcie-root-port", "addr": "0x1.0x2", "bus": "pcie.0", "chassis": 3}' \
      -device '{"id": "virtio_scsi_pci0", "driver": "virtio-scsi-pci", "bus": "pcie-root-port-2", "addr": "0x0"}' \
      -blockdev '{"node-name": "file_image1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/timao/virtiofsd/repoduce/share/rhel960-64-virtio-scsi-ovmf.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_image1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_image1"}' \
      -device '{"driver": "scsi-hd", "id": "image1", "drive": "drive_image1", "write-cache": "on"}' \
      -chardev socket,id=char_virtiofs_fs,path=/home/timao/virtiofsd/tgtvirtiofsd.sock \
      -device '{"id": "pcie-root-port-3", "port": 3, "driver": "pcie-root-port", "addr": "0x1.0x3", "bus": "pcie.0", "chassis": 4}' \
      -device '{"id": "vufs_virtiofs_fs", "chardev": "char_virtiofs_fs", "tag": "myfs", "queue-size": 1024, "driver": "vhost-user-fs-pci", "bus": "pcie-root-port-3", "addr": "0x0"}' \
      -device '{"id": "pcie-root-port-4", "port": 4, "driver": "pcie-root-port", "addr": "0x1.0x4", "bus": "pcie.0", "chassis": 5}' \
      -device '{"driver": "virtio-net-pci", "mac": "9a:93:d9:de:2e:86", "id": "id1hBnNv", "netdev": "id5s0vaW", "bus": "pcie-root-port-4", "addr": "0x0"}' \
      -netdev  '{"id": "id5s0vaW", "type": "tap", "vhost": true}'  \
      -vnc :1  \
      -rtc base=utc,clock=host,driftfix=slew  \
      -boot menu=off,order=cdn,once=c,strict=off \
      -enable-kvm \
      -monitor stdio \
      -incoming defe
      

      5.In source node, kill the virtiofsd process

      #kill -9 166020 ----> the pid of the first virtiofsd daemon

      6.Start migration

       

      (target)(qemu) migrate_incoming tcp:[::]:3456
      (source)(qemu) migrate tcp:10.73.196.56:2345
      

       

      Expected results

      Migration fails with some hint information but no coredumped

      Actual results

      On source node:

      (qemu) migrate tcp:10.73.196.56:2345
      qemu-kvm: Failed to set msg fds.
      qemu-kvm: vhost_set_log_base failed: Invalid argument (22)
      qemu-kvm: Failed to set msg fds.
      qemu-kvm: vhost_set_features failed: Invalid argument (22)
      qemu.sh: line 46: 166028 Aborted                 (core dumped) /usr/libexec/qemu-kvm -S -name 'avocado-vt-vm1' -sandbox on,elevateprivileges=deny,obsolete=deny,resourcecontrol=deny -blockdev '{"node-name": "file_ovmf_code", "driver": "file", "filename": "/usr/share/OVMF/OVMF_CODE.secboot.fd", "auto-read-only": true, "discard": "unmap"}' -blockdev '{"node-name": "drive_ovmf_code", "driver": "raw", "read-only": true, "file": "file_ovmf_code"}' -blockdev '{"node-name": "file_ovmf_vars", "driver": "file", "filename": "/root/avocado/data/avocado-vt/avocado-vt-vm1_rhel960-64-virtio-scsi-ovmf_qcow2_filesystem_VARS.raw", "auto-read-only": true, "discard": "unmap"}' -blockdev '{"node-name": "drive_ovmf_vars", "driver": "raw", "read-only": false, "file": "file_ovmf_vars"}' -machine q35,pflash0=drive_ovmf_code,pflash1=drive_ovmf_vars -device '{"id": "pcie-root-port-0", "driver": "pcie-root-port", "multifunction": true, "bus": "pcie.0", "addr": "0x1", "chassis": 1}' -device '{"id": "pcie-pci-bridge-0", "driver": "pcie-pci-bridge", "addr": "0x0", "bus": "pcie-root-port-0"}' -nodefaults -device '{"driver": "VGA", "bus": "pcie.0", "addr": "0x2"}' -m 8192 -object '{"size": 8589934592, "share": true, "id": "mem-mem1", "qom-type": "memory-backend-memfd"}' -smp 48,maxcpus=48,cores=24,threads=1,dies=1,sockets=2 -numa node,memdev=mem-mem1,nodeid=0 -cpu 'Icelake-Server-v2',+kvm_pv_unhalt -chardev socket,path=/var/tmp/avocado_z_kvxqan/monitor-qmpmonitor1-20250529-045340-AFg74M9u,server=on,id=qmp_id_qmpmonitor1,wait=off -mon chardev=qmp_id_qmpmonitor1,mode=control -chardev socket,path=/var/tmp/avocado_z_kvxqan/monitor-catch_monitor-20250529-045340-AFg74M9u,server=on,id=qmp_id_catch_monitor,wait=off -mon chardev=qmp_id_catch_monitor,mode=control -device '{"ioport": 1285, "driver": "pvpanic", "id": "idznA8Ae"}' -chardev socket,path=/var/tmp/avocado_z_kvxqan/serial-serial0-20250529-045340-AFg74M9u,server=on,id=chardev_serial0,wait=off -device '{"id": "serial0", "driver": "isa-serial", "chardev": "chardev_serial0"}' -chardev socket,id=seabioslog_id_20250529-045340-AFg74M9u,path=/var/tmp/avocado_z_kvxqan/seabios-20250529-045340-AFg74M9u,server=on,wait=off -device isa-debugcon,chardev=seabioslog_id_20250529-045340-AFg74M9u,iobase=0x402 -device '{"id": "pcie-root-port-1", "port": 1, "driver": "pcie-root-port", "addr": "0x1.0x1", "bus": "pcie.0", "chassis": 2}' -device '{"driver": "qemu-xhci", "id": "usb1", "bus": "pcie-root-port-1", "addr": "0x0"}' -device '{"driver": "usb-tablet", "id": "usb-tablet1", "bus": "usb1.0", "port": "1"}' -device '{"id": "pcie-root-port-2", "port": 2, "driver": "pcie-root-port", "addr": "0x1.0x2", "bus": "pcie.0", "chassis": 3}' -device '{"id": "virtio_scsi_pci0", "driver": "virtio-scsi-pci", "bus": "pcie-root-port-2", "addr": "0x0"}' -blockdev '{"node-name": "file_image1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/timao/virtiofsd/share/rhel960-64-virtio-scsi-ovmf.qcow2", "cache": {"direct": true, "no-flush": false}}' -blockdev '{"node-name": "drive_image1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_image1"}' -device '{"driver": "scsi-hd", "id": "image1", "drive": "drive_image1", "write-cache": "on"}' -chardev socket,id=char_virtiofs_fs,path=/home/timao/virtiofsd/virtiofsd.sock -device '{"id": "pcie-root-port-3", "port": 3, "driver": "pcie-root-port", "addr": "0x1.0x3", "bus": "pcie.0", "chassis": 4}' -device '{"id": "vufs_virtiofs_fs", "chardev": "char_virtiofs_fs", "tag": "myfs", "queue-size": 1024, "driver": "vhost-user-fs-pci", "bus": "pcie-root-port-3", "addr": "0x0"}' -device '{"id": "pcie-root-port-4", "port": 4, "driver": "pcie-root-port", "addr": "0x1.0x4", "bus": "pcie.0", "chassis": 5}' -device '{"driver": "virtio-net-pci", "mac": "9a:93:d9:de:2e:86", "id": "id1hBnNv", "netdev": "id5s0vaW", "bus": "pcie-root-port-4", "addr": "0x0"}' -netdev '{"id": "id5s0vaW", "type": "tap", "vhost": true}' -vnc :0 -rtc base=utc,clock=host,driftfix=slew -boot menu=off,order=cdn,once=c,strict=off -enable-kvm -monitor stdio

      Additaional info:

      (gdb) bt
      #0  0x00007f31392c0e9c in __pthread_kill_implementation () at /lib64/libc.so.6
      #1  0x00007f313926aa96 in raise () at /lib64/libc.so.6
      #2  0x00007f31392528fa in abort () at /lib64/libc.so.6
      #3  0x000055b070a34602 in vhost_log_global_start (listener=0x55b0928471f8, errp=<optimized out>) at ../hw/virtio/vhost.c:1127
      #4  0x000055b070c62c6d in memory_global_dirty_log_do_start (errp=0x7f3136eccfc8) at ../system/memory.c:2934
      #5  memory_global_dirty_log_start (flags=<optimized out>, errp=0x7f3136eccfc8) at ../system/memory.c:2973
      #6  0x000055b070c7f796 in ram_init_bitmaps (rs=0x7f2df4001530, errp=0x7f3136eccfc8) at ../migration/ram.c:2824
      #7  ram_init_all (rsp=0x55b071eb7158 <ram_state.llvm>, errp=0x7f3136eccfc8) at ../migration/ram.c:2858
      #8  ram_save_setup (f=0x55b091193c80, opaque=0x55b071eb7158 <ram_state.llvm>, errp=0x7f3136eccfc8) at ../migration/ram.c:3052
      #9  0x000055b070ab8828 in qemu_savevm_state_setup (f=0x55b091193c80, errp=0x7f3136eccfc8) at ../migration/savevm.c:1380
      #10 0x000055b070aa5416 in migration_thread (opaque=0x55b090f8d0e0) at ../migration/migration.c:3690
      #11 0x000055b070f8231a in qemu_thread_start (args=0x55b091154d80) at ../util/qemu-thread-posix.c:541
      #12 0x00007f31392bf11a in start_thread () at /lib64/libc.so.6
      #13 0x00007f313932fc3c in clone3 () at /lib64/libc.so.6 

              hreitz@redhat.com Hanna Czenczek
              timao@redhat.com Tingting Mao
              virt-maint virt-maint
              Tingting Mao Tingting Mao
              Votes:
              0 Vote for this issue
              Watchers:
              7 Start watching this issue

                Created:
                Updated: