Uploaded image for project: 'RHEL'
  1. RHEL
  2. RHEL-122216

virtiofs: migration with --cache metadata can not finish within 2 hours and guest hangs sometimes

Linking RHIVOS CVEs to...Migration: Automation ...SWIFT: POC ConversionSync from "Extern...XMLWordPrintable

    • Icon: Bug Bug
    • Resolution: Unresolved
    • Icon: Undefined Undefined
    • None
    • rhel-10.1
    • virtiofsd
    • None
    • None
    • Low
    • rhel-virt-storage
    • None
    • False
    • False
    • Hide

      None

      Show
      None
    • None
    • None
    • None
    • None
    • Unspecified
    • Unspecified
    • Unspecified
    • None

      What were you trying to do that didn't work?

      Virtiofs migration with --cache metadata can not finish within 2 hours and guest hangs sometimes, but with other cache modes, the migration finishes very soon

      What is the impact of this issue to you?

      Please provide the package NVR for which the bug is seen:

      qemu-kvm-10.0.0-14.el10_1

      kernel-6.12.0-124.7.1.el10_1.x86_64

      virtiofsd-1.13.2-1.el10_0.x86_64

      How reproducible is this bug?:

      100%

      Steps to reproduce

      1.Mount nfs server in both source and target nodes

      (source node)# mount -t nfs 10.72.140.12:/home/timao/nfs_share /var/mnt/fs/
(target node)# mount -t nfs 10.72.140.12:/home/timao/nfs_share /var/mnt/targetfs/

      2.In source node, start the virtiofsd and guest as below

      # /usr/libexec/virtiofsd --socket-path=/home/timao/virtiofsd/source/avocado-vt-vm1-fs-virtiofsd.sock --shared-dir /var/mnt/fs --cache metadata &
      /usr/libexec/qemu-kvm \
-S \
-name 'avocado-vt-vm1' \
-sandbox on,elevateprivileges=deny,obsolete=deny,resourcecontrol=deny \
-blockdev '{"node-name": "file_ovmf_code", "driver": "file", "filename": "/usr/share/OVMF/OVMF_CODE.secboot.fd", "auto-read-only": true, "discard": "unmap"}' \
-blockdev '{"node-name": "drive_ovmf_code", "driver": "raw", "read-only": true, "file": "file_ovmf_code"}' \
-blockdev '{"node-name": "file_ovmf_vars", "driver": "file", "filename": "/root/avocado/data/avocado-vt/avocado-vt-vm1_rhel960-64-virtio-ovmf_qcow2_filesystem_VARS.raw", "auto-read-only": true, "discard": "unmap"}' \
-blockdev '{"node-name": "drive_ovmf_vars", "driver": "raw", "read-only": false, "file": "file_ovmf_vars"}' \
-machine q35,pflash0=drive_ovmf_code,pflash1=drive_ovmf_vars \
-device '{"id": "pcie-root-port-0", "driver": "pcie-root-port", "multifunction": true, "bus": "pcie.0", "addr": "0x1", "chassis": 1}' \
-device '{"id": "pcie-pci-bridge-0", "driver": "pcie-pci-bridge", "addr": "0x0", "bus": "pcie-root-port-0"}' \
-nodefaults \
-device '{"driver": "VGA", "bus": "pcie.0", "addr": "0x2"}' \
-m 30720 \
-object '{"size": 32212254720, "share": true, "id": "mem-mem1", "qom-type": "memory-backend-memfd"}' \
-smp 20,maxcpus=20,cores=10,threads=1,dies=1,sockets=2 \
-numa node,memdev=mem-mem1,nodeid=0 \
-cpu 'Icelake-Server-v2',+kvm_pv_unhalt \
-chardev socket,wait=off,path=/var/tmp/avocado_kfuxq3q5/monitor-qmpmonitor1-20251015-221550-XL74d0QG,server=on,id=qmp_id_qmpmonitor1 \
-mon chardev=qmp_id_qmpmonitor1,mode=control \
-chardev socket,wait=off,path=/var/tmp/avocado_kfuxq3q5/monitor-catch_monitor-20251015-221550-XL74d0QG,server=on,id=qmp_id_catch_monitor \
-mon chardev=qmp_id_catch_monitor,mode=control \
-device '{"ioport": 1285, "driver": "pvpanic", "id": "idHvZFQ6"}' \
-chardev socket,wait=off,path=/var/tmp/avocado_kfuxq3q5/serial-serial0-20251015-221550-XL74d0QG,server=on,id=chardev_serial0 \
-device '{"id": "serial0", "driver": "isa-serial", "chardev": "chardev_serial0"}' \
-chardev socket,id=seabioslog_id_20251015-221550-XL74d0QG,path=/var/tmp/avocado_kfuxq3q5/seabios-20251015-221550-XL74d0QG,server=on,wait=off \
-device isa-debugcon,chardev=seabioslog_id_20251015-221550-XL74d0QG,iobase=0x402 \
-device '{"id": "pcie-root-port-1", "port": 1, "driver": "pcie-root-port", "addr": "0x1.0x1", "bus": "pcie.0", "chassis": 2}' \
-device '{"driver": "qemu-xhci", "id": "usb1", "bus": "pcie-root-port-1", "addr": "0x0"}' \
-device '{"driver": "usb-tablet", "id": "usb-tablet1", "bus": "usb1.0", "port": "1"}' \
-object '{"id": "iothread0", "qom-type": "iothread"}' \
-object '{"id": "iothread1", "qom-type": "iothread"}' \
-object '{"id": "iothread2", "qom-type": "iothread"}' \
-blockdev '{"node-name": "file_image1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/var/mnt/fs/rhel960-64-virtio-ovmf.qcow2", "cache": {"direct": true, "no-flush": false}}' \
-blockdev '{"node-name": "drive_image1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_image1"}' \
-device '{"id": "pcie-root-port-2", "port": 2, "driver": "pcie-root-port", "addr": "0x1.0x2", "bus": "pcie.0", "chassis": 3}' \
-device '{"driver": "virtio-blk-pci", "id": "image1", "drive": "drive_image1", "write-cache": "on", "iothread-vq-mapping": [{"iothread": "iothread0"}, {"iothread": "iothread1"}, {"iothread": "iothread2"}], "bus": "pcie-root-port-2", "addr": "0x0"}' \
-chardev socket,id=char_virtiofs_fs,path=/home/timao/virtiofsd/source/avocado-vt-vm1-fs-virtiofsd.sock \
-device '{"id": "pcie-root-port-3", "port": 3, "driver": "pcie-root-port", "addr": "0x1.0x3", "bus": "pcie.0", "chassis": 4}' \
-device '{"id": "vufs_virtiofs_fs", "chardev": "char_virtiofs_fs", "tag": "myfs", "queue-size": 1024, "driver": "vhost-user-fs-pci", "bus": "pcie-root-port-3", "addr": "0x0"}' \
-device '{"id": "pcie-root-port-4", "port": 4, "driver": "pcie-root-port", "addr": "0x1.0x4", "bus": "pcie.0", "chassis": 5}' \
-device '{"driver": "virtio-net-pci", "mac": "9a:a0:d4:b0:9a:46", "id": "idPrFE1q", "netdev": "idDzSo0N", "bus": "pcie-root-port-4", "addr": "0x0"}' \
-netdev '{"id": "idDzSo0N", "type": "tap", "vhost": true}' \
-vnc :0 \
-rtc base=utc,clock=host,driftfix=slew \
-boot menu=off,order=cdn,once=c,strict=off \
-enable-kvm \
-monitor stdio

      3.In the guest mount the virtiofs and writng data to it

      (guest)# mount -t virtiofs myfs /mnt
(guest)# fio --name=fiotest --filename=/mnt/fiotest --size=5G --bs=4k --rw=rw --ioengine=libaio --runtime=9999999 --time_based

      4.Start guest in target node

      # /usr/libexec/virtiofsd --socket-path=/home/timao/virtiofsd/target/avocado-vt-vm1-targetfs-virtiofsd.sock --shared-dir /var/mnt/targetfs --cache metadata &
      /usr/libexec/qemu-kvm \
-S \
-name 'avocado-vt-vm1' \
-sandbox on,elevateprivileges=deny,obsolete=deny,resourcecontrol=deny \
-blockdev '{"node-name": "file_ovmf_code", "driver": "file", "filename": "/usr/share/OVMF/OVMF_CODE.secboot.fd", "auto-read-only": true, "discard": "unmap"}' \
-blockdev '{"node-name": "drive_ovmf_code", "driver": "raw", "read-only": true, "file": "file_ovmf_code"}' \
-blockdev '{"node-name": "file_ovmf_vars", "driver": "file", "filename": "/root/avocado/data/avocado-vt/avocado-vt-vm1_rhel960-64-virtio-ovmf_qcow2_filesystem_VARS.raw", "auto-read-only": true, "discard": "unmap"}' \
-blockdev '{"node-name": "drive_ovmf_vars", "driver": "raw", "read-only": false, "file": "file_ovmf_vars"}' \
-machine q35,pflash0=drive_ovmf_code,pflash1=drive_ovmf_vars \
-device '{"id": "pcie-root-port-0", "driver": "pcie-root-port", "multifunction": true, "bus": "pcie.0", "addr": "0x1", "chassis": 1}' \
-device '{"id": "pcie-pci-bridge-0", "driver": "pcie-pci-bridge", "addr": "0x0", "bus": "pcie-root-port-0"}' \
-nodefaults \
-device '{"driver": "VGA", "bus": "pcie.0", "addr": "0x2"}' \
-m 30720 \
-object '{"size": 32212254720, "share": true, "id": "mem-mem1", "qom-type": "memory-backend-memfd"}' \
-smp 20,maxcpus=20,cores=10,threads=1,dies=1,sockets=2 \
-numa node,memdev=mem-mem1,nodeid=0 \
-cpu 'Icelake-Server-v2',+kvm_pv_unhalt \
-chardev socket,wait=off,path=/var/tmp/avocado_kfuxq3q5/monitor-qmpmonitor1-20251015-221550-XL74d0QG,server=on,id=qmp_id_qmpmonitor1 \
-mon chardev=qmp_id_qmpmonitor1,mode=control \
-chardev socket,wait=off,path=/var/tmp/avocado_kfuxq3q5/monitor-catch_monitor-20251015-221550-XL74d0QG,server=on,id=qmp_id_catch_monitor \
-mon chardev=qmp_id_catch_monitor,mode=control \
-device '{"ioport": 1285, "driver": "pvpanic", "id": "idHvZFQ6"}' \
-chardev socket,wait=off,path=/var/tmp/avocado_kfuxq3q5/serial-serial0-20251015-221550-XL74d0QG,server=on,id=chardev_serial0 \
-device '{"id": "serial0", "driver": "isa-serial", "chardev": "chardev_serial0"}' \
-chardev socket,id=seabioslog_id_20251015-221550-XL74d0QG,path=/var/tmp/avocado_kfuxq3q5/seabios-20251015-221550-XL74d0QG,server=on,wait=off \
-device isa-debugcon,chardev=seabioslog_id_20251015-221550-XL74d0QG,iobase=0x402 \
-device '{"id": "pcie-root-port-1", "port": 1, "driver": "pcie-root-port", "addr": "0x1.0x1", "bus": "pcie.0", "chassis": 2}' \
-device '{"driver": "qemu-xhci", "id": "usb1", "bus": "pcie-root-port-1", "addr": "0x0"}' \
-device '{"driver": "usb-tablet", "id": "usb-tablet1", "bus": "usb1.0", "port": "1"}' \
-object '{"id": "iothread0", "qom-type": "iothread"}' \
-object '{"id": "iothread1", "qom-type": "iothread"}' \
-object '{"id": "iothread2", "qom-type": "iothread"}' \
-blockdev '{"node-name": "file_image1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/var/mnt/targetfs/rhel960-64-virtio-ovmf.qcow2", "cache": {"direct": true, "no-flush": false}}' \
-blockdev '{"node-name": "drive_image1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_image1"}' \
-device '{"id": "pcie-root-port-2", "port": 2, "driver": "pcie-root-port", "addr": "0x1.0x2", "bus": "pcie.0", "chassis": 3}' \
-device '{"driver": "virtio-blk-pci", "id": "image1", "drive": "drive_image1", "write-cache": "on", "iothread-vq-mapping": [{"iothread": "iothread0"}, {"iothread": "iothread1"}, {"iothread": "iothread2"}], "bus": "pcie-root-port-2", "addr": "0x0"}' \
-chardev socket,id=char_virtiofs_fs,path=/home/timao/virtiofsd/target/avocado-vt-vm1-targetfs-virtiofsd.sock \
-device '{"id": "pcie-root-port-3", "port": 3, "driver": "pcie-root-port", "addr": "0x1.0x3", "bus": "pcie.0", "chassis": 4}' \
-device '{"id": "vufs_virtiofs_fs", "chardev": "char_virtiofs_fs", "tag": "myfs", "queue-size": 1024, "driver": "vhost-user-fs-pci", "bus": "pcie-root-port-3", "addr": "0x0"}' \
-device '{"id": "pcie-root-port-4", "port": 4, "driver": "pcie-root-port", "addr": "0x1.0x4", "bus": "pcie.0", "chassis": 5}' \
-device '{"driver": "virtio-net-pci", "mac": "9a:a0:d4:b0:9a:46", "id": "idPrFE1q", "netdev": "idDzSo0N", "bus": "pcie-root-port-4", "addr": "0x0"}' \
-netdev '{"id": "idDzSo0N", "type": "tap", "vhost": true}' \
-vnc :0 \
-rtc base=utc,clock=host,driftfix=slew \
-boot menu=off,order=cdn,once=c,strict=off \
-enable-kvm \
-monitor stdio \
-incoming defer

      5.Start migration

      target node: (qemu) migrate_incoming tcp:[::]:5800
source node: {"execute": "migrate","arguments":{"uri":"tcp:10.72.140.40:5800"}}

      Expected results

      Migration successes.

      Actual results

      Migration can not finished within 2 hours, and sometimes the guest hangs in source node, and the qemu quit expected in target node as below

      (qemu) migrate_incoming tcp:[::]:5800
(qemu) qemu-kvm: Putting registers after init: Failed to set special registers: Invalid argument
[2025-10-17T03:16:05Z INFO  virtiofsd] Client disconnected, shutting down
[1]+  Done                    /usr/libexec/virtiofsd --socket-path=/home/timao/virtiofsd/target/avocado-vt-vm1-targetfs-virtiofsd.sock --shared-dir /var/mnt/targetfs --cache metadata

       

      Additional info:

      1.The same operations but  with other cache modes(always, never, auto), the migration finishes very soon

      2.The migration can finish quickly if setting  migrate_set_parameter downtime-limit 50000 before migration in source node

       

              gmaglion German Maglione
              timao@redhat.com Tingting Mao
              virt-maint virt-maint
              Tingting Mao Tingting Mao
              Votes:
              0 Vote for this issue
              Watchers:
              4 Start watching this issue

                Created:
                Updated: