Uploaded image for project: 'RHEL'
  1. RHEL
  2. RHEL-28813

qemu crash with kvm_irqchip_commit_routes: Assertion `ret == 0' failed if booting with many virtio disks and vcpus

    • qemu-kvm-9.0.0-1.el9
    • Major
    • ZStream
    • sst_virtualization
    • ssg_virtualization
    • 2
    • QE ack
    • False
    • Hide

      None

      Show
      None
    • Red Hat Enterprise Linux
    • Approved Blocker
    • x86_64
    • Linux

      
      What were you trying to do that didn't work?
      qemu crash with kvm_irqchip_commit_routes: Assertion `ret == 0' failed if booting with many virtio disks and vcpus
      
                      #0  0x00007f03daa9152f raise (libc.so.6)
                      #1  0x00007f03daa64e65 abort (libc.so.6)
                      #2  0x00007f03daa64d39 __assert_fail_base.cold.0 (libc.so.6)
                      #3  0x00007f03daa89e86 __assert_fail (libc.so.6)
                      #4  0x000055d1628d826e kvm_irqchip_commit_routes (qemu-kvm)
                      #5  0x000055d1628d8369 kvm_irqchip_add_msi_route (qemu-kvm)
                      #6  0x000055d1627948b1 virtio_pci_set_guest_notifiers (qemu-kvm)
                      #7  0x000055d1628523bd virtio_blk_data_plane_start (qemu-kvm)
                      #8  0x000055d162790bda virtio_bus_start_ioeventfd (qemu-kvm)
                      #9  0x000055d1627923e7 virtio_pci_common_write (qemu-kvm)
                      #10 0x000055d16282dbf7 memory_region_write_accessor (qemu-kvm)
                      #11 0x000055d16282d70e access_with_adjusted_size (qemu-kvm)
                      #12 0x000055d16282d973 memory_region_dispatch_write (qemu-kvm)
                      #13 0x000055d16281f26e flatview_write_continue (qemu-kvm)
                      #14 0x000055d16281f3d3 flatview_write (qemu-kvm)
                      #15 0x000055d162822faf address_space_write (qemu-kvm)
                      #16 0x000055d1628d9ef9 kvm_cpu_exec (qemu-kvm)
                      #17 0x000055d1628dad25 kvm_vcpu_thread_fn (qemu-kvm)
                      #18 0x000055d162a17734 qemu_thread_start (qemu-kvm)
                      #19 0x00007f03dae211ca start_thread (libpthread.so.0)
                      #20 0x00007f03daa7c8d3 __clone (libc.so.6)
      
      Please provide the package NVR for which bug is seen:
      Red Hat Enterprise Linux release 8.10 Beta (Ootpa)
      4.18.0-544.el8.x86_64
      qemu-kvm-6.2.0-48.module+el8.10.0+21368+722e1fbd.x86_64
      seabios-bin-1.16.0-4.module+el8.9.0+19570+14a90618.noarch
      edk2-ovmf-20220126gitbb1bba3d77-12.el8.noarch
      virtio-win-prewhql-0.1-248.iso
      
      How reproducible:
      100% 
      
      
      Steps to reproduce
      
      
      1.Boot VM with vcpu > 44 and 100 virtio disks
       
       /usr/libexec/qemu-kvm \
      -S  \
      -name 'avocado-vt-vm1'  \
      -sandbox on \
      -blockdev node-name=file_ovmf_code,driver=file,filename=/usr/share/OVMF/OVMF_CODE.secboot.fd,auto-read-only=on,discard=unmap \
      -blockdev node-name=drive_ovmf_code,driver=raw,read-only=on,file=file_ovmf_code \
      -blockdev node-name=file_ovmf_vars,driver=file,filename=/home/kar/workspace/root/avocado/data/avocado-vt/avocado-vt-vm1_rhel8100-64-virtio-ovmf_qcow2_filesystem_VARS.raw,auto-read-only=on,discard=unmap \
      -blockdev node-name=drive_ovmf_vars,driver=raw,read-only=off,file=file_ovmf_vars \
      -machine q35,pflash0=drive_ovmf_code,pflash1=drive_ovmf_vars,memory-backend=mem-machine_mem \
      -device pcie-root-port,id=pcie-root-port-0,multifunction=on,bus=pcie.0,addr=0x1,chassis=1 \
      -device pcie-pci-bridge,id=pcie-pci-bridge-0,addr=0x0,bus=pcie-root-port-0  \
      -nodefaults \
      -device VGA,bus=pcie.0,addr=0x2 \
      -m 12288 \
      -object memory-backend-ram,size=12288M,id=mem-machine_mem  \
      -smp 64,maxcpus=64,cores=32,threads=1,dies=1,sockets=2  \
      -cpu 'Icelake-Server',ss=on,vmx=on,pdcm=on,hypervisor=on,tsc-adjust=on,avx512ifma=on,sha-ni=on,rdpid=on,fsrm=on,md-clear=on,stibp=on,arch-capabilities=on,xsaves=on,ibpb=on,ibrs=on,amd-stibp=on,amd-ssbd=on,rdctl-no=on,ibrs-all=on,skip-l1dfl-vmentry=on,mds-no=on,pschange-mc-no=on,tsx-ctrl=on,hle=off,rtm=off,mpx=off,intel-pt=off,kvm_pv_unhalt=on \
      -chardev socket,wait=off,path=/var/tmp/avocado_7ewtrdvk/monitor-qmpmonitor1-20240311-060251-MwBmTN4f,server=on,id=qmp_id_qmpmonitor1  \
      -mon chardev=qmp_id_qmpmonitor1,mode=control \
      -chardev socket,wait=off,path=/var/tmp/avocado_7ewtrdvk/monitor-catch_monitor-20240311-060251-MwBmTN4f,server=on,id=qmp_id_catch_monitor  \
      -mon chardev=qmp_id_catch_monitor,mode=control \
      -device pvpanic,ioport=0x505,id=idH3X5Hy \
      -chardev socket,wait=off,path=/var/tmp/avocado_7ewtrdvk/serial-serial0-20240311-060251-MwBmTN4f,server=on,id=chardev_serial0 \
      -device isa-serial,id=serial0,chardev=chardev_serial0  \
      -chardev socket,id=seabioslog_id_20240311-060251-MwBmTN4f,path=/var/tmp/avocado_7ewtrdvk/seabios-20240311-060251-MwBmTN4f,server=on,wait=off \
      -device isa-debugcon,chardev=seabioslog_id_20240311-060251-MwBmTN4f,iobase=0x402 \
      -device pcie-root-port,id=pcie-root-port-1,port=0x1,addr=0x1.0x1,bus=pcie.0,chassis=2 \
      -device qemu-xhci,id=usb1,bus=pcie-root-port-1,addr=0x0 \
      -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \
      -object '{"qom-type": "iothread", "id": "iothread0"}' \
      -object '{"qom-type": "iothread", "id": "iothread1"}' \
      -blockdev '{"node-name": "file_image1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/rhel8100-64-virtio-ovmf.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_image1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_image1"}' \
      -device pcie-root-port,id=pcie-root-port-2,port=0x2,addr=0x1.0x2,bus=pcie.0,chassis=3 \
      -device virtio-blk-pci,id=image1,drive=drive_image1,bootindex=0,write-cache=on,bus=pcie-root-port-2,addr=0x0,iothread=iothread0 \
      -blockdev '{"node-name": "file_stg0", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/stg0.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_stg0", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_stg0"}' \
      -device pcie-root-port,id=pcie-root-port-3,port=0x3,addr=0x1.0x3,bus=pcie.0,chassis=4 \
      -device virtio-blk-pci,id=stg0,drive=drive_stg0,bootindex=1,write-cache=on,serial=stg0,bus=pcie-root-port-3,addr=0x0,iothread=iothread1 \
      -blockdev '{"node-name": "file_stg1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/stg1.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_stg1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_stg1"}' \
      -device pcie-root-port,id=pcie-root-port-4,port=0x4,addr=0x1.0x4,bus=pcie.0,chassis=5 \
      -device virtio-blk-pci,id=stg1,drive=drive_stg1,bootindex=2,write-cache=on,serial=stg1,bus=pcie-root-port-4,addr=0x0,iothread=iothread0 \
      -blockdev '{"node-name": "file_stg2", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/stg2.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_stg2", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_stg2"}' \
      -device pcie-root-port,id=pcie-root-port-5,port=0x5,addr=0x1.0x5,bus=pcie.0,chassis=6 \
      -device virtio-blk-pci,id=stg2,drive=drive_stg2,bootindex=3,write-cache=on,serial=stg2,bus=pcie-root-port-5,addr=0x0,iothread=iothread1 \
      -blockdev '{"node-name": "file_stg3", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/stg3.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_stg3", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_stg3"}' \
      -device pcie-root-port,id=pcie-root-port-6,port=0x6,addr=0x1.0x6,bus=pcie.0,chassis=7 \
      -device virtio-blk-pci,id=stg3,drive=drive_stg3,bootindex=4,write-cache=on,serial=stg3,bus=pcie-root-port-6,addr=0x0,iothread=iothread0 \
      -blockdev '{"node-name": "file_stg4", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/stg4.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_stg4", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_stg4"}' \
      -device pcie-root-port,id=pcie-root-port-7,port=0x7,addr=0x1.0x7,bus=pcie.0,chassis=8 \
      -device virtio-blk-pci,id=stg4,drive=drive_stg4,bootindex=5,write-cache=on,serial=stg4,bus=pcie-root-port-7,addr=0x0,iothread=iothread1 \
      -blockdev '{"node-name": "file_stg5", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/stg5.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_stg5", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_stg5"}' \
      -device pcie-root-port,id=pcie-root-port-8,port=0x8,multifunction=on,bus=pcie.0,addr=0x3,chassis=9 \
      -device virtio-blk-pci,id=stg5,drive=drive_stg5,bootindex=6,write-cache=on,serial=stg5,bus=pcie-root-port-8,addr=0x0,iothread=iothread0 \
      ...
      ...
      ...
      -blockdev '{"node-name": "file_stg96", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/stg96.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_stg96", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_stg96"}' \
      -device pcie-root-port,id=pcie-root-port-99,port=0x63,addr=0xe.0x3,bus=pcie.0,chassis=100 \
      -device virtio-blk-pci,id=stg96,drive=drive_stg96,bootindex=97,write-cache=on,serial=stg96,bus=pcie-root-port-99,addr=0x0,iothread=iothread1 \
      -blockdev '{"node-name": "file_stg97", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/stg97.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_stg97", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_stg97"}' \
      -device pcie-root-port,id=pcie-root-port-100,port=0x64,addr=0xe.0x4,bus=pcie.0,chassis=101 \
      -device virtio-blk-pci,id=stg97,drive=drive_stg97,bootindex=98,write-cache=on,serial=stg97,bus=pcie-root-port-100,addr=0x0,iothread=iothread0 \
      -blockdev '{"node-name": "file_stg98", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/stg98.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_stg98", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_stg98"}' \
      -device pcie-root-port,id=pcie-root-port-101,port=0x65,addr=0xe.0x5,bus=pcie.0,chassis=102 \
      -device virtio-blk-pci,id=stg98,drive=drive_stg98,bootindex=99,write-cache=on,serial=stg98,bus=pcie-root-port-101,addr=0x0,iothread=iothread1 \
      -device pcie-root-port,id=pcie-root-port-102,port=0x66,addr=0xe.0x6,bus=pcie.0,chassis=103 \
      -device virtio-net-pci,mac=9a:36:7a:e2:03:9d,id=iddKwuDC,netdev=idtzc809,bus=pcie-root-port-102,addr=0x0 \
      -netdev tap,id=idtzc809,vhost=on,vhostfd=15,fd=9  \
      -vnc :0  \
      -rtc base=utc,clock=host,driftfix=slew  \
      -boot menu=off,order=cdn,once=c,strict=off \
      -enable-kvm \
      -device pcie-root-port,id=pcie_extra_root_port_0,multifunction=on,bus=pcie.0,addr=0xf,chassis=104
       
      2.
      3.
      
      Expected results
      qemu may boot failed but did not crash
      
      Actual results
      qemu crash
      
      
      It may pass using less vcpus , like as 40
        
      
      

        1. dmesg.txt
          897 kB
        2. dmesg-1.txt
          897 kB
        3. Screenshot from 2024-03-29 14-14-34.png
          Screenshot from 2024-03-29 14-14-34.png
          492 kB

            rhn-engineering-imammedo Igor Mammedov
            qingwangrh qing wang
            virt-maint virt-maint
            qing wang qing wang
            Votes:
            0 Vote for this issue
            Watchers:
            21 Start watching this issue

              Created:
              Updated: