Uploaded image for project: 'RHEL'
  1. RHEL
  2. RHEL-65618

[RHEL10] Failed to hot add PCIe device behind xio3130 downstream

    • No
    • None
    • 1
    • rhel-sst-virtualization
    • ssg_virtualization
    • 1
    • False
    • Hide

      None

      Show
      None
    • None
    • Virt ARM 24-1
    • None
    • None
    • None

      What were you trying to do that didn't work?

      Boot guest, with Upstream Port and Downstream Port topology
      Hot plug Downstream Port device failed
      [  121.833021] pcieport 0000:08:01.0: pciehp: Slot(2): Cannot train link: status 0x2000

      Please provide the package NVR for which the bug is seen:

      qemu-kvm-9.1.0-3.el10
      6.11.0-27.el10.aarch64

      edk2-20240524-9.el10

      How reproducible is this bug?:

      100%

      Steps to reproduce

      1. Boot guest, with Upstream Port and Downstream Port topology

      /usr/libexec/qemu-kvm \
      -name 'avocado-vt-vm1'  \
      -sandbox on \
      -blockdev '{"node-name": "file_aavmf_code", "driver": "file", "filename": "/usr/share/edk2/aarch64/QEMU_EFI-silent-pflash.qcow2", "auto-read-only": true, "discard": "unmap"}' \
      -blockdev '{"node-name": "drive_aavmf_code", "driver": "qcow2", "read-only": true, "file": "file_aavmf_code"}' \
      -blockdev '{"node-name": "file_aavmf_vars", "driver": "file", "filename": "/home/kar/workspace/root/avocado/data/avocado-vt/avocado-vt-vm1_rhel100-aarch64-4k-virtio-scsi_qcow2_filesystem_VARS.qcow2", "auto-read-only": true, "discard": "unmap"}' \
      -blockdev '{"node-name": "drive_aavmf_vars", "driver": "qcow2", "read-only": false, "file": "file_aavmf_vars"}' \
      -machine virt,gic-version=host,its=on,ras=on,pflash0=drive_aavmf_code,pflash1=drive_aavmf_vars,memory-backend=mem-machine_mem \
      -device '{"id": "pcie-root-port-0", "driver": "pcie-root-port", "multifunction": true, "bus": "pcie.0", "addr": "0x1", "chassis": 1}' \
      -device '{"id": "pcie-pci-bridge-0", "driver": "pcie-pci-bridge", "addr": "0x0", "bus": "pcie-root-port-0"}'  \
      -nodefaults \
      -device '{"id": "pcie-root-port-1", "port": 1, "driver": "pcie-root-port", "addr": "0x1.0x1", "bus": "pcie.0", "chassis": 2}' \
      -device '{"driver": "virtio-gpu-pci", "bus": "pcie-root-port-1", "addr": "0x0"}' \
      -m 8192 \
      -object '{"size": 8589934592, "id": "mem-machine_mem", "qom-type": "memory-backend-ram"}'  \
      -smp 4,maxcpus=4,cores=2,threads=1,clusters=1,sockets=2  \
      -cpu 'host' \
      -chardev socket,path=/var/tmp/monitor-qmpmonitor1,wait=off,server=on,id=qmp_id_qmpmonitor1  \
      -mon chardev=qmp_id_qmpmonitor1,mode=control \
      -serial unix:'/var/tmp/serial-serial0',server=on,wait=off \
      -object '{"qom-type": "rng-random", "filename": "/dev/urandom", "id": "passthrough-WH2urJxR"}' \
      -device '{"id": "pcie-root-port-2", "port": 2, "driver": "pcie-root-port", "addr": "0x1.0x2", "bus": "pcie.0", "chassis": 3}' \
      -device '{"driver": "virtio-rng-pci", "id": "virtio-rng-YZyzc5ag", "rng": "passthrough-WH2urJxR", "bus": "pcie-root-port-2", "addr": "0x0"}' \
      -device '{"id": "pcie-root-port-3", "port": 3, "driver": "pcie-root-port", "addr": "0x1.0x3", "bus": "pcie.0", "chassis": 4}' \
      -device '{"driver": "qemu-xhci", "id": "usb1", "bus": "pcie-root-port-3", "addr": "0x0"}' \
      -device '{"driver": "usb-tablet", "id": "usb-tablet1", "bus": "usb1.0", "port": "1"}' \
      -device '{"id": "pcie-root-port-4", "port": 4, "driver": "pcie-root-port", "addr": "0x1.0x4", "bus": "pcie.0", "chassis": 5}' \
      -device '{"id": "virtio_scsi_pci0", "driver": "virtio-scsi-pci", "bus": "pcie-root-port-4", "addr": "0x0"}' \
      -blockdev '{"node-name": "file_image1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/rhel100-aarch64-4k-virtio-scsi.qcow2", "cache": {"direct": true, "no-flush": false}}' \
      -blockdev '{"node-name": "drive_image1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_image1"}' \
      -device '{"driver": "scsi-hd", "id": "image1", "drive": "drive_image1", "write-cache": "on"}' \
      -device '{"id": "pcie-root-port-5", "port": 5, "driver": "pcie-root-port", "addr": "0x1.0x5", "bus": "pcie.0", "chassis": 6}' \
      -device '{"driver": "virtio-net-pci", "mac": "9a:6c:cf:ec:ac:24", "rombar": 0, "id": "idJPRxI0", "netdev": "idndX448", "bus": "pcie-root-port-5", "addr": "0x0"}' \
      -netdev  '{"id": "idndX448", "type": "tap", "vhost": true}'  \
      -device pcie-root-port,id=pcie-root-port-6,chassis=6,slot=5,bus=pcie.0,addr=0x1.0x6  \
      -device x3130-upstream,id=upstream_port1,bus=pcie-root-port-6,addr=0x0          \
      -device xio3130-downstream,id=downstream_port1,bus=upstream_port1,chassis=1,slot=1,addr=0x0 \
      -device pci-testdev,id=test1,bus=downstream_port1,addr=0x0         \
      -device xio3130-downstream,id=downstream_port2,bus=upstream_port1,chassis=2,slot=2,addr=0x1 \
      -vnc :20  \
      -rtc base=utc,clock=host \
      -enable-kvm \
      -monitor stdio

      2. Hot plugging Downstream Port devices hit this issue:

      (qemu) device_add pci-testdev,id=test2,bus=downstream_port2
      [root@dhcp-10-26-1-211 ~]# lspci
      00:00.0 Host bridge: Red Hat, Inc. QEMU PCIe Host bridge
      00:01.0 PCI bridge: Red Hat, Inc. QEMU PCIe Root port
      00:01.1 PCI bridge: Red Hat, Inc. QEMU PCIe Root port
      00:01.2 PCI bridge: Red Hat, Inc. QEMU PCIe Root port
      00:01.3 PCI bridge: Red Hat, Inc. QEMU PCIe Root port
      00:01.4 PCI bridge: Red Hat, Inc. QEMU PCIe Root port
      00:01.5 PCI bridge: Red Hat, Inc. QEMU PCIe Root port
      00:01.6 PCI bridge: Red Hat, Inc. QEMU PCIe Root port
      01:00.0 PCI bridge: Red Hat, Inc. Device 000e
      03:00.0 Display controller: Red Hat, Inc. Virtio 1.0 GPU (rev 01)
      04:00.0 Unclassified device [00ff]: Red Hat, Inc. Virtio 1.0 RNG (rev 01)
      05:00.0 USB controller: Red Hat, Inc. QEMU XHCI Host Controller (rev 01)
      06:00.0 SCSI storage controller: Red Hat, Inc. Virtio 1.0 SCSI (rev 01)
      07:00.0 Ethernet controller: Red Hat, Inc. Virtio 1.0 network device (rev 01)
      08:00.0 PCI bridge: Texas Instruments XIO3130 PCI Express Switch (Upstream) (rev 02)
      09:00.0 PCI bridge: Texas Instruments XIO3130 PCI Express Switch (Downstream) (rev 01)
      09:01.0 PCI bridge: Texas Instruments XIO3130 PCI Express Switch (Downstream) (rev 01)
      0a:00.0 Unclassified device [00ff]: Red Hat, Inc. QEMU PCI Test Device
      
      [root@dhcp-10-26-1-211 ~]# 
      [  120.642822] pcieport 0000:08:01.0: pciehp: Slot(2): Button press: will power on in 5 sec
      [  120.643956] pcieport 0000:08:01.0: pciehp: Slot(2): Card present
      [  120.644513] pcieport 0000:08:01.0: pciehp: Slot(2): Link Up
      [  121.833021] pcieport 0000:08:01.0: pciehp: Slot(2): Cannot train link: status 0x2000

      Other:
      When I manually ignore the link negotiation status detection, 
      hot plugging can succeed. So the problem may be in the link negotiation.
      [root@virt-mtsnow-02 linux.main]# git diff

      diff --git a/drivers/pci/hotplug/pciehp_hpc.c b/drivers/pci/hotplug/pciehp_hpc.c
      index 736ad8baa2a5..66c44571f053 100644
      --- a/drivers/pci/hotplug/pciehp_hpc.c
      +++ b/drivers/pci/hotplug/pciehp_hpc.c
      @@ -312,12 +312,14 @@ int pciehp_check_link_status(struct controller *ctrl)        pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status);
              ctrl_dbg(ctrl, "%s: lnk_status = %x\n", __func__, lnk_status);
      +# if 0 /* ignore the link negotiation status */
              if ((lnk_status & PCI_EXP_LNKSTA_LT) ||
                  !(lnk_status & PCI_EXP_LNKSTA_NLW)) {
                      ctrl_info(ctrl, "Slot(%s): Cannot train link: status %#06x\n",
                                slot_name(ctrl), lnk_status);
                      return -1;
              }
      +#endif
      

       

              rh-ee-seott Sebastian Ott
              zhenyzha@redhat.com Zhenyu Zhang
              virt-maint virt-maint
              virt-bugs virt-bugs
              Votes:
              0 Vote for this issue
              Watchers:
              14 Start watching this issue

                Created:
                Updated: