Uploaded image for project: 'RHEL'
  1. RHEL
  2. RHEL-51895

console PTY hung when using '--console pty,target.type=virtio'

    • Yes
    • Impediment
    • Important
    • rhel-sst-virtualization
    • ssg_virtualization
    • None
    • True
    • Hide

      network-qe use virsh console command to run test code on target guest. This issue always blocks our test.

      Show
      network-qe use virsh console command to run test code on target guest. This issue always blocks our test.
    • None
    • None
    • None
    • None
    • x86_64
    • None

      What were you trying to do that didn't work?

      virsh console hung on guest.

      Please provide the package NVR for which bug is seen:

      host:

      kernel version: 6.10.0-15.el10.x86_64+rt/kernel-6.10.0-0.rc5.12.el10+rt

      qemu-kvm-9.0.0-4.el10.x86_64

      host cmdline:

      [root@dell-per750-03 ~]# cat /proc/cmdline 
      BOOT_IMAGE=(hd0,gpt2)/vmlinuz-6.10.0-15.el10.x86_64+rt root=/dev/mapper/rhel_dell--per750--03-root ro intel_iommu=on ksdevice=bootif pci=realloc crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M resume=UUID=59dba638-5805-4b9b-947d-326713fbb703 rd.lvm.lv=rhel_dell-per750-03/root rd.lvm.lv=rhel_dell-per750-03/swap console=ttyS0,115200n8 

      guest:

      kernel version: 6.10.0-15.el10.x86_64+rt/kernel-6.10.0-0.rc5.12.el10+rt

      guest infomation:

       

      [root@localhost ~]# cat /proc/cmdline 
      BOOT_IMAGE=(hd0,gpt2)/vmlinuz-6.10.0-15.el10.x86_64+rt root=UUID=1f3fcfbc-b18a-4b37-b23c-4ea97149bb2a ro rhgb quiet crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M resume=UUID=04c94d23-175f-4a4c-a469-a5cac78f6c59 console=ttyS0,115200
      # systemctl --version      
      systemd 256 (256-6.el10)
      +PAM +AUDIT +SELINUX -APPARMOR +IMA +SMACK +SECCOMP -GCRYPT -GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 -IDN -IPTC +KMOD +LIBCRYPTSETUP +LIBCRYPTSETUP_PLUGINS +LIBFDISK +PCRE2 +PWQUALITY +P11KIT -QRENCODE +TPM2 +BZIP2 +LZ4 +XZ +ZLIB +ZSTD +BPF_FRAMEWORK +XKBCOMMON +UTMP +SYSVINIT +LIBARCHIV
      [root@localhost ~]# localectl 
      System Locale: LANG=en_US.UTF-8
          VC Keymap: us
         X11 Layout: us
          X11 Model: pc105
      [root@localhost ~]# cat /etc/vconsole.conf 
      KEYMAP="us"
      FONT="eurlatgr" [root@localhost ~]# cat /usr/lib/systemd/system/systemd-vconsole-setup.service | grep -v ^# | grep -v ^$
      [Unit]
      Description=Virtual Console Setup
      Documentation=man:systemd-vconsole-setup.service(8) man:vconsole.conf(5)
      ConditionPathExists=/dev/tty0
      DefaultDependencies=no
      Before=sysinit.target
      Before=initrd-switch-root.target shutdown.target
      StartLimitIntervalSec=0
      [Service]
      Type=oneshot
      SuccessExitStatus=SIGTERM
      RemainAfterExit=yes
      ExecStart=/usr/lib/systemd/systemd-vconsole-setup
      ImportCredential=vconsole.*
      

       

      How reproducible: always

      Steps to reproduce

      1.create a guest

      virt-install \
          --name g1 \
          --machine=q35 \
          --cpu host-passthrough \
          --vcpus=2 \
          --ram=2048 \
          --disk path=/var/lib/libvirt/images/g1.qcow2,device=disk,bus=virtio,format=qcow2 \
          --import \
          --network bridge=virbr0,model=virtio \
          --boot hd \
          --accelerate \
          --force \
          --graphics none \
          --console pty,target.type=virtio \
          --noautoconsole \
          --os-variant=rhel-unknown 

      guest xml

      [root@dell-per750-03 ~]# virsh dumpxml g2
      <domain type='kvm' id='2'>
        <name>g2</name>
        <uuid>a8b4f21f-fa93-4048-8da2-4f56c574773e</uuid>
        <metadata>
          <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
            <libosinfo:os id="http://redhat.com/rhel/unknown"/>
          </libosinfo:libosinfo>
        </metadata>
        <memory unit='KiB'>2097152</memory>
        <currentMemory unit='KiB'>2097152</currentMemory>
        <vcpu placement='static'>2</vcpu>
        <resource>
          <partition>/machine</partition>
        </resource>
        <os>
          <type arch='x86_64' machine='pc-q35-rhel9.4.0'>hvm</type>
          <boot dev='hd'/>
        </os>
        <features>
          <acpi/>
          <apic/>
        </features>
        <cpu mode='host-passthrough' check='none' migratable='on'/>
        <clock offset='utc'>
          <timer name='rtc' tickpolicy='catchup'/>
          <timer name='pit' tickpolicy='delay'/>
          <timer name='hpet' present='no'/>
        </clock>
        <on_poweroff>destroy</on_poweroff>
        <on_reboot>restart</on_reboot>
        <on_crash>destroy</on_crash>
        <pm>
          <suspend-to-mem enabled='no'/>
          <suspend-to-disk enabled='no'/>
        </pm>
        <devices>
          <emulator>/usr/libexec/qemu-kvm</emulator>
          <disk type='file' device='disk'>
            <driver name='qemu' type='qcow2'/>
            <source file='/var/lib/libvirt/images/g2.qcow2' index='1'/>
            <backingStore/>
            <target dev='vda' bus='virtio'/>
            <alias name='virtio-disk0'/>
            <address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
          </disk>
          <controller type='usb' index='0' model='ich9-ehci1'>
            <alias name='usb'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x1d' function='0x7'/>
          </controller>
          <controller type='usb' index='0' model='ich9-uhci1'>
            <alias name='usb'/>
            <master startport='0'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x1d' function='0x0' multifunction='on'/>
          </controller>
          <controller type='usb' index='0' model='ich9-uhci2'>
            <alias name='usb'/>
            <master startport='2'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x1d' function='0x1'/>
          </controller>
          <controller type='usb' index='0' model='ich9-uhci3'>
            <alias name='usb'/>
            <master startport='4'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x1d' function='0x2'/>
          </controller>
          <controller type='pci' index='0' model='pcie-root'>
            <alias name='pcie.0'/>
          </controller>
          <controller type='pci' index='1' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='1' port='0x8'/>
            <alias name='pci.1'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' multifunction='on'/>
          </controller>
          <controller type='pci' index='2' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='2' port='0x9'/>
            <alias name='pci.2'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
          </controller>
          <controller type='pci' index='3' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='3' port='0xa'/>
            <alias name='pci.3'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
          </controller>
          <controller type='pci' index='4' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='4' port='0xb'/>
            <alias name='pci.4'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x3'/>
          </controller>
          <controller type='pci' index='5' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='5' port='0xc'/>
            <alias name='pci.5'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x4'/>
          </controller>
          <controller type='pci' index='6' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='6' port='0xd'/>
            <alias name='pci.6'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x5'/>
          </controller>
          <controller type='pci' index='7' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='7' port='0xe'/>
            <alias name='pci.7'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x6'/>
          </controller>
          <controller type='pci' index='8' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='8' port='0xf'/>
            <alias name='pci.8'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x7'/>
          </controller>
          <controller type='pci' index='9' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='9' port='0x10'/>
            <alias name='pci.9'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
          </controller>
          <controller type='pci' index='10' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='10' port='0x11'/>
            <alias name='pci.10'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
          </controller>
          <controller type='pci' index='11' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='11' port='0x12'/>
            <alias name='pci.11'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
          </controller>
          <controller type='pci' index='12' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='12' port='0x13'/>
            <alias name='pci.12'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
          </controller>
          <controller type='pci' index='13' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='13' port='0x14'/>
            <alias name='pci.13'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
          </controller>
          <controller type='pci' index='14' model='pcie-root-port'>
            <model name='pcie-root-port'/>
            <target chassis='14' port='0x15'/>
            <alias name='pci.14'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
          </controller>
          <controller type='sata' index='0'>
            <alias name='ide'/>
            <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
          </controller>
          <controller type='virtio-serial' index='0'>
            <alias name='virtio-serial0'/>
            <address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
          </controller>
          <interface type='bridge'>
            <mac address='52:54:00:a8:6d:9d'/>
            <source bridge='virbr0'/>
            <target dev='vnet1'/>
            <model type='virtio'/>
            <alias name='net0'/>
            <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
          </interface>
          <console type='pty' tty='/dev/pts/5'>
            <source path='/dev/pts/5'/>
            <target type='virtio' port='0'/>
            <alias name='console0'/>
          </console>
          <channel type='unix'>
            <source mode='bind' path='/run/libvirt/qemu/channel/2-g2/org.qemu.guest_agent.0'/>
            <target type='virtio' name='org.qemu.guest_agent.0' state='connected'/>
            <alias name='channel0'/>
            <address type='virtio-serial' controller='0' bus='0' port='1'/>
          </channel>
          <input type='mouse' bus='ps2'>
            <alias name='input0'/>
          </input>
          <input type='keyboard' bus='ps2'>
            <alias name='input1'/>
          </input>
          <audio id='1' type='none'/>
          <watchdog model='itco' action='reset'>
            <alias name='watchdog0'/>
          </watchdog>
          <memballoon model='virtio'>
            <alias name='balloon0'/>
            <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
          </memballoon>
        </devices>
        <seclabel type='dynamic' model='selinux' relabel='yes'>
          <label>system_u:system_r:svirt_t:s0:c139,c580</label>
          <imagelabel>system_u:object_r:svirt_image_t:s0:c139,c580</imagelabel>
        </seclabel>
        <seclabel type='dynamic' model='dac' relabel='yes'>
          <label>+107:+107</label>
          <imagelabel>+107:+107</imagelabel>
        </seclabel>
      </domain>
       

      2.use expect script(vmsh) that call 'virsh cosole' command. To run command through vmsh.

      pkt="Ether(src='00:de:ad:00:00:02',dst='00:de:ad:00:00:01')/IPv6(src='2001::1',dst='2001::2')/SCTP(sport=1234,dport=4321)"
      script_scapy=$(cat <<- EOF
      from scapy.all import *
      class MPLS(Packet):
          name='MPLS'
          fields_desc=[ BitField('label', 3, 20), BitField('tc', 0, 3), BitField('s', 1, 1), ByteField('ttl', 64) ]
      bind_layers(Ether, MPLS, type=0x8847)
      sendp($pkt, iface='lo', loop=1, inter=0.1)
      EOF
      )
      python="/usr/libexec/platform-python"
      while ((i<1000)); do
             echo "== cycle=$i ====================================="
             vmsh run_cmd g2 "nohup $python -u -c \"$script_scapy\" </dev/null &>/dev/null &"
             sleep 1
             vmsh run_cmd g2 "ps -ef | grep 'scapy' | awk '{system(\"kill \"\$2\"&>/dev/null\")}'"
      done 
      # sh test.sh
      ....snipet....
      == cycle=6 =====================================
      spawn virsh console g2
      Connected to domain 'g2'
      Escape character is ^] (Ctrl + ])localhost login: root
      Password: 
      Last login: Wed Jul 31 03:24:32 on hvc0
      [root@localhost ~]# nohup /usr/libexec/platform-python -u -c "from scapy.all import *
      > class MPLS(Packet):
      , 3, 20), BitField('tc', 0, 3), BitField('s', 1, 1), ByteField('ttl', 64) ]
      bind_layers(Ether, MPLS, type=0x8847)
      sendp(Ether(src='00:de:ad:00:00:02',dst='00:de:ad:00:00:01')/IPv6(src='2001::1',dst='2001::2')/SCTP(sport=1234,dport=4321), iface='lo', loop=1, inter=0.1)" </dev/null &>/dev/null &

      3. check guest virtio-serial thread status.

      [root@localhost ~]# ps -ef | grep 713
      root         713       1  0 03:19 ?        00:00:00 /usr/bin/qemu-ga --method=virtio-serial --path=/dev/virtio-ports/org.qemu.guest_agent.0 --allow-rpcs=guest-sync-delimited,guest-sync,guest-ping,guest-get-time,guest-set-time,guest-info,guest-shutdown,guest-fsfreeze-status,guest-fsfreeze-freeze,guest-fsfreeze-freeze-list,guest-fsfreeze-thaw,guest-fstrim,guest-suspend-disk,guest-suspend-ram,guest-suspend-hybrid,guest-network-get-interfaces,guest-get-vcpus,guest-set-vcpus,guest-get-disks,guest-get-fsinfo,guest-set-user-password,guest-get-memory-blocks,guest-set-memory-blocks,guest-get-memory-block-info,guest-get-host-name,guest-get-users,guest-get-timezone,guest-get-osinfo,guest-get-devices,guest-ssh-get-authorized-keys,guest-ssh-add-authorized-keys,guest-ssh-remove-authorized-keys,guest-get-diskstats,guest-get-cpustats -F/etc/qemu-ga/fsfreeze-hook
      
      [root@localhost ~]# strace -p 713
      strace: Process 713 attached
      restart_syscall(<... resuming interrupted read ...>
        

      4. check vmsh thread status on host

      [root@dell-per750-03 ~]# ps -aux | grep vmsh
      root       63758  0.0  0.0  14444  4660 ?        Sl   02:31   0:00 /usr/bin/expect /usr/local/bin/vmsh run_cmd g1 nohup /usr/libexec/platform-python -u -c "from scapy.all import * class MPLS(Packet):     name='MPLS'     fields_desc=[ BitField('label', 3, 20), BitField('tc', 0, 3), BitField('s', 1, 1), ByteField('ttl', 64) ] bind_layers(Ether, MPLS, type=0x8847) sendp(Ether(src='00:de:ad:00:00:02',dst='00:de:ad:00:00:01')/IPv6(src='2001::1',dst='2001::2')/SCTP(sport=1234,dport=4321), iface='$NIC_TEST', loop=1, inter=0.1)" </dev/null &>/dev/null &
      root       64803  0.0  0.0  14516  4668 pts/6    Sl+  03:24   0:00 /usr/bin/expect /usr/local/bin/vmsh run_cmd g2 nohup /usr/libexec/platform-python -u -c "from scapy.all import * class MPLS(Packet): name='MPLS' fields_desc=[ BitField('label', 3, 20), BitField('tc', 0, 3), BitField('s', 1, 1), ByteField('ttl', 64) ] bind_layers(Ether, MPLS, type=0x8847) sendp(Ether(src='00:de:ad:00:00:02',dst='00:de:ad:00:00:01')/IPv6(src='2001::1',dst='2001::2')/SCTP(sport=1234,dport=4321), iface='lo', loop=1, inter=0.1)" </dev/null &>/dev/null &
      root       64887  0.0  0.0   6360  1920 pts/4    S+   03:30   0:00 grep --color=auto vmsh
      
      [root@dell-per750-03 ~]# strace -p 63758
      strace: Process 63758 attached
      restart_syscall(<... resuming interrupted read ...>^Cstrace: Process 63758 detached
       <detached ...>[root@dell-per750-03 ~]# strace -p 64803
      strace: Process 64803 attached
      restart_syscall(<... resuming interrupted read ...>

      Expected results

      Actual results

      1.virsh console hung

      2.guest systemd print failed to lock /dev.console

      [root@localhost ~]# journalctl -u systemd-vconsole-setup.service -b
      Jul 31 00:12:28 localhost systemd-vconsole-setup[328]: Failed to lock /dev/console, proceeding without lock: Input/output error
      Jul 31 00:12:28 localhost systemd-vconsole-setup[328]: /usr/bin/setfont failed with a "system error" (EX_OSERR), ignoring.
      Jul 31 00:12:28 localhost systemd-vconsole-setup[334]: setfont: ERROR kdfontop.c:211 put_font_kdfontop: ioctl(KDFONTOP): Invalid argument
      Jul 31 00:12:28 localhost systemd-vconsole-setup[328]: Setting source virtual console failed, ignoring remaining ones.
      Jul 31 00:12:28 localhost systemd[1]: Finished systemd-vconsole-setup.service - Virtual Console Setup.
      Jul 31 00:12:29 localhost systemd[1]: systemd-vconsole-setup.service: Deactivated successfully.
      Jul 31 00:12:29 localhost systemd[1]: Stopped systemd-vconsole-setup.service - Virtual Console Setup.
      Jul 31 00:12:29 localhost systemd[1]: Stopping systemd-vconsole-setup.service - Virtual Console Setup...
      Jul 31 00:12:29 localhost systemd[1]: Starting systemd-vconsole-setup.service - Virtual Console Setup...
      Jul 31 00:12:29 localhost systemd-vconsole-setup[448]: Failed to lock /dev/console, proceeding without lock: Input/output error
      Jul 31 00:12:29 localhost systemd-vconsole-setup[467]: setfont: ERROR kdfontop.c:211 put_font_kdfontop: ioctl(KDFONTOP): Invalid argument
      Jul 31 00:12:29 localhost systemd-vconsole-setup[448]: /usr/bin/setfont failed with a "system error" (EX_OSERR), ignoring.
      Jul 31 00:12:29 localhost systemd-vconsole-setup[448]: Setting source virtual console failed, ignoring remaining ones.
      Jul 31 00:12:29 localhost systemd[1]: Finished systemd-vconsole-setup.service - Virtual Console Setup.
      Jul 31 00:12:34 localhost systemd[1]: systemd-vconsole-setup.service: Deactivated successfully.
      Jul 31 00:12:34 localhost systemd[1]: Stopped systemd-vconsole-setup.service - Virtual Console Setup.
      Jul 31 00:12:34 localhost systemd[1]: Stopping systemd-vconsole-setup.service - Virtual Console Setup...
      Jul 31 00:12:34 localhost systemd[1]: Starting systemd-vconsole-setup.service - Virtual Console Setup...
      Jul 31 00:12:34 localhost systemd-vconsole-setup[651]: Failed to lock /dev/console, proceeding without lock: Input/output error
      Jul 31 00:12:34 localhost systemd-vconsole-setup[656]: setfont: ERROR kdfontop.c:211 put_font_kdfontop: ioctl(KDFONTOP): Invalid argument
      Jul 31 00:12:34 localhost systemd-vconsole-setup[651]: /usr/bin/setfont failed with a "system error" (EX_OSERR), ignoring.
      Jul 31 00:12:34 localhost systemd-vconsole-setup[651]: Setting source virtual console failed, ignoring remaining ones.
      Jul 31 00:12:34 localhost systemd[1]: Finished systemd-vconsole-setup.service - Virtual Console Setup.
       

       

        1. all_stock_dmesg.log
          66 kB
        2. dmesg.log
          67 kB
        3. libvirtd.log
          3.54 MB
        4. vmsh
          9 kB

              mlureau Marc-Andre Lureau
              mhou@redhat.com Minxi Hou
              virt-maint virt-maint
              NaNa Liu NaNa Liu
              Votes:
              0 Vote for this issue
              Watchers:
              21 Start watching this issue

                Created:
                Updated: