-
Bug
-
Resolution: Unresolved
-
Major
-
None
-
rhel-10.0
-
Yes
-
Impediment
-
Important
-
rhel-sst-virtualization
-
ssg_virtualization
-
None
-
True
-
-
None
-
None
-
None
-
None
-
-
x86_64
-
None
What were you trying to do that didn't work?
virsh console hung on guest.
Please provide the package NVR for which bug is seen:
host:
kernel version: 6.10.0-15.el10.x86_64+rt/kernel-6.10.0-0.rc5.12.el10+rt
qemu-kvm-9.0.0-4.el10.x86_64
host cmdline:
[root@dell-per750-03 ~]# cat /proc/cmdline BOOT_IMAGE=(hd0,gpt2)/vmlinuz-6.10.0-15.el10.x86_64+rt root=/dev/mapper/rhel_dell--per750--03-root ro intel_iommu=on ksdevice=bootif pci=realloc crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M resume=UUID=59dba638-5805-4b9b-947d-326713fbb703 rd.lvm.lv=rhel_dell-per750-03/root rd.lvm.lv=rhel_dell-per750-03/swap console=ttyS0,115200n8
guest:
kernel version: 6.10.0-15.el10.x86_64+rt/kernel-6.10.0-0.rc5.12.el10+rt
guest infomation:
[root@localhost ~]# cat /proc/cmdline BOOT_IMAGE=(hd0,gpt2)/vmlinuz-6.10.0-15.el10.x86_64+rt root=UUID=1f3fcfbc-b18a-4b37-b23c-4ea97149bb2a ro rhgb quiet crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M resume=UUID=04c94d23-175f-4a4c-a469-a5cac78f6c59 console=ttyS0,115200 # systemctl --version systemd 256 (256-6.el10) +PAM +AUDIT +SELINUX -APPARMOR +IMA +SMACK +SECCOMP -GCRYPT -GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 -IDN -IPTC +KMOD +LIBCRYPTSETUP +LIBCRYPTSETUP_PLUGINS +LIBFDISK +PCRE2 +PWQUALITY +P11KIT -QRENCODE +TPM2 +BZIP2 +LZ4 +XZ +ZLIB +ZSTD +BPF_FRAMEWORK +XKBCOMMON +UTMP +SYSVINIT +LIBARCHIV [root@localhost ~]# localectl System Locale: LANG=en_US.UTF-8 VC Keymap: us X11 Layout: us X11 Model: pc105 [root@localhost ~]# cat /etc/vconsole.conf KEYMAP="us" FONT="eurlatgr" [root@localhost ~]# cat /usr/lib/systemd/system/systemd-vconsole-setup.service | grep -v ^# | grep -v ^$ [Unit] Description=Virtual Console Setup Documentation=man:systemd-vconsole-setup.service(8) man:vconsole.conf(5) ConditionPathExists=/dev/tty0 DefaultDependencies=no Before=sysinit.target Before=initrd-switch-root.target shutdown.target StartLimitIntervalSec=0 [Service] Type=oneshot SuccessExitStatus=SIGTERM RemainAfterExit=yes ExecStart=/usr/lib/systemd/systemd-vconsole-setup ImportCredential=vconsole.*
How reproducible: always
Steps to reproduce
1.create a guest
virt-install \ --name g1 \ --machine=q35 \ --cpu host-passthrough \ --vcpus=2 \ --ram=2048 \ --disk path=/var/lib/libvirt/images/g1.qcow2,device=disk,bus=virtio,format=qcow2 \ --import \ --network bridge=virbr0,model=virtio \ --boot hd \ --accelerate \ --force \ --graphics none \ --console pty,target.type=virtio \ --noautoconsole \ --os-variant=rhel-unknown
guest xml
[root@dell-per750-03 ~]# virsh dumpxml g2 <domain type='kvm' id='2'> <name>g2</name> <uuid>a8b4f21f-fa93-4048-8da2-4f56c574773e</uuid> <metadata> <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0"> <libosinfo:os id="http://redhat.com/rhel/unknown"/> </libosinfo:libosinfo> </metadata> <memory unit='KiB'>2097152</memory> <currentMemory unit='KiB'>2097152</currentMemory> <vcpu placement='static'>2</vcpu> <resource> <partition>/machine</partition> </resource> <os> <type arch='x86_64' machine='pc-q35-rhel9.4.0'>hvm</type> <boot dev='hd'/> </os> <features> <acpi/> <apic/> </features> <cpu mode='host-passthrough' check='none' migratable='on'/> <clock offset='utc'> <timer name='rtc' tickpolicy='catchup'/> <timer name='pit' tickpolicy='delay'/> <timer name='hpet' present='no'/> </clock> <on_poweroff>destroy</on_poweroff> <on_reboot>restart</on_reboot> <on_crash>destroy</on_crash> <pm> <suspend-to-mem enabled='no'/> <suspend-to-disk enabled='no'/> </pm> <devices> <emulator>/usr/libexec/qemu-kvm</emulator> <disk type='file' device='disk'> <driver name='qemu' type='qcow2'/> <source file='/var/lib/libvirt/images/g2.qcow2' index='1'/> <backingStore/> <target dev='vda' bus='virtio'/> <alias name='virtio-disk0'/> <address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/> </disk> <controller type='usb' index='0' model='ich9-ehci1'> <alias name='usb'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x1d' function='0x7'/> </controller> <controller type='usb' index='0' model='ich9-uhci1'> <alias name='usb'/> <master startport='0'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x1d' function='0x0' multifunction='on'/> </controller> <controller type='usb' index='0' model='ich9-uhci2'> <alias name='usb'/> <master startport='2'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x1d' function='0x1'/> </controller> <controller type='usb' index='0' model='ich9-uhci3'> <alias name='usb'/> <master startport='4'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x1d' function='0x2'/> </controller> <controller type='pci' index='0' model='pcie-root'> <alias name='pcie.0'/> </controller> <controller type='pci' index='1' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='1' port='0x8'/> <alias name='pci.1'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' multifunction='on'/> </controller> <controller type='pci' index='2' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='2' port='0x9'/> <alias name='pci.2'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/> </controller> <controller type='pci' index='3' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='3' port='0xa'/> <alias name='pci.3'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/> </controller> <controller type='pci' index='4' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='4' port='0xb'/> <alias name='pci.4'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x3'/> </controller> <controller type='pci' index='5' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='5' port='0xc'/> <alias name='pci.5'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x4'/> </controller> <controller type='pci' index='6' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='6' port='0xd'/> <alias name='pci.6'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x5'/> </controller> <controller type='pci' index='7' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='7' port='0xe'/> <alias name='pci.7'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x6'/> </controller> <controller type='pci' index='8' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='8' port='0xf'/> <alias name='pci.8'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x7'/> </controller> <controller type='pci' index='9' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='9' port='0x10'/> <alias name='pci.9'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/> </controller> <controller type='pci' index='10' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='10' port='0x11'/> <alias name='pci.10'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/> </controller> <controller type='pci' index='11' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='11' port='0x12'/> <alias name='pci.11'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/> </controller> <controller type='pci' index='12' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='12' port='0x13'/> <alias name='pci.12'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/> </controller> <controller type='pci' index='13' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='13' port='0x14'/> <alias name='pci.13'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/> </controller> <controller type='pci' index='14' model='pcie-root-port'> <model name='pcie-root-port'/> <target chassis='14' port='0x15'/> <alias name='pci.14'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/> </controller> <controller type='sata' index='0'> <alias name='ide'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/> </controller> <controller type='virtio-serial' index='0'> <alias name='virtio-serial0'/> <address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/> </controller> <interface type='bridge'> <mac address='52:54:00:a8:6d:9d'/> <source bridge='virbr0'/> <target dev='vnet1'/> <model type='virtio'/> <alias name='net0'/> <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/> </interface> <console type='pty' tty='/dev/pts/5'> <source path='/dev/pts/5'/> <target type='virtio' port='0'/> <alias name='console0'/> </console> <channel type='unix'> <source mode='bind' path='/run/libvirt/qemu/channel/2-g2/org.qemu.guest_agent.0'/> <target type='virtio' name='org.qemu.guest_agent.0' state='connected'/> <alias name='channel0'/> <address type='virtio-serial' controller='0' bus='0' port='1'/> </channel> <input type='mouse' bus='ps2'> <alias name='input0'/> </input> <input type='keyboard' bus='ps2'> <alias name='input1'/> </input> <audio id='1' type='none'/> <watchdog model='itco' action='reset'> <alias name='watchdog0'/> </watchdog> <memballoon model='virtio'> <alias name='balloon0'/> <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/> </memballoon> </devices> <seclabel type='dynamic' model='selinux' relabel='yes'> <label>system_u:system_r:svirt_t:s0:c139,c580</label> <imagelabel>system_u:object_r:svirt_image_t:s0:c139,c580</imagelabel> </seclabel> <seclabel type='dynamic' model='dac' relabel='yes'> <label>+107:+107</label> <imagelabel>+107:+107</imagelabel> </seclabel> </domain>
2.use expect script(vmsh) that call 'virsh cosole' command. To run command through vmsh.
pkt="Ether(src='00:de:ad:00:00:02',dst='00:de:ad:00:00:01')/IPv6(src='2001::1',dst='2001::2')/SCTP(sport=1234,dport=4321)" script_scapy=$(cat <<- EOF from scapy.all import * class MPLS(Packet): name='MPLS' fields_desc=[ BitField('label', 3, 20), BitField('tc', 0, 3), BitField('s', 1, 1), ByteField('ttl', 64) ] bind_layers(Ether, MPLS, type=0x8847) sendp($pkt, iface='lo', loop=1, inter=0.1) EOF ) python="/usr/libexec/platform-python" while ((i<1000)); do echo "== cycle=$i =====================================" vmsh run_cmd g2 "nohup $python -u -c \"$script_scapy\" </dev/null &>/dev/null &" sleep 1 vmsh run_cmd g2 "ps -ef | grep 'scapy' | awk '{system(\"kill \"\$2\"&>/dev/null\")}'" done # sh test.sh ....snipet.... == cycle=6 ===================================== spawn virsh console g2 Connected to domain 'g2' Escape character is ^] (Ctrl + ])localhost login: root Password: Last login: Wed Jul 31 03:24:32 on hvc0 [root@localhost ~]# nohup /usr/libexec/platform-python -u -c "from scapy.all import * > class MPLS(Packet): , 3, 20), BitField('tc', 0, 3), BitField('s', 1, 1), ByteField('ttl', 64) ] bind_layers(Ether, MPLS, type=0x8847) sendp(Ether(src='00:de:ad:00:00:02',dst='00:de:ad:00:00:01')/IPv6(src='2001::1',dst='2001::2')/SCTP(sport=1234,dport=4321), iface='lo', loop=1, inter=0.1)" </dev/null &>/dev/null &
3. check guest virtio-serial thread status.
[root@localhost ~]# ps -ef | grep 713
root 713 1 0 03:19 ? 00:00:00 /usr/bin/qemu-ga --method=virtio-serial --path=/dev/virtio-ports/org.qemu.guest_agent.0 --allow-rpcs=guest-sync-delimited,guest-sync,guest-ping,guest-get-time,guest-set-time,guest-info,guest-shutdown,guest-fsfreeze-status,guest-fsfreeze-freeze,guest-fsfreeze-freeze-list,guest-fsfreeze-thaw,guest-fstrim,guest-suspend-disk,guest-suspend-ram,guest-suspend-hybrid,guest-network-get-interfaces,guest-get-vcpus,guest-set-vcpus,guest-get-disks,guest-get-fsinfo,guest-set-user-password,guest-get-memory-blocks,guest-set-memory-blocks,guest-get-memory-block-info,guest-get-host-name,guest-get-users,guest-get-timezone,guest-get-osinfo,guest-get-devices,guest-ssh-get-authorized-keys,guest-ssh-add-authorized-keys,guest-ssh-remove-authorized-keys,guest-get-diskstats,guest-get-cpustats -F/etc/qemu-ga/fsfreeze-hook
[root@localhost ~]# strace -p 713
strace: Process 713 attached
restart_syscall(<... resuming interrupted read ...>
4. check vmsh thread status on host
[root@dell-per750-03 ~]# ps -aux | grep vmsh root 63758 0.0 0.0 14444 4660 ? Sl 02:31 0:00 /usr/bin/expect /usr/local/bin/vmsh run_cmd g1 nohup /usr/libexec/platform-python -u -c "from scapy.all import * class MPLS(Packet): name='MPLS' fields_desc=[ BitField('label', 3, 20), BitField('tc', 0, 3), BitField('s', 1, 1), ByteField('ttl', 64) ] bind_layers(Ether, MPLS, type=0x8847) sendp(Ether(src='00:de:ad:00:00:02',dst='00:de:ad:00:00:01')/IPv6(src='2001::1',dst='2001::2')/SCTP(sport=1234,dport=4321), iface='$NIC_TEST', loop=1, inter=0.1)" </dev/null &>/dev/null & root 64803 0.0 0.0 14516 4668 pts/6 Sl+ 03:24 0:00 /usr/bin/expect /usr/local/bin/vmsh run_cmd g2 nohup /usr/libexec/platform-python -u -c "from scapy.all import * class MPLS(Packet): name='MPLS' fields_desc=[ BitField('label', 3, 20), BitField('tc', 0, 3), BitField('s', 1, 1), ByteField('ttl', 64) ] bind_layers(Ether, MPLS, type=0x8847) sendp(Ether(src='00:de:ad:00:00:02',dst='00:de:ad:00:00:01')/IPv6(src='2001::1',dst='2001::2')/SCTP(sport=1234,dport=4321), iface='lo', loop=1, inter=0.1)" </dev/null &>/dev/null & root 64887 0.0 0.0 6360 1920 pts/4 S+ 03:30 0:00 grep --color=auto vmsh [root@dell-per750-03 ~]# strace -p 63758 strace: Process 63758 attached restart_syscall(<... resuming interrupted read ...>^Cstrace: Process 63758 detached <detached ...>[root@dell-per750-03 ~]# strace -p 64803 strace: Process 64803 attached restart_syscall(<... resuming interrupted read ...>
Expected results
Actual results
1.virsh console hung
2.guest systemd print failed to lock /dev.console
[root@localhost ~]# journalctl -u systemd-vconsole-setup.service -b Jul 31 00:12:28 localhost systemd-vconsole-setup[328]: Failed to lock /dev/console, proceeding without lock: Input/output error Jul 31 00:12:28 localhost systemd-vconsole-setup[328]: /usr/bin/setfont failed with a "system error" (EX_OSERR), ignoring. Jul 31 00:12:28 localhost systemd-vconsole-setup[334]: setfont: ERROR kdfontop.c:211 put_font_kdfontop: ioctl(KDFONTOP): Invalid argument Jul 31 00:12:28 localhost systemd-vconsole-setup[328]: Setting source virtual console failed, ignoring remaining ones. Jul 31 00:12:28 localhost systemd[1]: Finished systemd-vconsole-setup.service - Virtual Console Setup. Jul 31 00:12:29 localhost systemd[1]: systemd-vconsole-setup.service: Deactivated successfully. Jul 31 00:12:29 localhost systemd[1]: Stopped systemd-vconsole-setup.service - Virtual Console Setup. Jul 31 00:12:29 localhost systemd[1]: Stopping systemd-vconsole-setup.service - Virtual Console Setup... Jul 31 00:12:29 localhost systemd[1]: Starting systemd-vconsole-setup.service - Virtual Console Setup... Jul 31 00:12:29 localhost systemd-vconsole-setup[448]: Failed to lock /dev/console, proceeding without lock: Input/output error Jul 31 00:12:29 localhost systemd-vconsole-setup[467]: setfont: ERROR kdfontop.c:211 put_font_kdfontop: ioctl(KDFONTOP): Invalid argument Jul 31 00:12:29 localhost systemd-vconsole-setup[448]: /usr/bin/setfont failed with a "system error" (EX_OSERR), ignoring. Jul 31 00:12:29 localhost systemd-vconsole-setup[448]: Setting source virtual console failed, ignoring remaining ones. Jul 31 00:12:29 localhost systemd[1]: Finished systemd-vconsole-setup.service - Virtual Console Setup. Jul 31 00:12:34 localhost systemd[1]: systemd-vconsole-setup.service: Deactivated successfully. Jul 31 00:12:34 localhost systemd[1]: Stopped systemd-vconsole-setup.service - Virtual Console Setup. Jul 31 00:12:34 localhost systemd[1]: Stopping systemd-vconsole-setup.service - Virtual Console Setup... Jul 31 00:12:34 localhost systemd[1]: Starting systemd-vconsole-setup.service - Virtual Console Setup... Jul 31 00:12:34 localhost systemd-vconsole-setup[651]: Failed to lock /dev/console, proceeding without lock: Input/output error Jul 31 00:12:34 localhost systemd-vconsole-setup[656]: setfont: ERROR kdfontop.c:211 put_font_kdfontop: ioctl(KDFONTOP): Invalid argument Jul 31 00:12:34 localhost systemd-vconsole-setup[651]: /usr/bin/setfont failed with a "system error" (EX_OSERR), ignoring. Jul 31 00:12:34 localhost systemd-vconsole-setup[651]: Setting source virtual console failed, ignoring remaining ones. Jul 31 00:12:34 localhost systemd[1]: Finished systemd-vconsole-setup.service - Virtual Console Setup.