This epic tracks all the effort needed to deliver the solution related to the bug described below.
[Imported from FD-2589]

Description of problem:
virsh start guest hung when running ovs dpdk pvp case with viommu mode on rhe8.6

Version-Release number of selected component (if applicable):
kernel 4.18.0-372.39.1.el8_6.x86_64
[root@dell-per740-87 perf]# rpm -qa|grep openvswitch
kernel-kernel-networking-openvswitch-common-2.0-279.noarch
openvswitch2.17-2.17.0-64.el8fdp.x86_64
kernel-kernel-networking-openvswitch-perf-1.0-370.noarch
openvswitch-selinux-extra-policy-1.0-29.el8fdp.noarch
[root@dell-per740-87 perf]# rpm -qa|grep libvirt
libvirt-daemon-driver-storage-iscsi-direct-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-secret-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-config-nwfilter-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-glib-3.0.0-1.el8.x86_64
libvirt-daemon-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-storage-rbd-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-libs-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-storage-gluster-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-storage-mpath-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-interface-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-network-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-qemu-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-client-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-storage-disk-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-storage-logical-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-storage-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
python3-libvirt-8.0.0-1.1.module+el8.6.0+16381+3abc475c.x86_64
libvirt-gconfig-3.0.0-1.el8.x86_64
libvirt-gobject-3.0.0-1.el8.x86_64
libvirt-daemon-driver-storage-core-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-storage-scsi-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-nodedev-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-kvm-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-storage-iscsi-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-driver-nwfilter-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64
libvirt-daemon-config-network-8.0.0-5.5.module+el8.6.0+16828+96e76c36.x86_64

How reproducible:

Steps to Reproduce:
1.configure hugepage and isolcpu as follows
[root@dell-per740-87 perf]# cat /proc/cmdline
BOOT_IMAGE=(hd0,msdos1)/vmlinuz-4.18.0-372.39.1.el8_6.x86_64 root=/dev/mapper/rhel_dell-per74087-root ro ksdevice=bootif crashkernel=auto resume=/dev/mapper/rhel_dellper740-87-swap rd.lvm.lv=rhel_dell-per740-87/root rd.lvm.lv=rhel_dell-per740-87/swap console=ttyS0,115200n81 skew_tick=1 nohz=on nohz_full=3,27,5,29,7,31,9,33,11,35,13,37,15,39,17,41,19,43,21,45,23,47 rcu_nocbs=3,27,5,29,7,31,9,33,11,35,13,37,15,39,17,41,19,43,21,45,23,47 tuned.non_isolcpus=00005555,57555557 intel_pstate=disable nosoftlockup default_hugepagesz=1G hugepagesz=1G hugepages=24 isolcpus=3,27,5,29,7,31,9,33,11,35,13,37,15,39,17,41,19,43,21,45,23,47 intel_iommu=on iommu=pt intel_idle.max_cstate=0 processor.max_cstate=0 intel_pstate=disable pci=realloc

2. Run ovs dpdk pvp case with viommu mode
[root@dell-per740-87 perf]# ovs-vsctl set Open_vSwitch . other_config={}
[root@dell-per740-87 perf]# ovs-vsctl set Open_vSwitch . other_config:dpdk-socket-mem=0,4096
[root@dell-per740-87 perf]# ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=aa0000aa0000
[root@dell-per740-87 perf]# ovs-vsctl set Open_vSwitch . other_config:dpdk-lcore-mask=0x1
[root@dell-per740-87 perf]# ovs-vsctl set Open_vSwitch . other_config:vhost-iommu-support=true
[root@dell-per740-87 perf]# ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true
[root@dell-per740-87 perf]# systemctl restart openvswitch
[root@dell-per740-87 perf]# ovs-vsctl add-br br0 – set bridge br0 datapath_type=netdev
[root@dell-per740-87 perf]# ovs-vsctl add-port br0 dpdk0 – set Interface dpdk0 type=dpdk options:dpdk-devargs=0000:3b:00.0
[root@dell-per740-87 perf]# ovs-vsctl add-port br0 dpdk1 – set Interface dpdk1 type=dpdk options:dpdk-devargs=0000:3b:00.1
[root@dell-per740-87 perf]# systemctl restart openvswitch
[root@dell-per740-87 perf]# ovs-vsctl get Open_vSwitch . other_config

3. start guest with following xml
[root@dell-per740-87 perf]# virsh dumpxml g1
<domain type='kvm' id='1'>
<name>g1</name>
<uuid>4304eeb7-bdad-4c7a-a33a-87f46f8a2e87</uuid>
<memory unit='KiB'>8388608</memory>
<currentMemory unit='KiB'>8388608</currentMemory>
<memoryBacking>
<hugepages>
<page size='1048576' unit='KiB'/>
</hugepages>
<locked/>
<access mode='shared'/>
</memoryBacking>
<vcpu placement='static'>9</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='11'/>
<vcpupin vcpu='1' cpuset='33'/>
<vcpupin vcpu='2' cpuset='9'/>
<vcpupin vcpu='3' cpuset='31'/>
<vcpupin vcpu='4' cpuset='7'/>
<vcpupin vcpu='5' cpuset='29'/>
<vcpupin vcpu='6' cpuset='5'/>
<vcpupin vcpu='7' cpuset='27'/>
<vcpupin vcpu='8' cpuset='3'/>
<emulatorpin cpuset='1,25'/>
</cputune>
<numatune>
<memory mode='strict' nodeset='0'/>
</numatune>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='x86_64' machine='pc-q35-rhel8.6.0'>hvm</type>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<apic/>
<pmu state='off'/>
<vmport state='off'/>
<ioapic driver='qemu'/>
</features>
<cpu mode='host-passthrough' check='none' migratable='on'>
<feature policy='require' name='tsc-deadline'/>
<numa>
<cell id='0' cpus='0-8' memory='8388608' unit='KiB' memAccess='shared'/>
</numa>
</cpu>
<clock offset='utc'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/libexec/qemu-kvm</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/var/lib/libvirt/images/g1.qcow2' index='1'/>
<backingStore/>
<target dev='vda' bus='virtio'/>
<alias name='virtio-disk0'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</disk>
<controller type='usb' index='0' model='none'>
<alias name='usb'/>
</controller>
<controller type='pci' index='0' model='pcie-root'>
<alias name='pcie.0'/>
</controller>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x10'/>
<alias name='pci.1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x11'/>
<alias name='pci.2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x8'/>
<alias name='pci.3'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0x9'/>
<alias name='pci.4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0xa'/>
<alias name='pci.5'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0xb'/>
<alias name='pci.6'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
</controller>
<controller type='sata' index='0'>
<alias name='ide'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
</controller>
<interface type='bridge'>
<mac address='52:54:00:01:02:03'/>
<source bridge='virbr0'/>
<target dev='vnet0'/>
<model type='virtio'/>
<alias name='net0'/>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</interface>
<interface type='vhostuser'>
<mac address='00:de:ad:00:00:01'/>
<source type='unix' path='/tmp/vhostuser/vhost0' mode='server'/>
<target dev='vhost0'/>
<model type='virtio'/>
<driver name='vhost' queues='4' rx_queue_size='1024' tx_queue_size='1024' iommu='on' ats='on'>
<host mrg_rxbuf='off'/>
</driver>
<alias name='net1'/>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</interface>
<interface type='vhostuser'>
<mac address='00:de:ad:00:00:02'/>
<source type='unix' path='/tmp/vhostuser/vhost1' mode='server'/>
<target dev='vhost1'/>
<model type='virtio'/>
<driver name='vhost' queues='4' rx_queue_size='1024' tx_queue_size='1024' iommu='on' ats='on'>
<host mrg_rxbuf='off'/>
</driver>
<alias name='net2'/>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
</interface>
<serial type='pty'>
<source path='/dev/pts/1'/>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
<alias name='serial0'/>
</serial>
<console type='pty' tty='/dev/pts/1'>
<source path='/dev/pts/1'/>
<target type='serial' port='0'/>
<alias name='serial0'/>
</console>
<input type='mouse' bus='ps2'>
<alias name='input0'/>
</input>
<input type='keyboard' bus='ps2'>
<alias name='input1'/>
</input>
<graphics type='vnc' port='5900' autoport='yes' listen='0.0.0.0'>
<listen type='address' address='0.0.0.0'/>
</graphics>
<audio id='1' type='none'/>
<video>
<model type='cirrus' vram='16384' heads='1' primary='yes'/>
<alias name='video0'/>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
</video>
<memballoon model='virtio'>
<alias name='balloon0'/>
<address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
</memballoon>
<iommu model='intel'>
<driver intremap='on' caching_mode='on' iotlb='on'/>
</iommu>
</devices>
<seclabel type='dynamic' model='selinux' relabel='yes'>
<label>system_u:system_r:svirt_t:s0:c419,c605</label>
<imagelabel>system_u:object_r:svirt_image_t:s0:c419,c605</imagelabel>
</seclabel>
<seclabel type='dynamic' model='dac' relabel='yes'>
<label>+107:+1001</label>
<imagelabel>+107:+1001</imagelabel>
</seclabel>
</domain>

Actual results:
Start guest with hung.
[root@dell-per740-87 perf]# virsh list --all
Id Name State
----------------------
1 g1 running

[root@dell-per740-87 perf]# virsh console g1
Connected to domain 'g1'
Escape character is ^] (Ctrl + ])

The guest can work well with no viommu mode.

Expected results:
The guest can work well with viommu mode.

Additional info:
The attachment is libvirt log, ovs log and sos report.
Driver is ice:
[root@dell-per740-87 perf]# lspci|grep E810-C
af:00.0 Ethernet controller: Intel Corporation Ethernet Controller E810-C for SFP (rev 02)
af:00.1 Ethernet controller: Intel Corporation Ethernet Controller E810-C for SFP (rev 02)
af:00.2 Ethernet controller: Intel Corporation Ethernet Controller E810-C for SFP (rev 02)
af:00.3 Ethernet controller: Intel Corporation Ethernet Controller E810-C for SFP (rev 02)
job link: https://beaker.engineering.redhat.com/jobs/7387683