Description of problem:
i40e card: nic partiion performance case run failed on rhel9.2 ctc2 version

Version-Release number of selected component (if applicable):
[root@dell-per730-52 ~]# uname -r
5.14.0-234.el9.x86_64
[root@dell-per730-52 ~]# rpm -qa|grep openvswitch
openvswitch-selinux-extra-policy-1.0-31.el9fdp.noarch
openvswitch3.0-3.0.0-27.el9fdp.x86_64

How reproducible:

Steps to Reproduce:
DUT i40e card <connect> Trex i40e card
1. Create two vfs for pf,
echo 2 > /sys/devices/pci0000:00/0000:00:03.2/0000:07:00.0/sriov_numvfs
set the trust to on
[root@dell-per730-52 ~]# ip link show
enp7s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
link/ether 3c:fd:fe:ad:bc:e8 brd ff:ff:ff:ff:ff:ff
vf 0 link/ether 00:60:2f:20:0c:36 brd ff:ff:ff:ff:ff:ff, spoof checking on, link-state auto, trust on
vf 1 link/ether 52:54:00:11:8f:ea brd ff:ff:ff:ff:ff:ff, spoof checking on, link-state auto, trust on

2. Bind vf1 to dpdk and add it ovs bridge, add dpdkvhostuserclient0 to ovs bridge.
driverctl -v set-override 0000:07:02.1 vfio-pci
ovs-vsctl set Open_vSwitch . 'other_config={}'
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=1024,1024
ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x50000005000000
ovs-vsctl --no-wait set Open_vSwitch . other_config:vhost-iommu-support=true
ovs-vsctl add-br ovsbr0 – set bridge ovsbr0 datapath_type=netdev
ovs-vsctl add-port ovsbr0 dpdk0 – set Interface dpdk0 type=dpdk type=dpdk options:dpdk-devargs=0000:07:02.1
ovs-vsctl add-port ovsbr0 dpdkvhostuserclient0 – set Interface dpdkvhostuserclient0 type=dpdkvhostuserclient – set Interface dpdkvhostuserclient0 options:vhost-server-path=/tmp/dpdkvhostuserclient0

3. Put vf0 directly pass through to master1
4. Start master2 with dpdkvhostuserclient0
[root@dell-per730-52 ~]# virsh list --all
Id Name State
-------------------------
1 master1 running
3 master2 running

5. Start testpmd inside master1
dpdk-testpmd -l 0,1,2 -n4 --socket-mem 1024 --legacy-mem – --burst=64 -i --rxd=512 --txd=512 --nb-cores=2 --rxq=1 --txq=1 --disable-rss --forward-mode=macswap --auto-start

6. Start testpmd inside master2
[root@localhost ~]# dpdk-testpmd -l 0,1,2 -n4 --socket-mem 1024 --legacy-mem – --burst=64 -i --rxd=512 --txd=512 --nb-cores=2 --rxq=1 --txq=1 --disable-rss --forward-mode=macswap --auto-start
EAL: Detected CPU lcores: 3
EAL: Detected NUMA nodes: 1
EAL: Detected shared linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'VA'
EAL: No available 2048 kB hugepages reported
EAL: VFIO support initialized
EAL: Probe PCI driver: net_virtio (1af4:1041) device: 0000:04:00.0 (socket 0)
EAL: Using IOMMU type 1 (Type 1)
TELEMETRY: No legacy callbacks, legacy socket not created
Interactive-mode selected
Set macswap packet forwarding mode
Auto-start selected
testpmd: create a new mbuf pool <mb_pool_0>: n=163456, size=2176, socket=0
testpmd: preferred mempool ops selected: ring_mp_mc

Warning! port-topology=paired and odd forward ports number, the last port will pair with itself.

Configuring Port 0 (socket 0)
EAL: Error disabling MSI-X interrupts for fd 24
Port 0: 52:54:00:11:8F:EA
Checking link statuses...
Done
Start automatic packet forwarding
macswap packet forwarding - ports=1 - cores=1 - streams=1 - NUMA support enabled, MP allocation mode: native
Logical Core 1 (socket 0) forwards packets on 1 streams:
RX P=0/Q=0 (socket 0) -> TX P=0/Q=0 (socket 0) peer=02:00:00:00:00:00

macswap packet forwarding packets/burst=64
nb forwarding cores=2 - nb forwarding ports=1
port 0: RX queue number: 1 Tx queue number: 1
Rx offloads=0x0 Tx offloads=0x0
RX queue: 0
RX desc=512 - RX free threshold=0
RX threshold registers: pthresh=0 hthresh=0 wthresh=0
RX Offloads=0x0
TX queue: 0
TX desc=512 - TX free threshold=0
TX threshold registers: pthresh=0 hthresh=0 wthresh=0
TX offloads=0x0 - TX RS bit threshold=0
testpmd>

7. Used Trex to end traffic, dst mac is master1's mac, it work well.
python3 /mnt/tests/kernel/networking/fd_nic_partition/no-bond/send.py --pkt_size=64 -c 127.0.0.1 -s 3c:fd:fe:ad:bf:c4 -d 00:60:2F:20:0C:36 --speed=14000000

8. Used Trex to send traffic, dst mac is master2's dpdkvhostuserclient0 mac,
python3 /mnt/tests/kernel/networking/fd_nic_partition/no-bond/send.py --pkt_size=64 -c 127.0.0.1 -s 3c:fd:fe:ad:bf:c4 -d 52:54:00:11:8f:ea --speed=14000000

Actual results:
Step 6 does not work well, it got 0 performance result.
The testpmd cannot got packet inside master2.
testpmd> stop
Telling cores to stop...
Waiting for lcores to finish...

---------------------- Forward statistics for port 0 ----------------------
RX-packets: 0 RX-dropped: 0 RX-total: 0
TX-packets: 0 TX-dropped: 0 TX-total: 0
----------------------------------------------------------------------------

+++++++++++++++ Accumulated forward statistics for all ports+++++++++++++++
RX-packets: 0 RX-dropped: 0 RX-total: 0
TX-packets: 0 TX-dropped: 0 TX-total: 0
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Done.

Expected results:
The testpmd can receive packet and it can got normal performance.
When changed the host from RHEL-9.2.0-20230115.7 to RHEL-9.2.0-20221122.2, it work well.

Additional info:
RHEL-9.2.0-20230115.7 job:
https://beaker.engineering.redhat.com/jobs/7507201
RHEL-9.2.0-20221122.2 job:
https://beaker.engineering.redhat.com/jobs/7506485

master1 xml:
[root@dell-per730-52 ~]# virsh dumpxml master1
<domain type='kvm' id='1'>
<name>master1</name>
<uuid>37425e76-af6a-44a6-aba0-73434afe34c1</uuid>
<memory unit='KiB'>5242880</memory>
<currentMemory unit='KiB'>5242880</currentMemory>
<memoryBacking>
<hugepages>
<page size='1048576' unit='KiB'/>
</hugepages>
<access mode='shared'/>
</memoryBacking>
<vcpu placement='static'>3</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='2'/>
<vcpupin vcpu='1' cpuset='4'/>
<vcpupin vcpu='2' cpuset='32'/>
<emulatorpin cpuset='2'/>
</cputune>
<numatune>
<memory mode='strict' nodeset='0'/>
</numatune>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='x86_64' machine='pc-q35-rhel9.2.0'>hvm</type>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<apic/>
<pmu state='off'/>
<vmport state='off'/>
<ioapic driver='qemu'/>
</features>
<cpu mode='host-passthrough' check='none' migratable='on'>
<feature policy='require' name='tsc-deadline'/>
<numa>
<cell id='0' cpus='0-2' memory='5242880' unit='KiB' memAccess='shared'/>
</numa>
</cpu>
<clock offset='utc'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/libexec/qemu-kvm</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/var/lib/libvirt/images/master1.qcow2' index='1'/>
<backingStore/>
<target dev='vda' bus='virtio'/>
<alias name='virtio-disk0'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</disk>
<controller type='usb' index='0' model='none'/>
<controller type='pci' index='0' model='pcie-root'>
<alias name='pcie.0'/>
</controller>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x10'/>
<alias name='pci.1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x11'/>
<alias name='pci.2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x8'/>
<alias name='pci.3'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0x9'/>
<alias name='pci.4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0xa'/>
<alias name='pci.5'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0xb'/>
<alias name='pci.6'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
</controller>
<controller type='sata' index='0'>
<alias name='ide'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
</controller>
<interface type='bridge'>
<mac address='52:54:00:bb:63:7b'/>
<source bridge='virbr0'/>
<target dev='vnet0'/>
<model type='virtio'/>
<alias name='net0'/>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</interface>
<interface type='hostdev' managed='yes'>
<mac address='00:60:2f:20:0c:36'/>
<driver name='vfio'/>
<source>
<address type='pci' domain='0x0000' bus='0x07' slot='0x02' function='0x0'/>
</source>
<alias name='hostdev0'/>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</interface>
<serial type='pty'>
<source path='/dev/pts/0'/>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
<alias name='serial0'/>
</serial>
<console type='pty' tty='/dev/pts/0'>
<source path='/dev/pts/0'/>
<target type='serial' port='0'/>
<alias name='serial0'/>
</console>
<input type='mouse' bus='ps2'>
<alias name='input0'/>
</input>
<input type='keyboard' bus='ps2'>
<alias name='input1'/>
</input>
<graphics type='vnc' port='5900' autoport='yes' listen='0.0.0.0'>
<listen type='address' address='0.0.0.0'/>
</graphics>
<audio id='1' type='none'/>
<video>
<model type='cirrus' vram='16384' heads='1' primary='yes'/>
<alias name='video0'/>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
</video>
<memballoon model='virtio'>
<alias name='balloon0'/>
<address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
</memballoon>
<iommu model='intel'>
<driver intremap='on' caching_mode='on' iotlb='on'/>
<alias name='iommu0'/>
</iommu>
</devices>
<seclabel type='dynamic' model='selinux' relabel='yes'>
<label>system_u:system_r:svirt_t:s0:c32,c476</label>
<imagelabel>system_u:object_r:svirt_image_t:s0:c32,c476</imagelabel>
</seclabel>
<seclabel type='dynamic' model='dac' relabel='yes'>
<label>+107:+1001</label>
<imagelabel>+107:+1001</imagelabel>
</seclabel>
</domain>

master2 xml:
[root@dell-per730-52 ~]# virsh dumpxml master2
<domain type='kvm' id='4'>
<name>master2</name>
<uuid>37425e76-af6a-44a6-aba0-73434afe34c0</uuid>
<memory unit='KiB'>6291456</memory>
<currentMemory unit='KiB'>6291456</currentMemory>
<memoryBacking>
<hugepages>
<page size='1048576' unit='KiB'/>
</hugepages>
<access mode='shared'/>
</memoryBacking>
<vcpu placement='static'>3</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='30'/>
<vcpupin vcpu='1' cpuset='6'/>
<vcpupin vcpu='2' cpuset='34'/>
<emulatorpin cpuset='30'/>
</cputune>
<numatune>
<memory mode='strict' nodeset='0'/>
</numatune>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='x86_64' machine='pc-q35-rhel9.2.0'>hvm</type>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<apic/>
<pmu state='off'/>
<vmport state='off'/>
<ioapic driver='qemu'/>
</features>
<cpu mode='host-passthrough' check='none' migratable='on'>
<feature policy='require' name='tsc-deadline'/>
<numa>
<cell id='0' cpus='0-2' memory='6291456' unit='KiB' memAccess='shared'/>
</numa>
</cpu>
<clock offset='utc'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/libexec/qemu-kvm</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/var/lib/libvirt/images/master2.qcow2' index='1'/>
<backingStore/>
<target dev='vda' bus='virtio'/>
<alias name='virtio-disk0'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</disk>
<controller type='usb' index='0' model='none'>
<alias name='usb'/>
</controller>
<controller type='pci' index='0' model='pcie-root'>
<alias name='pcie.0'/>
</controller>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x10'/>
<alias name='pci.1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x11'/>
<alias name='pci.2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x8'/>
<alias name='pci.3'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0x9'/>
<alias name='pci.4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0xa'/>
<alias name='pci.5'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0xb'/>
<alias name='pci.6'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
</controller>
<controller type='sata' index='0'>
<alias name='ide'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
</controller>
<interface type='vhostuser'>
<mac address='52:54:00:11:8f:ea'/>
<source type='unix' path='/tmp/dpdkvhostuserclient0' mode='server'/>
<target dev='dpdkvhostuserclient0'/>
<model type='virtio'/>
<driver name='vhost' iommu='on' ats='on'/>
<alias name='net0'/>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
</interface>
<serial type='pty'>
<source path='/dev/pts/1'/>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
<alias name='serial0'/>
</serial>
<console type='pty' tty='/dev/pts/1'>
<source path='/dev/pts/1'/>
<target type='serial' port='0'/>
<alias name='serial0'/>
</console>
<input type='mouse' bus='ps2'>
<alias name='input0'/>
</input>
<input type='keyboard' bus='ps2'>
<alias name='input1'/>
</input>
<graphics type='vnc' port='5901' autoport='yes' listen='0.0.0.0'>
<listen type='address' address='0.0.0.0'/>
</graphics>
<audio id='1' type='none'/>
<video>
<model type='cirrus' vram='16384' heads='1' primary='yes'/>
<alias name='video0'/>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
</video>
<memballoon model='virtio'>
<alias name='balloon0'/>
<address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
</memballoon>
<iommu model='intel'>
<driver intremap='on' caching_mode='on' iotlb='on'/>
<alias name='iommu0'/>
</iommu>
</devices>
<seclabel type='dynamic' model='selinux' relabel='yes'>
<label>system_u:system_r:svirt_t:s0:c414,c649</label>
<imagelabel>system_u:object_r:svirt_image_t:s0:c414,c649</imagelabel>
</seclabel>
<seclabel type='dynamic' model='dac' relabel='yes'>
<label>+107:+1001</label>
<imagelabel>+107:+1001</imagelabel>
</seclabel>
</domain>