2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest [req-c930949f-8b57-4d34-bbc7-8ed3b41c4990 f3c223c7ad8009b3253daae5c72a4244ec61ddafa2a362e32b3e1e9d1900f85d f2fdac2710674852ae99385620cdbff3 - 8cf3016a6a3f4b3197fb230609c225f7 8cf3016a6a3f4b3197fb230609c225f7] Error launching a defined domain with XML: <domain type='kvm'>
  <name>instance-0027c624</name>
  <uuid>909eed8f-428d-4af9-940d-467b59497b55</uuid>
  <metadata>
    <nova:instance xmlns:nova="http://openstack.org/xmlns/libvirt/nova/1.1">
      <nova:package version="23.2.3-17.1.20231018130821.2ace99d.el9ost"/>
      <nova:name>vlb21_FWvLB0031</nova:name>
      <nova:creationTime>2024-06-28 11:06:45</nova:creationTime>
      <nova:flavor name="vlb21_loadenv_ForwardingVLB_Node">
        <nova:memory>49152</nova:memory>
        <nova:disk>140</nova:disk>
        <nova:swap>0</nova:swap>
        <nova:ephemeral>0</nova:ephemeral>
        <nova:vcpus>16</nova:vcpus>
      </nova:flavor>
      <nova:owner>
        <nova:user uuid="f3c223c7ad8009b3253daae5c72a4244ec61ddafa2a362e32b3e1e9d1900f85d">svc-owm-rhos-load9</nova:user>
        <nova:project uuid="f2fdac2710674852ae99385620cdbff3">tm-dev-load1</nova:project>
      </nova:owner>
      <nova:root type="image" uuid="7996fdf3-4bd0-4949-954a-4f9c06830b94"/>
      <nova:ports>
        <nova:port uuid="cec35a2a-fe4c-4775-99e2-f195dc002a09">
          <nova:ip type="fixed" address="192.168.200.61" ipVersion="4"/>
        </nova:port>
        <nova:port uuid="5cb4a025-35a2-48c9-add2-2c0814ba7ae8">
          <nova:ip type="fixed" address="192.168.201.61" ipVersion="4"/>
        </nova:port>
        <nova:port uuid="ed82fb12-3e94-49e1-adb4-ccef9400b373">
          <nova:ip type="fixed" address="192.168.210.61" ipVersion="4"/>
        </nova:port>
        <nova:port uuid="e069eef7-c369-4d58-bc37-5a90e9117d00">
          <nova:ip type="fixed" address="192.168.202.61" ipVersion="4"/>
        </nova:port>
        <nova:port uuid="41319aaf-96c8-491a-a0e0-ed71f9d2dc3d">
          <nova:ip type="fixed" address="192.168.203.61" ipVersion="4"/>
        </nova:port>
        <nova:port uuid="932384b8-d560-4718-a5b9-00c1cd37df67">
          <nova:ip type="fixed" address="192.168.211.61" ipVersion="4"/>
        </nova:port>
        <nova:port uuid="06dc8a2c-a3f2-4214-9328-5b8f2cff0a32">
          <nova:ip type="fixed" address="192.168.204.61" ipVersion="4"/>
        </nova:port>
        <nova:port uuid="cc1c38ee-27ea-4af3-b1e4-5dc265fd5bd4">
          <nova:ip type="fixed" address="192.168.205.61" ipVersion="4"/>
        </nova:port>
        <nova:port uuid="9646f23d-5467-496f-9ec8-e59450284c61">
          <nova:ip type="fixed" address="192.168.206.61" ipVersion="4"/>
        </nova:port>
      </nova:ports>
    </nova:instance>
  </metadata>
  <memory unit='KiB'>50331648</memory>
  <currentMemory unit='KiB'>50331648</currentMemory>
  <memoryBacking>
    <hugepages>
      <page size='1048576' unit='KiB' nodeset='0'/>
    </hugepages>
  </memoryBacking>
  <vcpu placement='static'>16</vcpu>
  <cputune>
    <vcpupin vcpu='0' cpuset='2'/>
    <vcpupin vcpu='1' cpuset='42'/>
    <vcpupin vcpu='2' cpuset='7'/>
    <vcpupin vcpu='3' cpuset='47'/>
    <vcpupin vcpu='4' cpuset='3'/>
    <vcpupin vcpu='5' cpuset='43'/>
    <vcpupin vcpu='6' cpuset='16'/>
    <vcpupin vcpu='7' cpuset='56'/>
    <vcpupin vcpu='8' cpuset='12'/>
    <vcpupin vcpu='9' cpuset='52'/>
    <vcpupin vcpu='10' cpuset='13'/>
    <vcpupin vcpu='11' cpuset='53'/>
    <vcpupin vcpu='12' cpuset='8'/>
    <vcpupin vcpu='13' cpuset='48'/>
    <vcpupin vcpu='14' cpuset='17'/>
    <vcpupin vcpu='15' cpuset='57'/>
    <emulatorpin cpuset='2-3,7-8,12-13,16-17,42-43,47-48,52-53,56-57'/>
  </cputune>
  <numatune>
    <memory mode='strict' nodeset='0'/>
    <memnode cellid='0' mode='strict' nodeset='0'/>
  </numatune>
  <sysinfo type='smbios'>
    <system>
      <entry name='manufacturer'>Red Hat</entry>
      <entry name='product'>OpenStack Compute</entry>
      <entry name='version'>23.2.3-17.1.20231018130821.2ace99d.el9ost</entry>
      <entry name='serial'>909eed8f-428d-4af9-940d-467b59497b55</entry>
      <entry name='uuid'>909eed8f-428d-4af9-940d-467b59497b55</entry>
      <entry name='family'>Virtual Machine</entry>
    </system>
  </sysinfo>
  <os>
    <type arch='x86_64' machine='pc-q35-rhel9.2.0'>hvm</type>
    <boot dev='hd'/>
    <smbios mode='sysinfo'/>
  </os>
  <features>
    <acpi/>
    <apic/>
  </features>
  <cpu mode='host-model' check='partial'>
    <topology sockets='1' dies='1' cores='8' threads='2'/>
    <numa>
      <cell id='0' cpus='0-15' memory='50331648' unit='KiB' memAccess='shared'/>
    </numa>
  </cpu>
  <clock offset='utc'>
    <timer name='pit' tickpolicy='delay'/>
    <timer name='rtc' tickpolicy='catchup'/>
    <timer name='hpet' present='no'/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <devices>
    <emulator>/usr/libexec/qemu-kvm</emulator>
    <disk type='block' device='disk'>
      <driver name='qemu' type='raw' cache='none' io='native'/>
      <source dev='/dev/vm_disks/909eed8f-428d-4af9-940d-467b59497b55_disk'/>
      <target dev='vda' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
    </disk>
    <disk type='file' device='cdrom'>
      <driver name='qemu' type='raw' cache='none'/>
      <source file='/var/lib/nova/instances/909eed8f-428d-4af9-940d-467b59497b55/disk.config'/>
      <target dev='sda' bus='sata'/>
      <readonly/>
      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
    </disk>
    <controller type='pci' index='0' model='pcie-root'/>
    <controller type='pci' index='1' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='1' port='0x10'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
    </controller>
    <controller type='pci' index='2' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='2' port='0x11'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
    </controller>
    <controller type='pci' index='3' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='3' port='0x12'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
    </controller>
    <controller type='pci' index='4' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='4' port='0x13'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
    </controller>
    <controller type='pci' index='5' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='5' port='0x14'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
    </controller>
    <controller type='pci' index='6' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='6' port='0x15'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
    </controller>
    <controller type='pci' index='7' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='7' port='0x16'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x6'/>
    </controller>
    <controller type='pci' index='8' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='8' port='0x17'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x7'/>
    </controller>
    <controller type='pci' index='9' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='9' port='0x18'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0' multifunction='on'/>
    </controller>
    <controller type='pci' index='10' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='10' port='0x19'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x1'/>
    </controller>
    <controller type='pci' index='11' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='11' port='0x1a'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x2'/>
    </controller>
    <controller type='pci' index='12' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='12' port='0x1b'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x3'/>
    </controller>
    <controller type='pci' index='13' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='13' port='0x1c'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x4'/>
    </controller>
    <controller type='pci' index='14' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='14' port='0x1d'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x5'/>
    </controller>
    <controller type='pci' index='15' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='15' port='0x1e'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x6'/>
    </controller>
    <controller type='pci' index='16' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='16' port='0x1f'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x7'/>
    </controller>
    <controller type='pci' index='17' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='17' port='0x20'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </controller>
    <controller type='pci' index='18' model='pcie-to-pci-bridge'>
      <model name='pcie-pci-bridge'/>
      <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
    </controller>
    <controller type='usb' index='0' model='piix3-uhci'>
      <address type='pci' domain='0x0000' bus='0x12' slot='0x01' function='0x0'/>
    </controller>
    <controller type='sata' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
    </controller>
    <interface type='hostdev' managed='yes'>
      <mac address='fa:16:3e:c2:df:fa'/>
      <source>
        <address type='pci' domain='0x0000' bus='0x20' slot='0x05' function='0x6'/>
      </source>
      <vlan>
        <tag id='2208'/>
      </vlan>
      <address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
    </interface>
    <interface type='hostdev' managed='yes'>
      <mac address='fa:16:3e:49:39:c5'/>
      <source>
        <address type='pci' domain='0x0000' bus='0x20' slot='0x05' function='0x5'/>
      </source>
      <vlan>
        <tag id='2330'/>
      </vlan>
      <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
    </interface>
    <interface type='hostdev' managed='yes'>
      <mac address='fa:16:3e:fc:44:e2'/>
      <source>
        <address type='pci' domain='0x0000' bus='0x20' slot='0x05' function='0x4'/>
      </source>
      <vlan>
        <tag id='2239'/>
      </vlan>
      <address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
    </interface>
    <interface type='hostdev' managed='yes'>
      <mac address='fa:16:3e:ae:21:bb'/>
      <source>
        <address type='pci' domain='0x0000' bus='0x20' slot='0x05' function='0x3'/>
      </source>
      <vlan>
        <tag id='2382'/>
      </vlan>
      <address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
    </interface>
    <interface type='hostdev' managed='yes'>
      <mac address='fa:16:3e:43:d2:4f'/>
      <source>
        <address type='pci' domain='0x0000' bus='0x20' slot='0x05' function='0x2'/>
      </source>
      <vlan>
        <tag id='2100'/>
      </vlan>
      <address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
    </interface>
    <interface type='hostdev' managed='yes'>
      <mac address='fa:16:3e:56:88:23'/>
      <source>
        <address type='pci' domain='0x0000' bus='0x20' slot='0x05' function='0x1'/>
      </source>
      <vlan>
        <tag id='3020'/>
      </vlan>
      <address type='pci' domain='0x0000' bus='0x08' slot='0x00' function='0x0'/>
    </interface>
    <interface type='hostdev' managed='yes'>
      <mac address='fa:16:3e:6c:96:4e'/>
      <source>
        <address type='pci' domain='0x0000' bus='0x20' slot='0x05' function='0x0'/>
      </source>
      <vlan>
        <tag id='2982'/>
      </vlan>
      <address type='pci' domain='0x0000' bus='0x09' slot='0x00' function='0x0'/>
    </interface>
    <interface type='hostdev' managed='yes'>
      <mac address='fa:16:3e:79:b6:fa'/>
      <source>
        <address type='pci' domain='0x0000' bus='0x20' slot='0x04' function='0x7'/>
      </source>
      <vlan>
        <tag id='2451'/>
      </vlan>
      <address type='pci' domain='0x0000' bus='0x0a' slot='0x00' function='0x0'/>
    </interface>
    <interface type='hostdev' managed='yes'>
      <mac address='fa:16:3e:e5:81:45'/>
      <source>
        <address type='pci' domain='0x0000' bus='0x20' slot='0x04' function='0x6'/>
      </source>
      <vlan>
        <tag id='2581'/>
      </vlan>
      <address type='pci' domain='0x0000' bus='0x0b' slot='0x00' function='0x0'/>
    </interface>
    <serial type='pty'>
      <log file='/var/lib/nova/instances/909eed8f-428d-4af9-940d-467b59497b55/console.log' append='off'/>
      <target type='isa-serial' port='0'>
        <model name='isa-serial'/>
      </target>
    </serial>
    <console type='pty'>
      <log file='/var/lib/nova/instances/909eed8f-428d-4af9-940d-467b59497b55/console.log' append='off'/>
      <target type='serial' port='0'/>
    </console>
    <input type='tablet' bus='usb'>
      <address type='usb' bus='0' port='1'/>
    </input>
    <input type='mouse' bus='ps2'/>
    <input type='keyboard' bus='ps2'/>
    <graphics type='vnc' port='-1' autoport='yes' listen='172.16.0.163'>
      <listen type='address' address='172.16.0.163'/>
    </graphics>
    <audio id='1' type='none'/>
    <video>
      <model type='virtio' heads='1' primary='yes'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <stats period='10'/>
      <address type='pci' domain='0x0000' bus='0x0c' slot='0x00' function='0x0'/>
    </memballoon>
    <rng model='virtio'>
      <backend model='random'>/dev/urandom</backend>
      <address type='pci' domain='0x0000' bus='0x0d' slot='0x00' function='0x0'/>
    </rng>
  </devices>
</domain>
: libvirt.libvirtError: internal error: missing IFLA_VF_INFO in netlink response
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest Traceback (most recent call last):
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest   File "/usr/lib/python3.9/site-packages/nova/virt/libvirt/guest.py", line 165, in launch
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest     return self._domain.createWithFlags(flags)
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest   File "/usr/lib/python3.9/site-packages/eventlet/tpool.py", line 190, in doit
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest     result = proxy_call(self._autowrap, f, *args, **kwargs)
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest   File "/usr/lib/python3.9/site-packages/eventlet/tpool.py", line 148, in proxy_call
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest     rv = execute(f, *args, **kwargs)
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest   File "/usr/lib/python3.9/site-packages/eventlet/tpool.py", line 129, in execute
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest     six.reraise(c, e, tb)
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest   File "/usr/lib/python3.9/site-packages/six.py", line 709, in reraise
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest     raise value
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest   File "/usr/lib/python3.9/site-packages/eventlet/tpool.py", line 83, in tworker
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest     rv = meth(*args, **kwargs)
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest   File "/usr/lib64/python3.9/site-packages/libvirt.py", line 1409, in createWithFlags
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest     raise libvirtError('virDomainCreateWithFlags() failed')
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest libvirt.libvirtError: internal error: missing IFLA_VF_INFO in netlink response
2024-06-28 12:06:45.725 2 ERROR nova.virt.libvirt.guest
2024-06-28 12:06:45.726 2 ERROR nova.virt.libvirt.driver [req-c930949f-8b57-4d34-bbc7-8ed3b41c4990 f3c223c7ad8009b3253daae5c72a4244ec61ddafa2a362e32b3e1e9d1900f85d f2fdac2710674852ae99385620cdbff3 - 8cf3016a6a3f4b3197fb230609c225f7 8cf3016a6a3f4b3197fb230609c225f7] [instance: 909eed8f-428d-4af9-940d-467b59497b55] Failed to start libvirt guest: libvirt.libvirtError: internal error: missing IFLA_VF_INFO in netlink response