# /bin/virsh migrate --verbose --persistent --postcopy  --timeout 2 --timeout-postcopy --live rhel9.5 qemu+ssh://192.168.1.2/system

Migration: [99.89 %] error: internal error: qemu unexpectedly closed the monitor: 2024-08-21T07:07:50.193610Z qemu-kvm: -chardev socket,id=charnet2,path=/tmp/vhostuser1.sock,server=on: info: QEMU waiting for connection on: disconnected:unix:/tmp/vhostuser1.sock,server=on
2024-08-21T07:07:50.194196Z qemu-kvm: -chardev socket,id=charnet3,path=/tmp/vhostuser2.sock,server=on: info: QEMU waiting for connection on: disconnected:unix:/tmp/vhostuser2.sock,server=on
2024-08-21T07:08:09.557037Z qemu-kvm: Features 0x330afa7a7 unsupported. Allowed features: 0x378bfa7e6
2024-08-21T07:08:09.557070Z qemu-kvm: Failed to load virtio-net:virtio
2024-08-21T07:08:09.557079Z qemu-kvm: error while loading state for instance 0x0 of device '0000:00:02.7:00.0/virtio-net'
2024-08-21T07:08:09.594510Z qemu-kvm: load of migration failed: Operation not permitted

# /bin/virsh migrate --verbose --persistent --live rhel8.10 qemu+ssh://192.168.1.2/system
root@192.168.1.2's password: 
Migration: [100.00 %]error: internal error: qemu unexpectedly closed the monitor: 2024-08-21T08:56:25.895057Z qemu-kvm: -chardev socket,id=charnet2,path=/tmp/vhostuser1.sock,server=on: info: QEMU waiting for connection on: disconnected:unix:/tmp/vhostuser1.sock,server=on
2024-08-21T08:56:25.895839Z qemu-kvm: -chardev socket,id=charnet3,path=/tmp/vhostuser2.sock,server=on: info: QEMU waiting for connection on: disconnected:unix:/tmp/vhostuser2.sock,server=on
2024-08-21T08:56:29.959475Z qemu-kvm: -device cirrus-vga,id=video0,bus=pcie.0,addr=0x1: warning: 'cirrus-vga' is deprecated, please use a different VGA card instead
2024-08-21T08:56:36.957537Z qemu-kvm: Features 0x330afa7a7 unsupported. Allowed features: 0x378bfa7e6
2024-08-21T08:56:36.957556Z qemu-kvm: Failed to load virtio-net:virtio
2024-08-21T08:56:36.957565Z qemu-kvm: error while loading state for instance 0x0 of device '0000:00:02.5:00.0/virtio-net'
2024-08-21T08:56:36.959377Z qemu-kvm: load of migration failed: Operation not permitted

<domain type='kvm'>
  <name>rhel9.5</name>
  <uuid>bdcb9088-5f8a-11ef-a0ef-20040fec000c</uuid>
  <memory unit='KiB'>8388608</memory>
  <currentMemory unit='KiB'>8388608</currentMemory>
  <memoryBacking>
    <hugepages>
      <page size='1048576' unit='KiB'/>
    </hugepages>
    <locked/>
  </memoryBacking>
  <vcpu placement='static'>8</vcpu>
  <cputune>
    <vcpupin vcpu='0' cpuset='22'/>
    <vcpupin vcpu='1' cpuset='24'/>
    <vcpupin vcpu='2' cpuset='26'/>
    <vcpupin vcpu='3' cpuset='28'/>
    <vcpupin vcpu='4' cpuset='30'/>
    <vcpupin vcpu='5' cpuset='32'/>
    <vcpupin vcpu='6' cpuset='34'/>
    <vcpupin vcpu='7' cpuset='36'/>
    <emulatorpin cpuset='3,5,7,9'/>
  </cputune>
  <numatune>
    <memory mode='strict' nodeset='0'/>
    <memnode cellid='0' mode='strict' nodeset='0'/>
  </numatune>
  <resource>
    <partition>/machine</partition>
  </resource>
  <os>
    <type arch='x86_64' machine='pc-q35-rhel8.6.0'>hvm</type>
    <boot dev='hd'/>
  </os>
  <features>
    <acpi/>
    <pmu state='off'/>
    <vmport state='off'/>
    <ioapic driver='qemu'/>
  </features>
  <cpu mode='host-passthrough' check='none' migratable='on'>
    <topology sockets='4' dies='1' clusters='1' cores='1' threads='2'/>
    <numa>
      <cell id='0' cpus='0-7' memory='8388608' unit='KiB' memAccess='shared'/>
    </numa>
  </cpu>
  <clock offset='utc'>
    <timer name='rtc' tickpolicy='catchup'/>
    <timer name='pit' tickpolicy='delay'/>
    <timer name='hpet' present='no'/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>restart</on_crash>
  <devices>
    <emulator>/usr/libexec/qemu-kvm</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='qcow2' cache='none' io='threads' iommu='on' ats='on'/>
      <source file='/mnt/nfv/rhel9.5.qcow2'/>
      <backingStore/>
      <target dev='vda' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
    </disk>
    <controller type='usb' index='0' model='none'/>
    <controller type='pci' index='0' model='pcie-root'/>
    <controller type='pci' index='1' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='1' port='0x10'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
    </controller>
    <controller type='pci' index='2' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='2' port='0x11'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
    </controller>
    <controller type='pci' index='3' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='3' port='0x12'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
    </controller>
    <controller type='pci' index='4' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='4' port='0x13'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
    </controller>
    <controller type='pci' index='5' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='5' port='0x14'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
    </controller>
    <controller type='pci' index='6' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='6' port='0x15'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
    </controller>
    <controller type='pci' index='7' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='7' port='0x16'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x6'/>
    </controller>
    <controller type='pci' index='8' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='8' port='0x17'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x7'/>
    </controller>
    <controller type='sata' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
    </controller>
    <interface type='bridge'>
      <mac address='28:66:da:5f:dd:01'/>
      <source bridge='switch'/>
      <model type='virtio'/>
      <driver name='vhost' iommu='on' ats='on'/>
      <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
    </interface>
    <interface type='vhostuser'>
      <mac address='18:66:da:5f:dd:02'/>
      <source type='unix' path='/tmp/vhostuser0.sock' mode='server'/>
      <target dev='vhost-user0'/>
      <model type='virtio'/>
      <driver name='vhost' rx_queue_size='1024' tx_queue_size='1024' iommu='on' ats='on'>
        <host mrg_rxbuf='on'/>
      </driver>
      <address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
    </interface>
    <interface type='vhostuser'>
      <mac address='18:66:da:5f:dd:03'/>
      <source type='unix' path='/tmp/vhostuser1.sock' mode='server'/>
      <target dev='vhost-user1'/>
      <model type='virtio'/>
      <driver name='vhost' rx_queue_size='1024' tx_queue_size='1024' iommu='on' ats='on'>
        <host mrg_rxbuf='on'/>
      </driver>
      <address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
    </interface>
    <interface type='vhostuser'>
      <mac address='18:66:da:5f:dd:04'/>
      <source type='unix' path='/tmp/vhostuser2.sock' mode='server'/>
      <target dev='vhost-user2'/>
      <model type='virtio'/>
      <driver name='vhost' rx_queue_size='1024' tx_queue_size='1024' iommu='on' ats='on'>
        <host mrg_rxbuf='on'/>
      </driver>
      <address type='pci' domain='0x0000' bus='0x08' slot='0x00' function='0x0'/>
    </interface>
    <serial type='pty'>
      <target type='isa-serial' port='0'>
        <model name='isa-serial'/>
      </target>
    </serial>
    <console type='pty'>
      <target type='serial' port='0'/>
    </console>
    <input type='mouse' bus='ps2'/>
    <input type='keyboard' bus='ps2'/>
    <graphics type='vnc' port='-1' autoport='yes' listen='0.0.0.0'>
      <listen type='address' address='0.0.0.0'/>
    </graphics>
    <audio id='1' type='none'/>
    <video>
      <model type='cirrus' vram='16384' heads='1' primary='yes'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
    </video>
    <watchdog model='itco' action='reset'/>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
      <driver iommu='on' ats='on'/>
    </memballoon>
    <iommu model='intel'>
      <driver intremap='on' caching_mode='on' iotlb='on'/>
    </iommu>
  </devices>
  <seclabel type='dynamic' model='selinux' relabel='yes'/>
</domain>

/usr/libexec/qemu-kvm \
-name guest=rhel9.5,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-2-rhel9.5/master-key.aes"}' \
-machine pc-q35-rhel8.6.0,usb=off,vmport=off,kernel_irqchip=split,dump-guest-core=off,hpet=off,acpi=on \
-accel kvm \
-cpu host,migratable=on,pmu=off \
-m size=8388608k \
-overcommit mem-lock=on \
-smp 8,sockets=4,dies=1,clusters=1,cores=1,threads=2 \
-object '{"qom-type":"memory-backend-file","id":"ram-node0","mem-path":"/dev/hugepages/libvirt/qemu/2-rhel9.5","share":true,"prealloc":true,"size":8589934592,"host-nodes":[0],"policy":"bind"}' \
-numa node,nodeid=0,cpus=0-7,memdev=ram-node0 \
-uuid bdcb9088-5f8a-11ef-a0ef-20040fec000c \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=22,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=utc,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-shutdown \
-boot strict=on \
-device '{"driver":"intel-iommu","id":"iommu0","intremap":"on","caching-mode":true,"device-iotlb":true}' \
-device '{"driver":"pcie-root-port","port":16,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x2"}' \
-device '{"driver":"pcie-root-port","port":17,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x2.0x1"}' \
-device '{"driver":"pcie-root-port","port":18,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x2.0x2"}' \
-device '{"driver":"pcie-root-port","port":19,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x2.0x3"}' \
-device '{"driver":"pcie-root-port","port":20,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x2.0x4"}' \
-device '{"driver":"pcie-root-port","port":21,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x2.0x5"}' \
-device '{"driver":"pcie-root-port","port":22,"chassis":7,"id":"pci.7","bus":"pcie.0","addr":"0x2.0x6"}' \
-device '{"driver":"pcie-root-port","port":23,"chassis":8,"id":"pci.8","bus":"pcie.0","addr":"0x2.0x7"}' \
-blockdev '{"driver":"file","filename":"/mnt/nfv/rhel9.5.qcow2","aio":"threads","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap","cache":{"direct":true,"no-flush":false}}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":"libvirt-1-storage","backing":null}' \
-device '{"driver":"virtio-blk-pci","iommu_platform":true,"ats":true,"bus":"pci.2","addr":"0x0","drive":"libvirt-1-format","id":"virtio-disk0","bootindex":1,"write-cache":"on"}' \
-netdev '{"type":"tap","fd":"23","vhost":true,"vhostfd":"25","id":"hostnet0"}' \
-device '{"driver":"virtio-net-pci","iommu_platform":true,"ats":true,"netdev":"hostnet0","id":"net0","mac":"28:66:da:5f:dd:01","bus":"pci.1","addr":"0x0"}' \-chardev socket,id=charnet1,path=/tmp/vhostuser0.sock,server=on \
-netdev '{"type":"vhost-user","chardev":"charnet1","id":"hostnet1"}' \
-device '{"driver":"virtio-net-pci","iommu_platform":true,"ats":true,"mrg_rxbuf":true,"rx_queue_size":1024,"tx_queue_size":1024,"netdev":"hostnet1","id":"net1","mac":"18:66:da:5f:dd:02","bus":"pci.6","addr":"0x0"}' \
-chardev socket,id=charnet2,path=/tmp/vhostuser1.sock,server=on \
-netdev '{"type":"vhost-user","chardev":"charnet2","id":"hostnet2"}' \
-device '{"driver":"virtio-net-pci","iommu_platform":true,"ats":true,"mrg_rxbuf":true,"rx_queue_size":1024,"tx_queue_size":1024,"netdev":"hostnet2","id":"net2","mac":"18:66:da:5f:dd:03","bus":"pci.7","addr":"0x0"}' \
-chardev socket,id=charnet3,path=/tmp/vhostuser2.sock,server=on \
-netdev '{"type":"vhost-user","chardev":"charnet3","id":"hostnet3"}' \
-device '{"driver":"virtio-net-pci","iommu_platform":true,"ats":true,"mrg_rxbuf":true,"rx_queue_size":1024,"tx_queue_size":1024,"netdev":"hostnet3","id":"net3","mac":"18:66:da:5f:dd:04","bus":"pci.8","addr":"0x0"}' \
-chardev pty,id=charserial0 \
-device '{"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}' \
-audiodev '{"id":"audio1","driver":"none"}' \
-vnc 0.0.0.0:0,audiodev=audio1 \
-device '{"driver":"cirrus-vga","id":"video0","bus":"pcie.0","addr":"0x1"}' \
-global ICH9-LPC.noreboot=off \
-watchdog-action reset \
-incoming defer \
-device '{"driver":"virtio-balloon-pci","iommu_platform":true,"ats":true,"id":"balloon0","bus":"pci.4","addr":"0x0"}' \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on

echo 20 > /sys/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages
echo 20 > /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages
yum -y install dpdk*
modprobe vfio
modprobe vfio-pci
dpdk-devbind.py --bind=vfio-pci 0000:5e:00.0
dpdk-devbind.py --bind=vfio-pci 0000:5e:00.1
dpdk-devbind.py --bind=vfio-pci 0000:60:00.0
modprobe -r openvswitch
modprobe openvswitch
/usr/bin/ovsdb-tool create /etc/openvswitch/conf.db /usr/share/openvswitch/vswitch.ovsschema
/usr/sbin/ovsdb-server --remote=punix:/var/run/openvswitch/db.sock --remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach --log-file
/usr/bin/ovs-vsctl --no-wait init
/usr/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem='1024,1024'
/usr/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask='0x1'
/usr/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:vhost-iommu-support=true
/usr/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:vhost-postcopy-support=true
/usr/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
/usr/sbin/ovs-vswitchd unix:/var/run/openvswitch/db.sock --pidfile --detach --log-file=/var/log/openvswitch/ovs-vswitchd.log
/usr/bin/ovs-vsctl --if-exists del-br ovsbr0
/usr/bin/ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
/usr/bin/ovs-vsctl add-port ovsbr0 dpdk0 -- set Interface dpdk0 type=dpdk  options:dpdk-devargs=0000:5e:00.0  options:n_rxq=1  options:n_txq=1 
/usr/bin/ovs-vsctl add-port ovsbr0 dpdk1 -- set Interface dpdk1 type=dpdk  options:dpdk-devargs=0000:5e:00.1  options:n_rxq=1  options:n_txq=1 
/usr/bin/ovs-vsctl add-port ovsbr0 vhost-user0 -- set Interface vhost-user0 type=dpdkvhostuserclient options:vhost-server-path=/tmp/vhostuser0.sock
/usr/bin/ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/vhostuser1.sock
/usr/bin/ovs-ofctl del-flows ovsbr0
/usr/bin/ovs-ofctl add-flow ovsbr0 'in_port=1,idle_timeout=0 actions=output:3'
/usr/bin/ovs-ofctl add-flow ovsbr0 'in_port=3,idle_timeout=0 actions=output:1'
/usr/bin/ovs-ofctl add-flow ovsbr0 'in_port=2,idle_timeout=0 actions=output:4'
/usr/bin/ovs-ofctl add-flow ovsbr0 'in_port=4,idle_timeout=0 actions=output:2'
/usr/bin/ovs-vsctl --if-exists del-br ovsbr1
/usr/bin/ovs-vsctl add-br ovsbr1 -- set bridge ovsbr1 datapath_type=netdev
/usr/bin/ovs-vsctl add-port ovsbr1 dpdk2 -- set Interface dpdk2 type=dpdk  options:dpdk-devargs=0000:60:00.0  options:n_rxq=1  options:n_txq=1 
/usr/bin/ovs-vsctl add-port ovsbr1 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/vhostuser2.sock
/usr/bin/ovs-ofctl del-flows ovsbr1
/usr/bin/ovs-ofctl add-flow ovsbr1 'in_port=1,idle_timeout=0 actions=output:2'
/usr/bin/ovs-ofctl add-flow ovsbr1 'in_port=2,idle_timeout=0 actions=output:1'
/usr/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=0x15554
/usr/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0x1
/usr/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true