In dual stream cluster testing, a VM initially scheduled on a RHCOS10.1 node cannot be live migrated to a RHCOS9.6 node. However, a VM created on a RHCOS9.6 node can successfully migrate to RHCOS10.1.

CNV-v4.21.0.rhel9-8

100%

1.create a vm scheduled on rhcos10 node 
2.migrate to rhcos9 node
3.

live migration failed

live migration success

Status:
  Migration State:
    End Timestamp:   2025-11-05T06:55:17Z
    Failed:          true
    Failure Reason:  virError(Code=1, Domain=7, Message='internal error: client socket is closed')
    Migration Configuration:
      Allow Auto Converge:                    false
      Allow Post Copy:                        false
      Allow Workload Disruption:              false
      Bandwidth Per Migration:                0
      Completion Timeout Per Gi B:            150
      Node Drain Taint Key:                   kubevirt.io/drain
      Parallel Migrations Per Cluster:        5
      Parallel Outbound Migrations Per Node:  2
      Progress Timeout:                       150
      Unsafe Migration Override:              false

oc logs -f virt-launcher-rhel9-base-rhcos10-node-j27vv | grep "client socket is closed"
{"component":"virt-launcher","level":"error","msg":"internal error: client socket is closed","pos":"virNetClientSendInternal:2173","subcomponent":"libvirt","thread":"28","timestamp":"2025-11-05T06:55:17.154000Z"}
{"component":"virt-launcher","level":"info","msg":"set migration result in metadata: {a797409d-fd93-4a99-a5b6-dd21734d009e 2025-11-05 06:55:14.400054077 +0000 UTC m=+50.327880923 2025-11-05 06:55:17.166439795 +0000 UTC m=+53.094266646 true virError(Code=1, Domain=7, Message='internal error: client socket is closed')  PreCopy}","pos":"live-migration-source.go:442","timestamp":"2025-11-05T06:55:17.166458Z"}
{"component":"virt-launcher","kind":"","level":"error","msg":"migration failed with error: virError(Code=1, Domain=7, Message='internal error: client socket is closed')","name":"rhel9-base-rhcos10-node","namespace":"default","pos":"live-migration-source.go:1087","timestamp":"2025-11-05T06:55:17.166506Z","uid":"733a3870-f1e1-4fa1-8195-f4593ef66bf1"}
{"component":"virt-launcher","kind":"","level":"error","msg":"Live migration failed.","name":"rhel9-base-rhcos10-node","namespace":"default","pos":"live-migration-source.go:1135","reason":"error encountered during MigrateToURI3 libvirt api call: virError(Code=1, Domain=7, Message='internal error: client socket is closed')","timestamp":"2025-11-05T06:55:17.166539Z","uid":"733a3870-f1e1-4fa1-8195-f4593ef66bf1"}
{"component":"virt-launcher","kind":"","level":"error","msg":"Received a live migration error. Will check the latest migration status.","name":"rhel9-base-rhcos10-node","namespace":"default","pos":"live-migration-source.go:667","reason":"error encountered during MigrateToURI3 libvirt api call: virError(Code=1, Domain=7, Message='internal error: client socket is closed')","timestamp":"2025-11-05T06:55:17.166569Z","uid":"733a3870-f1e1-4fa1-8195-f4593ef66bf1"}
{"component":"virt-launcher","kind":"","level":"error","msg":"Live migration failed.","name":"rhel9-base-rhcos10-node","namespace":"default","pos":"live-migration-source.go:671","reason":"error encountered during MigrateToURI3 libvirt api call: virError(Code=1, Domain=7, Message='internal error: client socket is closed')","timestamp":"2025-11-05T06:55:17.167152Z","uid":"733a3870-f1e1-4fa1-8195-f4593ef66bf1"} 

  Migration Method:   BlockMigration
  Migration State:
    End Timestamp:   2025-11-05T06:55:17Z
    Failed:          true
    Failure Reason:  virError(Code=1, Domain=7, Message='internal error: client socket is closed')
    Migration Configuration:
      Allow Auto Converge:                    false
      Allow Post Copy:                        false
      Allow Workload Disruption:              false
      Bandwidth Per Migration:                0
      Completion Timeout Per Gi B:            150
      Node Drain Taint Key:                   kubevirt.io/drain
      Parallel Migrations Per Cluster:        5
      Parallel Outbound Migrations Per Node:  2
      Progress Timeout:                       150
      Unsafe Migration Override:              false
    Migration UID:                            a797409d-fd93-4a99-a5b6-dd21734d009e
    Mode:                                     PreCopy
    Source Node:                              cnv-qe-infra-07.cnvqe2.lab.eng.rdu2.redhat.com
    Source Pod:                               virt-launcher-rhel9-base-rhcos10-node-j27vv
    Start Timestamp:                          2025-11-05T06:55:14Z
    Target Direct Migration Node Ports:
      32879:                      49152
      32951:                      49153
      37669:                      0
    Target Node:                  cnv-qe-infra-06.cnvqe2.lab.eng.rdu2.redhat.com
    Target Node Address:          10.129.2.47
    Target Node Domain Detected:  true
    Target Pod:                   virt-launcher-rhel9-base-rhcos10-node-v7rhw
  Migration Transport:            Unix
  Node Name:                      cnv-qe-infra-07.cnvqe2.lab.eng.rdu2.redhat.com
  Phase:                          Running
  Phase Transition Timestamps:
    Phase:                        Pending
    Phase Transition Timestamp:   2025-11-05T06:54:11Z
    Phase:                        Scheduling
    Phase Transition Timestamp:   2025-11-05T06:54:11Z
    Phase:                        Scheduled
    Phase Transition Timestamp:   2025-11-05T06:54:24Z
    Phase:                        Running
    Phase Transition Timestamp:   2025-11-05T06:54:27Z
  Qos Class:                      Burstable
  Runtime User:                   107
  Selinux Context:                system_u:object_r:container_file_t:s0:c326,c338
  Virtual Machine Revision Name:  revision-start-vm-4bd65121-8e30-42ce-b579-6cd45875bf42-1
  Volume Status:
    Name:    cloudinitdisk
    Size:    1048576
    Target:  vdb
    Name:    rootdisk
    Persistent Volume Claim Info:
      Access Modes:
        ReadWriteMany
      Capacity:
        Storage:            30Gi
      Claim Name:           rhel9-base-rhcos10-node
      Filesystem Overhead:  0
      Requests:
        Storage:    32212254720
      Volume Mode:  Block
    Target:         vda
Events:
  Type     Reason            Age                  From                       Message
  ----     ------            ----                 ----                       -------
  Normal   SuccessfulCreate  168m                 virtualmachine-controller  Created virtual machine pod virt-launcher-rhel9-base-rhcos10-node-j27vv
  Normal   Created           168m                 virt-handler               VirtualMachineInstance defined.
  Normal   Started           168m                 virt-handler               VirtualMachineInstance started.
  Normal   PreparingTarget   167m (x2 over 167m)  virt-handler               VirtualMachineInstance Migration Target Prepared.
  Normal   PreparingTarget   167m                 virt-handler               Migration Target is listening at 10.129.2.47, on ports: 32879,32951,37669
  Normal   Migrating         167m                 virt-handler               VirtualMachineInstance is migrating.
  Warning  Migrated          167m                 virt-handler               VirtualMachineInstance migration uid a797409d-fd93-4a99-a5b6-dd21734d009e failed. reason:virError(Code=1, Domain=7, Message='internal error: client socket is closed')