[Multus] Failed to create pod sandbox after sdn2ovn migration

4.12.0-0.nightly-2022-10-20-104328

Several times in 4.12 build

1. Create a 4.12 GCP SDN cluster
2. Create NAD and multus pod, pod runs fine
3. Run sdn2ovn migration and migration steps pass
4. Check multus pod, it is in ContainerCreating state

[weliang@weliang sdn2ovn-multus]$ oc get co
NAME                                       VERSION                              AVAILABLE   PROGRESSING   DEGRADED   SINCE   MESSAGE
authentication                             4.12.0-0.nightly-2022-10-20-104328   True        False         False      142m    
baremetal                                  4.12.0-0.nightly-2022-10-20-104328   True        False         False      159m    
cloud-controller-manager                   4.12.0-0.nightly-2022-10-20-104328   True        False         False      160m    
cloud-credential                           4.12.0-0.nightly-2022-10-20-104328   True        False         False      161m    
cluster-autoscaler                         4.12.0-0.nightly-2022-10-20-104328   True        False         False      159m    
config-operator                            4.12.0-0.nightly-2022-10-20-104328   True        False         False      160m    
console                                    4.12.0-0.nightly-2022-10-20-104328   True        False         False      148m    
control-plane-machine-set                  4.12.0-0.nightly-2022-10-20-104328   True        False         False      159m    
csi-snapshot-controller                    4.12.0-0.nightly-2022-10-20-104328   True        False         False      160m    
dns                                        4.12.0-0.nightly-2022-10-20-104328   True        False         False      159m    
etcd                                       4.12.0-0.nightly-2022-10-20-104328   True        False         False      158m    
image-registry                             4.12.0-0.nightly-2022-10-20-104328   True        False         False      150m    
ingress                                    4.12.0-0.nightly-2022-10-20-104328   True        False         False      72m     
insights                                   4.12.0-0.nightly-2022-10-20-104328   True        False         False      154m    
kube-apiserver                             4.12.0-0.nightly-2022-10-20-104328   True        False         False      155m    
kube-controller-manager                    4.12.0-0.nightly-2022-10-20-104328   True        False         False      157m    
kube-scheduler                             4.12.0-0.nightly-2022-10-20-104328   True        False         False      156m    
kube-storage-version-migrator              4.12.0-0.nightly-2022-10-20-104328   True        False         False      83m     
machine-api                                4.12.0-0.nightly-2022-10-20-104328   True        False         False      150m    
machine-approver                           4.12.0-0.nightly-2022-10-20-104328   True        False         False      159m    
machine-config                             4.12.0-0.nightly-2022-10-20-104328   True        False         False      159m    
marketplace                                4.12.0-0.nightly-2022-10-20-104328   True        False         False      159m    
monitoring                                 4.12.0-0.nightly-2022-10-20-104328   True        False         False      148m    
network                                    4.12.0-0.nightly-2022-10-20-104328   True        False         False      161m    
node-tuning                                4.12.0-0.nightly-2022-10-20-104328   True        False         False      159m    
openshift-apiserver                        4.12.0-0.nightly-2022-10-20-104328   True        False         False      151m    
openshift-controller-manager               4.12.0-0.nightly-2022-10-20-104328   True        False         False      155m    
openshift-samples                          4.12.0-0.nightly-2022-10-20-104328   True        False         False      151m    
operator-lifecycle-manager                 4.12.0-0.nightly-2022-10-20-104328   True        False         False      159m    
operator-lifecycle-manager-catalog         4.12.0-0.nightly-2022-10-20-104328   True        False         False      159m    
operator-lifecycle-manager-packageserver   4.12.0-0.nightly-2022-10-20-104328   True        False         False      151m    
service-ca                                 4.12.0-0.nightly-2022-10-20-104328   True        False         False      160m    
storage                                    4.12.0-0.nightly-2022-10-20-104328   True        False         False      159m    [weliang@weliang sdn2ovn-multus]$ oc describe pod macvlan-bridge-whereabouts-pod1-v5bll -n multus-upgrade
Name:           macvlan-bridge-whereabouts-pod1-v5bll
Namespace:      multus-upgrade
Priority:       0
Node:           weliang-1020d-v5vmt-worker-a-dnlv6.c.openshift-qe.internal/10.0.128.4
Start Time:     Thu, 20 Oct 2022 14:46:30 -0400
Labels:         name=test-pod1
Annotations:    k8s.ovn.org/pod-networks:
                  {"default":{"ip_addresses":["10.130.0.18/23"],"mac_address":"0a:58:0a:82:00:12","gateway_ips":["10.130.0.1"],"ip_address":"10.130.0.18/23"...
                k8s.v1.cni.cncf.io/networks: macvlan-bridge-whereabouts
                openshift.io/scc: restricted-v2
                seccomp.security.alpha.kubernetes.io/pod: runtime/default
Status:         Pending
IP:             
IPs:            <none>
Controlled By:  ReplicationController/macvlan-bridge-whereabouts-pod1
Containers:
  macvlan-bridge-whereabouts:
    Container ID:   
    Image:          quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4
    Image ID:       
    Port:           <none>
    Host Port:      <none>
    State:          Waiting
      Reason:       ContainerCreating
    Ready:          False
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-b5sv4 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  kube-api-access-b5sv4:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
    ConfigMapName:           openshift-service-ca.crt
    ConfigMapOptional:       <nil>
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason                  Age                   From               Message
  ----     ------                  ----                  ----               -------
  Normal   Scheduled               36m                   default-scheduler  Successfully assigned multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll to weliang-1020d-v5vmt-worker-a-dnlv6.c.openshift-qe.internal
  Normal   AddedInterface          36m                   multus             Add eth0 [10.128.2.14/23] from openshift-sdn
  Warning  FailedCreatePodSandBox  36m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(6bdb9d78aa893e9448a585b36a4e0b6780a048dd73d1dcce674fcea1d35fcce6): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          36m                   multus             Add eth0 [10.128.2.18/23] from openshift-sdn
  Warning  FailedCreatePodSandBox  36m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(96808d079abc0c978f4e5e5e272d4e1512b1a027ccbc5b042730d3a44b59ce5e): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          35m                   multus             Add eth0 [10.128.2.19/23] from openshift-sdn
  Warning  FailedCreatePodSandBox  35m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(41c80f7d9f33ef6d48b9fbfcd0b3887e839e062cf4bb16c97349c7acdd8a8603): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          35m                   multus             Add eth0 [10.128.2.20/23] from openshift-sdn
  Warning  FailedCreatePodSandBox  35m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(619755ec3595a4ebefe2ff2f6b9850b7e705d7d86f328f9bfe3d9fde20ecc1f1): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          35m                   multus             Add eth0 [10.128.2.21/23] from openshift-sdn
  Warning  FailedCreatePodSandBox  35m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(fce5af4a0d84fa1370cc3c6a3dd1bb70d25dbd71f60dd3f236b7dd478c8cfd3d): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          35m                   multus             Add eth0 [10.128.2.22/23] from openshift-sdn
  Warning  FailedCreatePodSandBox  35m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(0c0170bde90564c450261a66ca73ccc990730bc42a7e84aee86677cf80e32253): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          34m                   multus             Add eth0 [10.128.2.23/23] from openshift-sdn
  Warning  FailedCreatePodSandBox  34m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(7ae5fca3ca15ee55da7e5f9e5bfb779eb7b0d0e0de26d579bc562e225a1736a1): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          34m                   multus             Add eth0 [10.128.2.24/23] from openshift-sdn
  Warning  FailedCreatePodSandBox  34m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(fc0f4ff67bd53883f06a6a84de45ff73be24201d68440b0143b020a554b92bcc): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          34m                   multus             Add eth0 [10.128.2.25/23] from openshift-sdn
  Warning  FailedCreatePodSandBox  34m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(b553f8cd284f0d60f1545ad7a4fc7fdf1407286f6f1bb310f69e9bac57e34a02): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          34m                   multus             Add eth0 [10.128.2.26/23] from openshift-sdn
  Normal   AddedInterface          33m                   multus             Add eth0 [10.128.2.27/23] from openshift-sdn
  Normal   AddedInterface          33m                   multus             Add eth0 [10.128.2.28/23] from openshift-sdn
  Normal   AddedInterface          33m                   multus             Add eth0 [10.128.2.29/23] from openshift-sdn
  Normal   AddedInterface          33m                   multus             Add eth0 [10.128.2.30/23] from openshift-sdn
  Normal   AddedInterface          32m                   multus             Add eth0 [10.128.2.32/23] from openshift-sdn
  Normal   AddedInterface          32m                   multus             Add eth0 [10.128.2.33/23] from openshift-sdn
  Normal   AddedInterface          32m                   multus             Add eth0 [10.128.2.34/23] from openshift-sdn
  Normal   AddedInterface          32m                   multus             Add eth0 [10.128.2.35/23] from openshift-sdn
  Normal   AddedInterface          31m                   multus             Add eth0 [10.128.2.36/23] from openshift-sdn
  Normal   AddedInterface          31m                   multus             Add eth0 [10.128.2.37/23] from openshift-sdn
  Normal   AddedInterface          31m                   multus             Add eth0 [10.128.2.38/23] from openshift-sdn
  Normal   AddedInterface          31m                   multus             Add eth0 [10.128.2.39/23] from openshift-sdn
  Normal   AddedInterface          30m                   multus             Add eth0 [10.128.2.40/23] from openshift-sdn
  Normal   AddedInterface          30m                   multus             Add eth0 [10.128.2.41/23] from openshift-sdn
  Normal   AddedInterface          30m                   multus             Add eth0 [10.128.2.42/23] from openshift-sdn
  Normal   AddedInterface          29m                   multus             Add eth0 [10.128.2.43/23] from openshift-sdn
  Warning  FailedCreatePodSandBox  29m (x17 over 34m)    kubelet            (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(cce75eee5db19384db08802ee9067eb9f80cfc223cb1771607dd3252af89804f): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          29m                   multus             Add eth0 [10.128.2.44/23] from openshift-sdn
  Normal   AddedInterface          29m                   multus             Add eth0 [10.128.2.45/23] from openshift-sdn
  Normal   AddedInterface          29m                   multus             Add eth0 [10.128.2.46/23] from openshift-sdn
  Normal   AddedInterface          28m                   multus             Add eth0 [10.128.2.47/23] from openshift-sdn
  Normal   AddedInterface          28m                   multus             Add eth0 [10.128.2.48/23] from openshift-sdn
  Normal   AddedInterface          28m                   multus             Add eth0 [10.128.2.49/23] from openshift-sdn
  Normal   AddedInterface          28m                   multus             Add eth0 [10.128.2.50/23] from openshift-sdn
  Normal   AddedInterface          27m                   multus             Add eth0 [10.128.2.51/23] from openshift-sdn
  Normal   AddedInterface          27m                   multus             Add eth0 [10.128.2.52/23] from openshift-sdn
  Normal   AddedInterface          27m                   multus             Add eth0 [10.128.2.53/23] from openshift-sdn
  Normal   AddedInterface          27m                   multus             Add eth0 [10.128.2.54/23] from openshift-sdn
  Normal   AddedInterface          26m                   multus             Add eth0 [10.128.2.55/23] from openshift-sdn
  Normal   AddedInterface          26m                   multus             Add eth0 [10.128.2.56/23] from openshift-sdn
  Warning  ErrorAddingLogicalPort  25m (x2 over 25m)     controlplane       addLogicalPort failed for multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll: unable to parse node L3 gw annotation: k8s.ovn.org/l3-gateway-config annotation not found for node "weliang-1020d-v5vmt-worker-a-dnlv6.c.openshift-qe.internal"
  Warning  FailedCreatePodSandBox  20m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(1b89bb264ada2dba28151bfa48044eb12cbfee2c06a7b4878e0561c81ef4e2f2): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): Multus: [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa]: have you checked that your default network is ready? still waiting for readinessindicatorfile @ /var/run/multus/cni/net.d/80-openshift-network.conf. pollimmediate error: timed out waiting for the condition
  Warning  FailedCreatePodSandBox  18m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(7cb3565ff672525f7d514e61dd31f700b6f80477b979635fac25d6ed47e9aa40): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): Multus: [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa]: have you checked that your default network is ready? still waiting for readinessindicatorfile @ /var/run/multus/cni/net.d/80-openshift-network.conf. pollimmediate error: timed out waiting for the condition
  Normal   AddedInterface          18m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Warning  FailedCreatePodSandBox  18m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(6d471dfe0ff378352a30ed63246ff3c97576cf2ad52e68fa5be7da382b724579): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          17m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Warning  FailedCreatePodSandBox  17m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(92d67e2426a19dd2be0b80391224240d5fc0d912aa20529b3485f94632ec55c3): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          17m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Warning  FailedCreatePodSandBox  17m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(cee5329dfd44427147f5092d8ced4f1ea1c1f3f0ba0b052b4829be61bb526c33): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          17m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Warning  FailedCreatePodSandBox  17m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(602617c114c0919ea64c38dd6c5432a3d9e63ae6c601e0024308941f644a1b4f): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          17m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Warning  FailedCreatePodSandBox  17m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(7515ccb947bc4b02df84eb2fb0d93ceb2e51302ad5d4ef835dcc6daf30ae1ad7): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          16m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Warning  FailedCreatePodSandBox  16m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(cd1c4735627517dac184af1ed4afaa32307e5d8ee257ba17d719c6e9ef5fdf83): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          16m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Warning  FailedCreatePodSandBox  16m                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(a884425bc929966742fd3c2de9acc94a7f6a235d19a2b4d1c4493d4fa4b440cf): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          16m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          16m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          16m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          15m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          15m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          15m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          15m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          14m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          14m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          14m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          14m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          14m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          13m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          13m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          13m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          13m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          12m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          12m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          12m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          12m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          12m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          11m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          11m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          11m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          11m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          10m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          10m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          10m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          10m                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          9m52s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          9m38s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          9m22s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          9m11s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          8m58s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          8m46s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          8m34s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          8m21s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          8m5s                  multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          7m52s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          7m36s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          7m22s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          7m8s                  multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          6m56s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          6m44s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          6m29s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          6m16s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          6m                    multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          5m49s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          5m36s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          5m21s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          5m7s                  multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          4m53s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Warning  FailedCreatePodSandBox  4m52s (x52 over 16m)  kubelet            (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_macvlan-bridge-whereabouts-pod1-v5bll_multus-upgrade_d392ce2b-b196-4c02-80b7-6d4de68652fa_0(3b7b2c067280d0813d5098570c94bbe134662fa5e62721b28699646387747b39): error adding pod multus-upgrade_macvlan-bridge-whereabouts-pod1-v5bll to CNI network "multus-cni-network": plugin type="multus" name="multus-cni-network" failed (add): [multus-upgrade/macvlan-bridge-whereabouts-pod1-v5bll/d392ce2b-b196-4c02-80b7-6d4de68652fa:macvlan-bridge-whereabouts]: error adding container to network "macvlan-bridge-whereabouts": failed to create macvlan: device or resource busy
  Normal   AddedInterface          4m37s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          4m21s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          4m7s                  multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          3m52s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          3m38s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          3m24s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          3m9s                  multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          2m53s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          2m41s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          2m27s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          2m12s                 multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          118s                  multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          103s                  multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          91s                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          79s                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          65s                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          49s                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          34s                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          22s                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
  Normal   AddedInterface          11s                   multus             Add eth0 [10.130.0.18/23] from ovn-kubernetes
[weliang@weliang sdn2ovn-multus]$ 

mutus pods keep running fine

must-gather: http://file.rdu.redhat.com/~weliang/must-gather/