[4.15] After reset the ovnkube-node pod for egressIP node, egressip is not assigned any more

4.15.0-0.nightly-2023-11-16-214550

1.  Label one worker node as egress-assignable 
$ oc get node
NAME                                                        STATUS   ROLES                  AGE     VERSION
jechen-1117a-xsxnc-master-0.c.openshift-qe.internal         Ready    control-plane,master   6h16m   v1.28.3+20a5764
jechen-1117a-xsxnc-master-1.c.openshift-qe.internal         Ready    control-plane,master   6h16m   v1.28.3+20a5764
jechen-1117a-xsxnc-master-2.c.openshift-qe.internal         Ready    control-plane,master   6h16m   v1.28.3+20a5764
jechen-1117a-xsxnc-worker-a-mnqsp.c.openshift-qe.internal   Ready    worker                 6h3m    v1.28.3+20a5764
jechen-1117a-xsxnc-worker-b-9vzkq.c.openshift-qe.internal   Ready    worker                 6h3m    v1.28.3+20a5764
jechen-1117a-xsxnc-worker-c-64mg4.c.openshift-qe.internal   Ready    worker                 6h3m    v1.28.3+20a5764

$ oc label node jechen-1117a-xsxnc-worker-a-mnqsp.c.openshift-qe.internal "k8s.ovn.org/egress-assignable"=""
node/jechen-1117a-xsxnc-worker-a-mnqsp.c.openshift-qe.internal labeled
 

2. create an egressip object with two ip addresses
$ cat /tmp/e2e-test-networking-apqfxthi-jkh76-wj6pn0dfresource.json
{
    "kind": "List",
    "apiVersion": "v1",
    "metadata": {},
    "items": [
        {
            "apiVersion": "k8s.ovn.org/v1",
            "kind": "EgressIP",
            "metadata": {
                "name": "egressip-47021"
            },
            "spec": {
                "egressIPs": [
                    "10.0.183.137",
                    "10.0.229.234"
                ],
                "namespaceSelector": {
                    "matchLabels": {
                        "name": "test"
                    }
                }
            }
        }
    ]
}

$ oc --kubeconfig=/home/jechen/kubeconfig apply -f /tmp/e2e-test-networking-apqfxthi-jkh76-wj6pn0dfresource.json
egressip.k8s.ovn.org/egressip-47021 created

$ oc get egressips.k8s.ovn.org 
NAME             EGRESSIPS      ASSIGNED NODE                                               ASSIGNED EGRESSIPS
egressip-47021   10.0.183.137   jechen-1117a-xsxnc-worker-a-mnqsp.c.openshift-qe.internal   10.0.183.137

$  oc get egressips.k8s.ovn.org egressip-47021 -oyaml
apiVersion: k8s.ovn.org/v1
kind: EgressIP
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"k8s.ovn.org/v1","kind":"EgressIP","metadata":{"annotations":{},"name":"egressip-47021"},"spec":{"egressIPs":["10.0.183.137","10.0.229.234"],"namespaceSelector":{"matchLabels":{"name":"test"}}}}
  creationTimestamp: "2023-11-17T20:05:40Z"
  generation: 2
  name: egressip-47021
  resourceVersion: "141641"
  uid: 6d692135-5bbb-488b-bed4-99d6c66dfd50
spec:
  egressIPs:
  - 10.0.183.137
  - 10.0.229.234
  namespaceSelector:
    matchLabels:
      name: test
status:
  items:
  - egressIP: 10.0.183.137
    node: jechen-1117a-xsxnc-worker-a-mnqsp.c.openshift-qe.internal


 3. Delete the ovnkube-node for the egress node
$ oc -n openshift-ovn-kubernetes get pod -l app=ovnkube-node -owide
NAME                 READY   STATUS    RESTARTS   AGE     IP           NODE                                                        NOMINATED NODE   READINESS GATES
ovnkube-node-5p8r4   8/8     Running   0          5h43m   10.0.0.3     jechen-1117a-xsxnc-master-0.c.openshift-qe.internal         <none>           <none>
ovnkube-node-bgw7d   8/8     Running   0          16m     10.0.128.2   jechen-1117a-xsxnc-worker-a-mnqsp.c.openshift-qe.internal   <none>           <none>
ovnkube-node-bvhmb   8/8     Running   0          5h34m   10.0.128.4   jechen-1117a-xsxnc-worker-c-64mg4.c.openshift-qe.internal   <none>           <none>
ovnkube-node-gl979   8/8     Running   0          5h34m   10.0.128.3   jechen-1117a-xsxnc-worker-b-9vzkq.c.openshift-qe.internal   <none>           <none>
ovnkube-node-hkh92   8/8     Running   0          5h43m   10.0.0.5     jechen-1117a-xsxnc-master-1.c.openshift-qe.internal         <none>           <none>
ovnkube-node-pmqjf   8/8     Running   0          5h43m   10.0.0.4     jechen-1117a-xsxnc-master-2.c.openshift-qe.internal         <none>           <none>


$ oc -n openshift-ovn-kubernetes delete pod ovnkube-node-bgw7d
pod "ovnkube-node-bgw7d" deleted


$ oc -n openshift-ovn-kubernetes get pod -l app=ovnkube-node -owide
NAME                 READY   STATUS    RESTARTS   AGE     IP           NODE                                                        NOMINATED NODE   READINESS GATES
ovnkube-node-5p8r4   8/8     Running   0          5h44m   10.0.0.3     jechen-1117a-xsxnc-master-0.c.openshift-qe.internal         <none>           <none>
ovnkube-node-bvhmb   8/8     Running   0          5h35m   10.0.128.4   jechen-1117a-xsxnc-worker-c-64mg4.c.openshift-qe.internal   <none>           <none>
ovnkube-node-gl979   8/8     Running   0          5h35m   10.0.128.3   jechen-1117a-xsxnc-worker-b-9vzkq.c.openshift-qe.internal   <none>           <none>
ovnkube-node-hkh92   8/8     Running   0          5h44m   10.0.0.5     jechen-1117a-xsxnc-master-1.c.openshift-qe.internal         <none>           <none>
ovnkube-node-lz4tr   5/8     Running   0          9s      10.0.128.2   jechen-1117a-xsxnc-worker-a-mnqsp.c.openshift-qe.internal   <none>           <none>
ovnkube-node-pmqjf   8/8     Running   0          5h44m   10.0.0.4     jechen-1117a-xsxnc-master-2.c.openshift-qe.internal         <none>           <none>

$ oc get egressips.k8s.ovn.org 
NAME             EGRESSIPS      ASSIGNED NODE   ASSIGNED EGRESSIPS
egressip-47021   10.0.183.137                   


$ oc get egressips.k8s.ovn.org egressip-47021 -oyaml
apiVersion: k8s.ovn.org/v1
kind: EgressIP
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"k8s.ovn.org/v1","kind":"EgressIP","metadata":{"annotations":{},"name":"egressip-47021"},"spec":{"egressIPs":["10.0.183.137","10.0.229.234"],"namespaceSelector":{"matchLabels":{"name":"test"}}}}
  creationTimestamp: "2023-11-17T20:05:40Z"
  generation: 3
  name: egressip-47021
  resourceVersion: "142251"
  uid: 6d692135-5bbb-488b-bed4-99d6c66dfd50
spec:
  egressIPs:
  - 10.0.183.137
  - 10.0.229.234
  namespaceSelector:
    matchLabels:
      name: test
status:
  items: []

partial CNCC logs:
I1117 20:05:44.690956       1 cloudprivateipconfig_controller.go:439] Added IP address to node: "jechen-1117a-xsxnc-worker-a-mnqsp.c.openshift-qe.internal" for CloudPrivateIPConfig: "10.0.183.137"
I1117 20:05:44.702007       1 controller.go:160] Dropping key '10.0.183.137' from the cloud-private-ip-config workqueue
I1117 20:05:44.709306       1 controller.go:160] Dropping key '10.0.183.137' from the cloud-private-ip-config workqueue
I1117 20:07:25.951612       1 controller.go:182] Assigning key: 10.0.183.137 to cloud-private-ip-config workqueue
I1117 20:07:25.960023       1 cloudprivateipconfig_controller.go:264] CloudPrivateIPConfig: "10.0.183.137" will be deleted from node: "jechen-1117a-xsxnc-worker-a-mnqsp.c.openshift-qe.internal"
I1117 20:07:25.970094       1 controller.go:182] Assigning key: 10.0.183.137 to cloud-private-ip-config workqueue
I1117 20:07:30.407246       1 cloudprivateipconfig_controller.go:328] CloudPrivateIPConfig: 10.0.183.137 object has been marked for complete deletion
I1117 20:07:30.407282       1 cloudprivateipconfig_controller.go:335] Cleaning up IP address and finalizer for CloudPrivateIPConfig: "10.0.183.137", deleting it completely
I1117 20:07:30.419548       1 controller.go:182] Assigning key: 10.0.183.137 to cloud-private-ip-config workqueue
I1117 20:07:30.419557       1 controller.go:160] Dropping key '10.0.183.137' from the cloud-private-ip-config workqueue
I1117 20:07:30.423730       1 cloudprivateipconfig_controller.go:507] CloudPrivateIPConfig: "10.0.183.137" in work queue no longer exists
I1117 20:07:30.423758       1 controller.go:160] Dropping key '10.0.183.137' from the cloud-private-ip-config workqueue
 

After restart of ovnkube-node of egressNode, egressip is not assigned any more

egressIP should still be reassigned