Multi-egress source route entries do not get properly updated with adminpolicybasedexternalroutes CR

Upstream ovn-kubernetes commit c60963123d28075288a8c23d2796c2df89f54601

Create a served/application pod after creating the adminpolicybasedexternalroutes CR. The corresponding source route entries wont be added to the worker routing table

1. Create a ovn-kubernetes kind cluster:
./kind.sh --install-cni-plugins --disable-snat-multiple-gws --multi-network-enable
2. Create two namespaces:
$ cat <<EOF | kubectl apply -f -
---
apiVersion: v1
kind: Namespace
metadata:
  name: frr
  labels:     
    gws: "true"
spec: {}
---
apiVersion: v1
kind: Namespace
metadata:
  name: bar
  labels:
    multiple_gws: "true"
spec: {}
EOF

3. Create a network attachment definition:
$ cat <<EOF | kubectl apply -f -
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
  name: internal-net
  namespace: frr
spec:
  config: |-
    {
      "cniVersion": "0.3.1",
      "name": "internal-net",
      "plugins": [
        {
          "type": "macvlan",
          "master": "breth0",
          "mode": "bridge",
          "ipam": {
            "type": "static"
          }
        },
        {
          "capabilities": {
            "mac": true,
            "ips": true
          },
          "type": "tuning"
        }
      ]
    }
EOF

4. Create the first dummy pod:
$ cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: dummy1
  namespace: bar
spec:
  containers:
  - name: dummy
    image: centos
    command:
      - sleep
      - infinity
  nodeSelector:
    kubernetes.io/hostname: ovn-worker2
EOF

5. Create the AdminPolicyBasedExternalRoute CR:
$ cat <<EOF | kubectl apply -f -
apiVersion: k8s.ovn.org/v1
kind: AdminPolicyBasedExternalRoute
metadata:
  name: honeypotting
spec:
## gateway example
  from:
    namespaceSelector:
      matchLabels:
          multiple_gws: "true"
  nextHops:       
    dynamic:
      - podSelector:
          matchLabels:
            gw: "true"
        bfdEnabled: true
        namespaceSelector:
          matchLabels:
            gws: "true"
        networkAttachmentName: frr/internal-net
EOF

6. Create the lb pod:
$ cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: ext-gw
  labels:
    gw: "true"
  namespace: frr
  annotations:
    k8s.v1.cni.cncf.io/networks: '[
        {
          "name": "internal-net",
          "ips": [ "172.18.0.10/16" ]
        }
      ]'
spec:
  containers:
  - name: frr
    image: centos
    command:
      - sleep
      - infinity
    securityContext:
      privileged: true
  nodeSelector:
    kubernetes.io/hostname: ovn-worker
EOF

7. Create a second dummy pod:
$ cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: dummy2
  namespace: bar
spec:
  containers:
  - name: dummy
    image: centos
    command:
      - sleep
      - infinity
  nodeSelector:
    kubernetes.io/hostname: ovn-worker2
EOF

Only source route entries for the first dummy pod were created:

$ kubectl get po -o wide -n bar
dummy1   Running  10.244.1.3
dummy2   Running  10.244.1.4

$ POD=$(kubectl get pod -n ovn-kubernetes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep ovnkube-db-) ; kubectl exec -ti $POD -n ovn-kubernetes -c nb-ovsdb -- bash

[root@ovn-control-plane ~]# ovn-nbctl lr-route-list GR_ovn-worker2
IPv4 Routes
Route Table <main>:
               10.244.1.3               172.18.0.10 src-ip exgw-rtoe-GR_ovn-worker2 ecmp-symmetric-reply bfd
         169.254.169.0/29             169.254.169.4 dst-ip rtoe-GR_ovn-worker2
            10.244.0.0/16                100.64.0.1 dst-ip
                0.0.0.0/0                172.18.0.1 dst-ip rtoe-GR_ovn-worker

Source route entries for both dummy pods created:
[root@ovn-control-plane ~]# ovn-nbctl lr-route-list GR_ovn-worker2
IPv4 Routes
Route Table <main>:
               10.244.1.3               172.18.0.10 src-ip exgw-rtoe-GR_ovn-worker2 ecmp-symmetric-reply bfd
               10.244.1.4               172.18.0.10 src-ip exgw-rtoe-GR_ovn-worker2 ecmp-symmetric-reply bfd
          169.254.169.0/29             169.254.169.4 dst-ip rtoe-GR_ovn-worker2
            10.244.0.0/16                100.64.0.1 dst-ip
                0.0.0.0/0                172.18.0.1 dst-ip rtoe-GR_ovn-worke

$ kubectl describe adminpolicybasedexternalroutes
...
Status:
  Last Transition Time:  2023-09-25T09:50:25Z
  Messages:
    Configured external gateway IPs: 172.18.0.10
    Status:  Success
Events:    <none>