IPv6 LoadBalancer type UDP service drops first packets to each node when deploying a UDP LB type service in a dualstack IPv6 primary OCP on OSP cluster.

OCP: 4.15.0-0.nightly-2024-04-23-013140
(OSP: 17.1.2)

Always

1. Deploy a dual-stack cluster with IPv6 primary
2. Create the UDP LB type service
 $ oc apply -f upd_lb_svc.yaml
 project.project.openshift.io/udp-lb-default-ns created
 deployment.apps/udp-lb-default-dep created
 service/udp-lb-default-svc created
    
3. Wait until the LB is provisioned and get the ingress IP from the svc
 $ oc -n udp-lb-default-ns get svc udp-lb-default-svc -o json | jq -r .status.loadBalancer.ingress[].ip
fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd

4. Query the UDP service   

$ cat <(echo hostname) <(sleep 1) | nc -w 1 -6u fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd 8082 <--- no response

$ cat <(echo hostname) <(sleep 1) | nc -w 1 -6u fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd 8082 <--- no response

$ cat <(echo hostname) <(sleep 1) | nc -w 1 -6u fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd 8082 <--- no response

$ cat <(echo hostname) <(sleep 1) | nc -w 1 -6u fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd 8082     <--- no response

$ cat <(echo hostname) <(sleep 1) | nc -w 1 -6u fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd 8082 <--- no response

$ cat <(echo hostname) <(sleep 1) | nc -w 1 -6u fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd 8082 <--- no response

$ cat <(echo hostname) <(sleep 1) | nc -w 1 -6u fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd 8082
udp-lb-default-dep-5d8d7576d-h47r9

$ cat <(echo hostname) <(sleep 1) | nc -w 1 -6u fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd 8082 udp-lb-default-dep-5d8d7576d-h47r9

$ cat <(echo hostname) <(sleep 1) | nc -w 1 -6u fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd 8082 udp-lb-default-dep-5d8d7576d-h47r9

$ cat <(echo hostname) <(sleep 1) | nc -w 1 -6u fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd 8082 udp-lb-default-dep-5d8d7576d-xdz24

$ cat <(echo hostname) <(sleep 1) | nc -w 1 -6u fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd 8082 udp-lb-default-dep-5d8d7576d-h47r9

$ cat <(echo hostname) <(sleep 1) | nc -w 1 -6u fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd 8082 udp-lb-default-dep-5d8d7576d-xdz24


tcpdump in the LB:
# tcpdump udp port 31245
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on eth1, link-type EN10MB (Ethernet), snapshot length 262144 bytes
11:41:10.231707 IP6 amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.32820 > fd2e:6f44:5dd8:c956:f816:3eff:fe14:e5bf.31245: UDP, length 9
11:41:16.379092 IP6 amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.52115 > fd2e:6f44:5dd8:c956:f816:3eff:fe51:8d35.31245: UDP, length 9
11:41:19.721497 IP6 amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.48782 > fd2e:6f44:5dd8:c956:f816:3eff:fe44:3dea.31245: UDP, length 9
11:41:21.008263 IP6 amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.37361 > fd2e:6f44:5dd8:c956:f816:3eff:fec7:9d9a.31245: UDP, length 9
11:41:22.003352 IP6 amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.53123 > fd2e:6f44:5dd8:c956:f816:3eff:fed3:2dc0.31245: UDP, length 9
11:41:23.155893 IP6 amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.39247 > fd2e:6f44:5dd8:c956:f816:3eff:fe26:6ab6.31245: UDP, length 9
11:41:26.111973 IP6 amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.43342 > fd2e:6f44:5dd8:c956:f816:3eff:fe14:e5bf.31245: UDP, length 9
11:41:26.116436 IP6 fd2e:6f44:5dd8:c956:f816:3eff:fe14:e5bf.31245 > amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.43342: UDP, length 34
11:41:34.602099 IP6 amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.49155 > fd2e:6f44:5dd8:c956:f816:3eff:fe51:8d35.31245: UDP, length 9
11:41:34.607049 IP6 fd2e:6f44:5dd8:c956:f816:3eff:fe51:8d35.31245 > amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.49155: UDP, length 34
11:41:41.278928 IP6 amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.37445 > fd2e:6f44:5dd8:c956:f816:3eff:fe44:3dea.31245: UDP, length 9
11:41:41.283027 IP6 fd2e:6f44:5dd8:c956:f816:3eff:fe44:3dea.31245 > amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.37445: UDP, length 34
11:41:56.537646 IP6 amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.33843 > fd2e:6f44:5dd8:c956:f816:3eff:fec7:9d9a.31245: UDP, length 9
11:41:56.543145 IP6 fd2e:6f44:5dd8:c956:f816:3eff:fec7:9d9a.31245 > amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.33843: UDP, length 34
11:42:08.621785 IP6 amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.34508 > fd2e:6f44:5dd8:c956:f816:3eff:fed3:2dc0.31245: UDP, length 9
11:42:08.627659 IP6 fd2e:6f44:5dd8:c956:f816:3eff:fed3:2dc0.31245 > amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.34508: UDP, length 34
11:42:14.385176 IP6 amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.41900 > fd2e:6f44:5dd8:c956:f816:3eff:fe26:6ab6.31245: UDP, length 9
11:42:14.390605 IP6 fd2e:6f44:5dd8:c956:f816:3eff:fe26:6ab6.31245 > amphora-4e0de19d-faed-4dab-b51c-d2b6283011e0.41900: UDP, length 34

All the queries should be answered by the svc.

$ oc get network cluster -o yaml
apiVersion: config.openshift.io/v1
kind: Network
metadata:
  creationTimestamp: "2024-04-23T12:06:56Z"
  generation: 2
  name: cluster
  resourceVersion: "4267"
  uid: 0aa561a9-5bc7-4152-ad93-57a673b1a00b
spec:
  clusterNetwork:
  - cidr: fd01::/48
    hostPrefix: 64
  - cidr: 10.128.0.0/14
    hostPrefix: 23
  externalIP:
    policy: {}
  networkType: OVNKubernetes
  serviceNetwork:
  - fd02::/112
  - 172.30.0.0/16
status:
  clusterNetwork:
  - cidr: fd01::/48
    hostPrefix: 64
  - cidr: 10.128.0.0/14
    hostPrefix: 23
  clusterNetworkMTU: 1342
  networkType: OVNKubernetes
  serviceNetwork:
  - fd02::/112
  - 172.30.0.0/16


$ oc -n udp-lb-default-ns get svc udp-lb-default-svc -o yaml
apiVersion: v1
kind: Service
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app":"udp-lb-default-dep"},"name":"udp-lb-default-svc","namespace":"udp-lb-default-ns"},"spec":{"ports":[{"port":8082,"protocol":"UDP","targetPort":8081}],"selector":{"app":"udp-lb-default-dep"},"type":"LoadBalancer"}}
    loadbalancer.openstack.org/load-balancer-address: fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd
    loadbalancer.openstack.org/load-balancer-id: d81e66f3-42ec-49bc-816c-5a774df40f9d
  creationTimestamp: "2024-04-23T13:30:48Z"
  finalizers:
  - service.kubernetes.io/load-balancer-cleanup
  labels:
    app: udp-lb-default-dep
  name: udp-lb-default-svc
  namespace: udp-lb-default-ns
  resourceVersion: "53456"
  uid: 017e6cef-734e-46d0-80a3-70195aac56ec
spec:
  allocateLoadBalancerNodePorts: true
  clusterIP: fd02::71a5
  clusterIPs:
  - fd02::71a5
  externalTrafficPolicy: Cluster
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv6
  ipFamilyPolicy: SingleStack
  ports:
  - nodePort: 31245
    port: 8082
    protocol: UDP
    targetPort: 8081
  selector:
    app: udp-lb-default-dep
  sessionAffinity: None
  type: LoadBalancer
status:
  loadBalancer:
    ingress:
    - ip: fd2e:6f44:5dd8:c956:f816:3eff:fec9:52cd

$ oc -n udp-lb-default-ns get pods -o wide
NAME                                 READY   STATUS    RESTARTS   AGE    IP               NODE                          NOMINATED NODE   READINESS GATES
udp-lb-default-dep-5d8d7576d-h47r9   1/1     Running   0          132m   fd01:0:0:5::20   ostest-dgcjd-worker-0-c6ngr   <none>           <none>
udp-lb-default-dep-5d8d7576d-xdz24   1/1     Running   0          132m   fd01:0:0:6::17   ostest-dgcjd-worker-0-q82br   <none>           <none>


$ oc get nodes -o custom-columns=Name:.metadata.name,IPs:.status.addresses[].address
Name                          IPs
ostest-dgcjd-master-0         fd2e:6f44:5dd8:c956:f816:3eff:fe14:e5bf
ostest-dgcjd-master-1         fd2e:6f44:5dd8:c956:f816:3eff:fec7:9d9a
ostest-dgcjd-master-2         fd2e:6f44:5dd8:c956:f816:3eff:fe44:3dea
ostest-dgcjd-worker-0-75pj9   fd2e:6f44:5dd8:c956:f816:3eff:fe26:6ab6
ostest-dgcjd-worker-0-c6ngr   fd2e:6f44:5dd8:c956:f816:3eff:fe51:8d35
ostest-dgcjd-worker-0-q82br   fd2e:6f44:5dd8:c956:f816:3eff:fed3:2dc0