https://issues.redhat.com/browse/NETOBSERV-1283 Not able to monitor Multus/SRIOV traffic on Network Observability Operator On an SRIOV environment (borrowed from Beijing team BM) with appropriate hardware: [root@dell-per740-36 demo]# oc get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES network-resources-injector-6685q 1/1 Running 0 9h 10.128.0.94 dell-per740-13.rhts.eng.pek2.redhat.com network-resources-injector-ck5vr 1/1 Running 0 9h 10.130.0.59 dell-per740-32.rhts.eng.pek2.redhat.com network-resources-injector-zdsgg 1/1 Running 0 9h 10.129.0.83 dell-per740-31.rhts.eng.pek2.redhat.com operator-webhook-696z7 1/1 Running 0 9h 10.129.0.82 dell-per740-31.rhts.eng.pek2.redhat.com operator-webhook-8fq7t 1/1 Running 0 9h 10.128.0.93 dell-per740-13.rhts.eng.pek2.redhat.com operator-webhook-gw4d9 1/1 Running 0 9h 10.130.0.58 dell-per740-32.rhts.eng.pek2.redhat.com sriov-device-plugin-nc55l 1/1 Running 0 5h52m 10.73.116.62 dell-per740-14.rhts.eng.pek2.redhat.com sriov-network-config-daemon-kzjvc 1/1 Running 0 9h 10.73.116.62 dell-per740-14.rhts.eng.pek2.redhat.com sriov-network-config-daemon-lwmxg 1/1 Running 0 9h 10.73.116.54 dell-per740-35.rhts.eng.pek2.redhat.com sriov-network-operator-85b76c9886-xrhx9 1/1 Running 0 9h 10.128.0.92 dell-per740-13.rhts.eng.pek2.redhat.com [root@dell-per740-36 demo]# oc get sriovnetworknodestates -n openshift-sriov-network-operator NAME SYNC STATUS AGE dell-per740-14.rhts.eng.pek2.redhat.com Succeeded 9h dell-per740-35.rhts.eng.pek2.redhat.com Succeeded 9h [root@dell-per740-36 demo]# Created Virtual function: [root@dell-per740-36 memodi]# cat /root/testdata/demo/snnp.yaml apiVersion: sriovnetwork.openshift.io/v1 kind: SriovNetworkNodePolicy metadata: name: intel-netdevice namespace: openshift-sriov-network-operator spec: deviceType: netdevice nicSelector: pfNames: - ens1f2#1-2 - ens1f0#1-3 vendor: '8086' nodeSelector: feature.node.kubernetes.io/sriov-capable: 'true' numVfs: 4 priority: 99 resourceName: intelnetdevice [root@dell-per740-36 memodi]# Created SRIOVNetwork so that it can create net-attach-def (NAD) [root@dell-per740-36 memodi]# cat sriov-net-attach-pod-1.yaml sriov-net-attach-pod-2.yaml apiVersion: sriovnetwork.openshift.io/v1 kind: SriovNetwork metadata: name: intel-netdevice-test spec: resourceName: intelnetdevice ipam: '{ "type": "static", "addresses": [{"address": "192.168.122.71/24"}]}' apiVersion: sriovnetwork.openshift.io/v1 kind: SriovNetwork metadata: name: intel-netdevice-test-1 spec: resourceName: intelnetdevice ipam: '{ "type": "static", "addresses": [{"address": "192.168.122.72/24"}]}' [root@dell-per740-36 memodi]# The resourceName should match between SriovNetwork and SriovNetworkNodePolicy. [root@dell-per740-36 demo]# oc get sriovnetwork NAME AGE intel-netdevice 5h33m intel-netdevice-test 3h22m intel-netdevice-test-1 3h [root@dell-per740-36 demo]# Note each pod needs to be attached to unique (NAD) Created 2 test pods with different netns: [root@dell-per740-36 demo]# cat pod.yaml apiVersion: v1 kind: Pod metadata: generateName: testpod1 labels: env: test annotations: k8s.v1.cni.cncf.io/networks: intel-netdevice-test spec: containers: - name: test-pod image: quay.io/openshifttest/hello-sdn@sha256:d5785550cf77b7932b090fcd1a2625472912fb3189d5973f177a5a2c347a1f95 [root@dell-per740-36 demo]# cat pod.yaml apiVersion: v1 kind: Pod metadata: generateName: testpod1 labels: env: test annotations: k8s.v1.cni.cncf.io/networks: intel-netdevice-test-1 spec: containers: - name: test-pod image: quay.io/openshifttest/hello-sdn@sha256:d5785550cf77b7932b090fcd1a2625472912fb3189d5973f177a5a2c347a1f95 [root@dell-per740-36 demo]# oc get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES network-resources-injector-6685q 1/1 Running 0 9h 10.128.0.94 dell-per740-13.rhts.eng.pek2.redhat.com network-resources-injector-ck5vr 1/1 Running 0 9h 10.130.0.59 dell-per740-32.rhts.eng.pek2.redhat.com network-resources-injector-zdsgg 1/1 Running 0 9h 10.129.0.83 dell-per740-31.rhts.eng.pek2.redhat.com operator-webhook-696z7 1/1 Running 0 9h 10.129.0.82 dell-per740-31.rhts.eng.pek2.redhat.com operator-webhook-8fq7t 1/1 Running 0 9h 10.128.0.93 dell-per740-13.rhts.eng.pek2.redhat.com operator-webhook-gw4d9 1/1 Running 0 9h 10.130.0.58 dell-per740-32.rhts.eng.pek2.redhat.com sriov-device-plugin-nc55l 1/1 Running 0 5h52m 10.73.116.62 dell-per740-14.rhts.eng.pek2.redhat.com sriov-network-config-daemon-kzjvc 1/1 Running 0 9h 10.73.116.62 dell-per740-14.rhts.eng.pek2.redhat.com sriov-network-config-daemon-lwmxg 1/1 Running 0 9h 10.73.116.54 dell-per740-35.rhts.eng.pek2.redhat.com sriov-network-operator-85b76c9886-xrhx9 1/1 Running 0 9h 10.128.0.92 dell-per740-13.rhts.eng.pek2.redhat.com testpod12plq4 1/1 Running 0 60m 10.128.2.84 dell-per740-14.rhts.eng.pek2.redhat.com testpod1h7tc5 1/1 Running 0 106m 10.128.2.82 dell-per740-14.rhts.eng.pek2.redhat.com testpod1j64hl 1/1 Running 0 115m 10.128.2.81 dell-per740-14.rhts.eng.pek2.redhat.com [root@dell-per740-36 memodi]# oc rsh testpod1h7tc5 / # ip a 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0@if109: mtu 1400 qdisc noqueue state UP group default qlen 1000 link/ether 0a:58:0a:80:02:52 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.128.2.82/23 brd 10.128.3.255 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::858:aff:fe80:252/64 scope link valid_lft forever preferred_lft forever 73: net1: mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 06:d8:93:db:56:9a brd ff:ff:ff:ff:ff:ff altname enp59s0f0v1 inet 192.168.122.72/24 brd 192.168.122.255 scope global net1 valid_lft forever preferred_lft forever inet6 fe80::4d8:93ff:fedb:569a/64 scope link valid_lft forever preferred_lft forever Notice net1 inet IP address. From another test pod do "ping 192.168.122.72" or curl "192.168.122.72:8080" On UI when filtered on network interface=net1, For curl request an egress flow looked like: { "AgentIP": "10.73.116.62", "Bytes": 406, "DstAddr": "192.168.122.72", "DstMac": "06:D8:93:DB:56:9A", "DstPort": 60256, "Duplicate": false, "Etype": 2048, "Flags": 784, "FlowDirection": "1", "IfDirection": 1, "Interface": "net1", "K8S_ClusterName": "5846f979-9d5b-4a1b-b08d-aee52e6b58a2", "Packets": 4, "Proto": 6, "SrcAddr": "192.168.122.71", "SrcMac": "AE:39:F4:7E:03:FE", "SrcPort": 8080, "TimeFlowEndMs": 1694120614010, "TimeFlowStartMs": 1694120614009, "TimeReceived": 1694120618, "app": "netobserv-flowcollector" } Ingress flow looked like: { "AgentIP": "10.73.116.62", "Bytes": 487, "DstAddr": "192.168.122.71", "DstMac": "AE:39:F4:7E:03:FE", "DstPort": 8080, "Duplicate": false, "Etype": 2048, "Flags": 530, "FlowDirection": "0", "IfDirection": 0, "Interface": "net1", "K8S_ClusterName": "5846f979-9d5b-4a1b-b08d-aee52e6b58a2", "Packets": 6, "Proto": 6, "SrcAddr": "192.168.122.72", "SrcMac": "06:D8:93:DB:56:9A", "SrcPort": 60256, "TimeFlowEndMs": 1694120614010, "TimeFlowStartMs": 1694120614009, "TimeReceived": 1694120618, "app": "netobserv-flowcollector" } Note if you don't filter on network interface=net1, you may see OVN flows from different interface if requests are made on Pod IP, however for this scenario we're interested in SRIOV interface. All the fields were showing as n/a like Namespace, pod since FLP couldn't enrich it on the network interface level. Tests were run with image: quay.io/netobserv/netobserv-ebpf-agent:2ef08da on PR https://github.com/netobserv/netobserv-ebpf-agent/pull/171 And Operator PR https://github.com/netobserv/network-observability-operator/pull/406 quay.io/netobserv/network-observability-operator:52588e3 Ran selected regression tests with ebpf.privileged=true: ./bin/extended-platform-tests run all --dry-run |grep -E "50504|49107|60701|63839|56362" |./bin/extended-platform-tests run --timeout 30m -f - Sep 7 17:29:49.654: INFO: The --provider flag is not set. Continuing as if --provider=skeleton had been used. started: (0/1/5) "[sig-netobserv] Network_Observability FLP, Console metrics: when process.metrics.TLS == DISABLED Author:aramesha-High-50504-Verify flowlogs-pipeline metrics and health [Serial]" passed: (5m39s) 2023-09-07T21:35:29 "[sig-netobserv] Network_Observability FLP, Console metrics: when process.metrics.TLS == DISABLED Author:aramesha-High-50504-Verify flowlogs-pipeline metrics and health [Serial]" started: (0/2/5) "[sig-netobserv] Network_Observability with KAFKA NonPreRelease-Longduration-Author:aramesha-High-56362-High-53597-High-56326-Verify network flows are captured with Kafka with TLS [Serial]" passed: (9m38s) 2023-09-07T21:45:06 "[sig-netobserv] Network_Observability with KAFKA NonPreRelease-Longduration-Author:aramesha-High-56362-High-53597-High-56326-Verify network flows are captured with Kafka with TLS [Serial]" started: (0/3/5) "[sig-netobserv] Network_Observability NonPreRelease-Longduration-Author:aramesha-High-60701-Verify connection tracking [Serial]" passed: (8m43s) 2023-09-07T21:53:49 "[sig-netobserv] Network_Observability NonPreRelease-Longduration-Author:aramesha-High-60701-Verify connection tracking [Serial]" started: (0/4/5) "[sig-netobserv] Network_Observability NonPreRelease-Longduration-Author:memodi-High-63839-Verify-multi-tenancy [Disruptive] [Serial][Slow]" passed: (13m20s) 2023-09-07T22:07:09 "[sig-netobserv] Network_Observability NonPreRelease-Longduration-Author:memodi-High-63839-Verify-multi-tenancy [Disruptive] [Serial][Slow]" started: (0/5/5) "[sig-netobserv] Network_Observability Author:memodi-High-53595-High-49107-High-45304-High-54929-High-54840-Verify flow correctness [Serial]" passed: (6m39s) 2023-09-07T22:13:48 "[sig-netobserv] Network_Observability Author:memodi-High-53595-High-49107-High-45304-High-54929-High-54840-Verify flow correctness [Serial]" 5 pass, 0 skip (43m59s)