NAME         READY   STATUS              RESTARTS   AGE
dpdk-6rznq   0/1     ContainerCreating   0          3m11s
dpdk-f67pm   1/1     Running             0          3m13s 

Name: dpdk-6rznq Namespace: dpdk-testing Priority: 0 Service Account: default Node: worker-0/10.46.77.2 Start Time: Fri, 06 Dec 2024 11:34:59 -0500 Labels: app=dpdk Annotations: k8s.ovn.org/pod-networks: {"default":{"ip_addresses":["10.131.0.34/23"],"mac_address":"0a:58:0a:83:00:22","gateway_ips":["10.131.0.1"],"routes":[{"dest":"10.128.0.0... k8s.v1.cni.cncf.io/networks: [{"name": "test-dpdk-network1", "mac": "60:00:00:00:00:01", "namespace": "dpdk-testing"}, {"name": "test-tap-network", "interface": "tap23... {"name": "test-tap-vlan100-network", "interface": "tap23.100", "namespace": "dpdk-testing"}, {"name": "test-tap-vlan200-network", "i... openshift.io/scc: privileged seccomp.security.alpha.kubernetes.io/pod: runtime/default Status: Pending SeccompProfile: RuntimeDefault IP: IPs: <none> Containers: dpdk: Container ID: Image: registry.hlxcl1.lab.eng.tlv2.redhat.com:5000/dpdk-base:v4.18.0-3 Image ID: Port: <none> Host Port: <none> Command: /bin/bash -c dpdk-testpmd --vdev=virtio_user0,path=/dev/vhost-net,queues=2,queue_size=1024,iface=tap23 -a ${PCIDEVICE_OPENSHIFT_IO_DPDKNET1} -- --stats-period 5 sleep INF State: Waiting Reason: ContainerCreating Ready: False Restart Count: 0 Limits: cpu: 4 hugepages-1Gi: 2Gi memory: 1Gi openshift.io/dpdknet1: 1 Requests: cpu: 4 hugepages-1Gi: 2Gi memory: 1Gi openshift.io/dpdknet1: 1 Environment: RUN_TYPE: testpmd CONTAINER_NAME: dpdk Mounts: /etc/podnetinfo from podnetinfo (ro) /mnt/huge from hugepages (rw) /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-24kbz (ro) Conditions: Type Status PodReadyToStartContainers False Initialized True Ready False ContainersReady False PodScheduled True Volumes: hugepages: Type: EmptyDir (a temporary directory that shares a pod's lifetime) Medium: HugePages SizeLimit: <unset> kube-api-access-24kbz: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: <nil> DownwardAPI: true ConfigMapName: openshift-service-ca.crt ConfigMapOptional: <nil> podnetinfo: Type: DownwardAPI (a volume populated by information about the pod) Items: metadata.labels -> labels metadata.annotations -> annotations requests.hugepages-1Gi -> hugepages_1G_request_dpdk limits.hugepages-1Gi -> hugepages_1G_limit_dpdk QoS Class: Guaranteed Node-Selectors: node-role.kubernetes.io/workercnf= Tolerations: node.kubernetes.io/memory-pressure:NoSchedule op=Exists node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled 42s default-scheduler Successfully assigned dpdk-testing/dpdk-6rznq to worker-0 Normal AddedInterface 42s multus Add net1 [1.1.1.2/24] from dpdk-testing/test-dpdk-network1 Warning FailedCreatePodSandBox 42s kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_dpdk-6rznq_dpdk-testing_2b29c0b8-167f-4ba5-adfd-55fc8c48bf65_0(2be116283c03427b899abfa50126fb50a349dc6e85ece3be47ee85cb633e198e): error adding pod dpdk-testing_dpdk-6rznq to CNI network "multus-cni-network": plugin type="multus-shim" name="multus-cni-network" failed (add): CmdAdd (shim): CNI request failed with status 400: 'ContainerID:"2be116283c03427b899abfa50126fb50a349dc6e85ece3be47ee85cb633e198e" Netns:"/var/run/netns/a0e02175-5305-4528-8c2d-328107b06ac1" IfName:"eth0" Args:"IgnoreUnknown=1;K8S_POD_NAMESPACE=dpdk-testing;K8S_POD_NAME=dpdk-6rznq;K8S_POD_INFRA_CONTAINER_ID=2be116283c03427b899abfa50126fb50a349dc6e85ece3be47ee85cb633e198e;K8S_POD_UID=2b29c0b8-167f-4ba5-adfd-55fc8c48bf65" Path:"" ERRORED: error configuring pod [dpdk-testing/dpdk-6rznq] networking: [dpdk-testing/dpdk-6rznq/2b29c0b8-167f-4ba5-adfd-55fc8c48bf65:test-tap-network]: error adding container to network "test-tap-network": plugin type="tap" failed (add): failed to run command open: Operation not permitted : exit status 1 ': StdinData: {"binDir":"/var/lib/cni/bin","clusterNetwork":"/host/run/multus/cni/net.d/10-ovn-kubernetes.conf","cniVersion":"0.3.1","daemonSocketDir":"/run/multus/socket","globalNamespaces":"default,openshift-multus,openshift-sriov-network-operator,openshift-cnv","logLevel":"verbose","logToStderr":true,"name":"multus-cni-network","namespaceIsolation":true,"type":"multus-shim"} Normal AddedInterface 41s multus Add net1 [1.1.1.3/24] from dpdk-testing/test-dpdk-network1 Warning FailedCreatePodSandBox 41s kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_dpdk-6rznq_dpdk-testing_2b29c0b8-167f-4ba5-adfd-55fc8c48bf65_0(d2e1439664f0810f2ccac10a8b538cee7ccb36e92b62e819120c291622b27efa): error adding pod dpdk-testing_dpdk-6rznq to CNI network "multus-cni-network": plugin type="multus-shim" name="multus-cni-network" failed (add): CmdAdd (shim): CNI request failed with status 400: 'ContainerID:"d2e1439664f0810f2ccac10a8b538cee7ccb36e92b62e819120c291622b27efa" Netns:"/var/run/netns/c029eb38-2d73-456c-8125-1dcb5072e269" IfName:"eth0" Args:"IgnoreUnknown=1;K8S_POD_NAMESPACE=dpdk-testing;K8S_POD_NAME=dpdk-6rznq;K8S_POD_INFRA_CONTAINER_ID=d2e1439664f0810f2ccac10a8b538cee7ccb36e92b62e819120c291622b27efa;K8S_POD_UID=2b29c0b8-167f-4ba5-adfd-55fc8c48bf65" Path:"" ERRORED: error configuring pod [dpdk-testing/dpdk-6rznq] networking: [dpdk-testing/dpdk-6rznq/2b29c0b8-167f-4ba5-adfd-55fc8c48bf65:test-tap-network]: error adding container to network "test-tap-network": plugin type="tap" failed (add): failed to run command open: Operation not permitted : exit status 1 ': StdinData: {"binDir":"/var/lib/cni/bin","clusterNetwork":"/host/run/multus/cni/net.d/10-ovn-kubernetes.conf","cniVersion":"0.3.1","daemonSocketDir":"/run/multus/socket","globalNamespaces":"default,openshift-multus,openshift-sriov-network-operator,openshift-cnv","logLevel":"verbose","logToStderr":true,"name":"multus-cni-network","namespaceIsolation":true,"type":"multus-shim"} Normal AddedInterface 27s multus Add net1 [1.1.1.4/24] from dpdk-testing/test-dpdk-network1 ': StdinData: {"binDir":"/var/lib/cni/bin","clusterNetwork":"/host/run/multus/cni/net.d/10-ovn-kubernetes.conf","cniVersion":"0.3.1","daemonSocketDir":"/run/multus/socket","globalNamespaces":"default,openshift-multus,openshift-sriov-network-operator,openshift-cnv","logLevel":"verbose","logToStderr":true,"name":"multus-cni-network","namespaceIsolation":true,"type":"multus-shim"} Normal AddedInterface 12s multus Add net1 [1.1.1.5/24] from dpdk-testing/test-dpdk-network1 Warning FailedCreatePodSandBox 12s kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_dpdk-6rznq_dpdk-testing_2b29c0b8-167f-4ba5-adfd-55fc8c48bf65_0(74695d159f3102d9d908077d70d1529505dc37a64d16b9f6e60499c114106b5d): error adding pod dpdk-testing_dpdk-6rznq to CNI network "multus-cni-network": plugin type="multus-shim" name="multus-cni-network" failed (add): CmdAdd (shim): CNI request failed with status 400: 'ContainerID:"74695d159f3102d9d908077d70d1529505dc37a64d16b9f6e60499c114106b5d" Netns:"/var/run/netns/948ba148-b4b2-44e3-bbe7-0bd470b12245" IfName:"eth0" Args:"IgnoreUnknown=1;K8S_POD_NAMESPACE=dpdk-testing;K8S_POD_NAME=dpdk-6rznq;K8S_POD_INFRA_CONTAINER_ID=74695d159f3102d9d908077d70d1529505dc37a64d16b9f6e60499c114106b5d;K8S_POD_UID=2b29c0b8-167f-4ba5-adfd-55fc8c48bf65" Path:"" ERRORED: error configuring pod [dpdk-testing/dpdk-6rznq] networking: [dpdk-testing/dpdk-6rznq/2b29c0b8-167f-4ba5-adfd-55fc8c48bf65:test-tap-network]: error adding container to network "test-tap-network": plugin type="tap" failed (add): failed to run command open: Operation not permitted : exit status 1 ': StdinData: {"binDir":"/var/lib/cni/bin","clusterNetwork":"/host/run/multus/cni/net.d/10-ovn-kubernetes.conf","cniVersion":"0.3.1","daemonSocketDir":"/run/multus/socket","globalNamespaces":"default,openshift-multus,openshift-sriov-network-operator,openshift-cnv","logLevel":"verbose","logToStderr":true,"name":"multus-cni-network","namespaceIsolation":true,"type":"multus-shim"} Normal AddedInterface 1s (x5 over 42s) multus Add eth0 [10.131.0.34/23] from ovn-kubernetes Normal AddedInterface 1s multus Add net1 [1.1.1.6/24] from dpdk-testing/test-dpdk-network1 Warning FailedCreatePodSandBox 1s kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_dpdk-6rznq_dpdk-testing_2b29c0b8-167f-4ba5-adfd-55fc8c48bf65_0(cca7e786dfc187c46e7a2718f22dcfcd03cb03331c61d0b1728fd628c6fba4dc): error adding pod dpdk-testing_dpdk-6rznq to CNI network "multus-cni-network": plugin type="multus-shim" name="multus-cni-network" failed (add): CmdAdd (shim): CNI request failed with status 400: 'ContainerID:"cca7e786dfc187c46e7a2718f22dcfcd03cb03331c61d0b1728fd628c6fba4dc" Netns:"/var/run/netns/f7b94cfa-0442-409f-8408-a9ff19e75ad9" IfName:"eth0" Args:"IgnoreUnknown=1;K8S_POD_NAMESPACE=dpdk-testing;K8S_POD_NAME=dpdk-6rznq;K8S_POD_INFRA_CONTAINER_ID=cca7e786dfc187c46e7a2718f22dcfcd03cb03331c61d0b1728fd628c6fba4dc;K8S_POD_UID=2b29c0b8-167f-4ba5-adfd-55fc8c48bf65" Path:"" ERRORED: error configuring pod [dpdk-testing/dpdk-6rznq] networking: [dpdk-testing/dpdk-6rznq/2b29c0b8-167f-4ba5-adfd-55fc8c48bf65:test-tap-network]: error adding container to network "test-tap-network": plugin type="tap" failed (add): failed to run command open: Operation not permitted : exit status 1 ': StdinData: {"binDir":"/var/lib/cni/bin","clusterNetwork":"/host/run/multus/cni/net.d/10-ovn-kubernetes.conf","cniVersion":"0.3.1","daemonSocketDir":"/run/multus/socket","globalNamespaces":"default,openshift-multus,openshift-sriov-network-operator,openshift-cnv","logLevel":"verbose","logToStderr":true,"name":"multus-cni-network","namespaceIsolation":true,"type":"multus-shim"}
 

$ oc logs --all-containers -n openshift-multus multus-gbxf2 2024-12-06T16:44:18Z [verbose] Del: dpdk-testing:dpdk-6rznq:unknownUID:test-tap-network:tap23 {"cniVersion":"0.4.0","name":"test-tap-network","plugins":[{ "type": "tap", "selinuxcontext": "system_u:system_r:container_t:s0", "multiQueue": true}]} 2024-12-06T16:44:18Z [verbose] Del: dpdk-testing:dpdk-6rznq:unknownUID:test-dpdk-network1:net1 {"cniVersion":"1.0.0","deviceID":"0000:86:01.1","ipam":{"dataDir":"/run/my-orchestrator/container-ipam-state","ranges":[[{"subnet":"1.1.1.0/24"}]],"type":"host-local"},"link_state":"enable","logLevel":"info","name":"test-dpdk-network1","pciBusID":"0000:86:01.1","type":"sriov","vlan":0,"vlanQoS":0} 2024-12-06T16:44:18Z [error] error in getting result from DelNetwork: failed to get netlink device with name net1: "Link not found" 2024-12-06T16:44:18Z [error] DelegateDel: error invoking DelegateDel - "sriov": error in getting result from DelNetwork: failed to get netlink device with name net1: "Link not found" 2024-12-06T16:44:18Z [verbose] Del: dpdk-testing:dpdk-6rznq:unknownUID:ovn-kubernetes:eth0 {"cniVersion":"0.4.0","name":"ovn-kubernetes","type":"ovn-k8s-cni-overlay","ipam":{},"dns":{},"logFile":"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log","logLevel":"4","logfile-maxsize":100,"logfile-maxbackups":5,"logfile-maxage":0,"runtimeConfig":{}} 2024-12-06T16:44:18Z [error] [dpdk-testing/dpdk-6rznq/2b29c0b8-167f-4ba5-adfd-55fc8c48bf65:test-tap-network]: error adding container to network "test-tap-network": plugin type="tap" failed (add): failed to run command open: Operation not permitted : exit status 1 2024-12-06T16:44:18Z [verbose] ADD finished CNI request ContainerID:"18b5dc1edaec254917ef0122635b909b731da353d4ce17acfd04db0b599665e5" Netns:"/var/run/netns/6e15fc77-fed5-4733-9210-ab6950c374ed" IfName:"eth0" Args:"IgnoreUnknown=1;K8S_POD_NAMESPACE=dpdk-testing;K8S_POD_NAME=dpdk-6rznq;K8S_POD_INFRA_CONTAINER_ID=18b5dc1edaec254917ef0122635b909b731da353d4ce17acfd04db0b599665e5;K8S_POD_UID=2b29c0b8-167f-4ba5-adfd-55fc8c48bf65" Path:"", result: "", err: error configuring pod [dpdk-testing/dpdk-6rznq] networking: [dpdk-testing/dpdk-6rznq/2b29c0b8-167f-4ba5-adfd-55fc8c48bf65:test-tap-network]: error adding container to network "test-tap-network": plugin type="tap" failed (add): failed to run command open: Operation not permitted : exit status 1 2024-12-06T16:44:18Z [verbose] DEL starting CNI request ContainerID:"18b5dc1edaec254917ef0122635b909b731da353d4ce17acfd04db0b599665e5" Netns:"/var/run/netns/6e15fc77-fed5-4733-9210-ab6950c374ed" IfName:"eth0" Args:"IgnoreUnknown=1;K8S_POD_NAMESPACE=dpdk-testing;K8S_POD_NAME=dpdk-6rznq;K8S_POD_INFRA_CONTAINER_ID=18b5dc1edaec254917ef0122635b909b731da353d4ce17acfd04db0b599665e5;K8S_POD_UID=2b29c0b8-167f-4ba5-adfd-55fc8c48bf65" Path:"" 2024-12-06T16:44:18Z [verbose] Del: dpdk-testing:dpdk-6rznq:2b29c0b8-167f-4ba5-adfd-55fc8c48bf65:test-tap-vlan200-network:tap23.200 {"cniVersion":"0.4.0","name":"test-tap-vlan200-network","plugins":[{"ipam": {"type": "host-local", "subnet": "1.1.1.0/24"}, "type": "vlan", "master": "tap23", "vlanId": 200, "linkInContainer": true}]} 2024-12-06T16:44:18Z [error] conflistDel: error in getting result from DelNetworkList: plugin type="vlan" failed (delete): Link not found 2024-12-06T16:44:18Z [error] DelegateDel: error invoking ConflistDel - "test-tap-vlan200-network": conflistDel: error in getting result from DelNetworkList: plugin type="vlan" failed (delete): Link not found 2024-12-06T16:44:18Z [verbose] Del: dpdk-testing:dpdk-6rznq:2b29c0b8-167f-4ba5-adfd-55fc8c48bf65:test-tap-vlan100-network:tap23.100 {"cniVersion":"0.4.0","name":"test-tap-vlan100-network","plugins":[{"ipam": {"type": "host-local", "subnet": "1.1.1.0/24"}, "type": "vlan", "master": "tap23", "vlanId": 100, "linkInContainer": true}]} 2024-12-06T16:44:18Z [error] conflistDel: error in getting result from DelNetworkList: plugin type="vlan" failed (delete): Link not found 2024-12-06T16:44:18Z [error] DelegateDel: error invoking ConflistDel - "test-tap-vlan100-network": conflistDel: error in getting result from DelNetworkList: plugin type="vlan" failed (delete): Link not found 2024-12-06T16:44:18Z [verbose] Del: dpdk-testing:dpdk-6rznq:2b29c0b8-167f-4ba5-adfd-55fc8c48bf65:test-tap-network:tap23 {"cniVersion":"0.4.0","name":"test-tap-network","plugins":[{ "type": "tap", "selinuxcontext": "system_u:system_r:container_t:s0", "multiQueue": true}]} 2024-12-06T16:44:18Z [verbose] Del: dpdk-testing:dpdk-6rznq:2b29c0b8-167f-4ba5-adfd-55fc8c48bf65:test-dpdk-network1:net1 {"cniVersion":"1.0.0","deviceID":"0000:86:01.1","ipam":{"dataDir":"/run/my-orchestrator/container-ipam-state","ranges":[[{"subnet":"1.1.1.0/24"}]],"type":"host-local"},"link_state":"enable","logLevel":"info","name":"test-dpdk-network1","pciBusID":"0000:86:01.1","type":"sriov","vlan":0,"vlanQoS":0} time="2024-12-06T16:44:18.634010554Z" level="error" msg="Cannot load config file from cache" cniName="sriov-cni" containerID="18b5dc1edaec254917ef0122635b909b731da353d4ce17acfd04db0b599665e5" netns="/var/run/netns/6e15fc77-fed5-4733-9210-ab6950c374ed" ifname="net1" func="cmdDel" err="error reading cached NetConf in /var/lib/cni/sriov with name 18b5dc1edaec254917ef0122635b909b731da353d4ce17acfd04db0b599665e5-net1" 2024-12-06T16:44:18Z [verbose] Del: dpdk-testing:dpdk-6rznq:2b29c0b8-167f-4ba5-adfd-55fc8c48bf65:ovn-kubernetes:eth0 {"cniVersion":"0.4.0","name":"ovn-kubernetes","type":"ovn-k8s-cni-overlay","ipam":{},"dns":{},"logFile":"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log","logLevel":"4","logfile-maxsize":100,"logfile-maxbackups":5,"logfile-maxage":0,"runtimeConfig":{}} 2024-12-06T16:44:18Z [verbose] DEL finished CNI request ContainerID:"18b5dc1edaec254917ef0122635b909b731da353d4ce17acfd04db0b599665e5" Netns:"/var/run/netns/6e15fc77-fed5-4733-9210-ab6950c374ed" IfName:"eth0" Args:"IgnoreUnknown=1;K8S_POD_NAMESPACE=dpdk-testing;K8S_POD_NAME=dpdk-6rznq;K8S_POD_INFRA_CONTAINER_ID=18b5dc1edaec254917ef0122635b909b731da353d4ce17acfd04db0b599665e5;K8S_POD_UID=2b29c0b8-167f-4ba5-adfd-55fc8c48bf65" Path:"", result: "", err: DelegateDel: error invoking ConflistDel - "test-tap-vlan200-network": conflistDel: error in getting result from DelNetworkList: plugin type="vlan" failed (delete): Link not found / DelegateDel: error invoking ConflistDel - "test-tap-vlan100-network": conflistDel: error in getting result from DelNetworkList: plugin type="vlan" failed (delete): Link not found