========================= Pod to pod [east-west] ========================= *** 1-a: Pod to Pod (Same Node) *** admin:worker-advnetlab48 -> admin:worker-advnetlab48 kubectl exec -it -n default ft-client-pod-sriov-7c7kw -- curl -m 5 "http://10.131.0.7:8080/etc/httpserver/" SUCCESS admin:worker-advnetlab48 -> admin:worker-advnetlab48 Client Pod on Client Host VF Representor Results: kubectl exec -n default ft-client-pod-sriov-7c7kw -- iperf3 -c 10.131.0.8 -p 5201 -t 40 kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "ethtool -S 5e4c68e6cdf0428 | sed -n 's/^\s\+//p'" kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i 5e4c68e6cdf0428 -n not arp" Summary (see hwol-logs/01-a-pod2pod-sameNode.txt for full detail): Summary Ethtool results for 5e4c68e6cdf0428: RX Packets: 2110133 - 2110133 = 0 TX Packets: 1529762921 - 1529762921 = 0 Summary Tcpdump Output: dropped privs to tcpdump tcpdump: listening on 5e4c68e6cdf0428, link-type EN10MB (Ethernet), snapshot length 262144 bytes 0 packets captured 0 packets received by filter 0 packets dropped by kernel Summary Iperf Output: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-40.00 sec 147 GBytes 31.6 Gbits/sec 28331 sender [ 5] 0.00-40.00 sec 147 GBytes 31.6 Gbits/sec receiver Client Pod on Server Host VF Representor Results: kubectl exec -n default ft-client-pod-sriov-7c7kw -- iperf3 -c 10.131.0.8 -p 5201 -t 40 kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "ethtool -S 5e4c68e6cdf0428 | sed -n 's/^\s\+//p'" kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i 5e4c68e6cdf0428 -n not arp" Summary (see hwol-logs/01-a-pod2pod-sameNode.txt for full detail): Summary Ethtool results for 5e4c68e6cdf0428: RX Packets: 2111344 - 2111344 = 0 TX Packets: 1529763129 - 1529763129 = 0 Summary Tcpdump Output: dropped privs to tcpdump tcpdump: listening on 5e4c68e6cdf0428, link-type EN10MB (Ethernet), snapshot length 262144 bytes 0 packets captured 0 packets received by filter 0 packets dropped by kernel Summary Iperf Output: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-40.00 sec 128 GBytes 27.4 Gbits/sec 18819 sender [ 5] 0.00-40.00 sec 128 GBytes 27.4 Gbits/sec receiver Server Pod on Server Host VF Representor Results: kubectl exec -n default ft-client-pod-sriov-7c7kw -- iperf3 -c 10.131.0.8 -p 5201 -t 40 kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "ethtool -S e934c8ad11a9898 | sed -n 's/^\s\+//p'" kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i e934c8ad11a9898 -n not arp" Summary (see hwol-logs/01-a-pod2pod-sameNode.txt for full detail): Summary Ethtool results for 5e4c68e6cdf0428: RX Packets: 49229067 - 49229067 = 0 TX Packets: 256117386 - 256117386 = 0 Summary Tcpdump Output: dropped privs to tcpdump tcpdump: listening on e934c8ad11a9898, link-type EN10MB (Ethernet), snapshot length 262144 bytes 0 packets captured 0 packets received by filter 0 packets dropped by kernel Summary Iperf Output: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-40.00 sec 129 GBytes 27.8 Gbits/sec 17577 sender [ 5] 0.00-40.00 sec 129 GBytes 27.8 Gbits/sec receiver SUCCESS admin:worker-advnetlab48 -(Reverse)-> admin:worker-advnetlab48 Client Pod on Client Host VF Representor Results (Reverse): kubectl exec -n default ft-client-pod-sriov-7c7kw -- iperf3 -R -c 10.131.0.8 -p 5201 -t 40 kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "ethtool -S 5e4c68e6cdf0428 | sed -n 's/^\s\+//p'" kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i 5e4c68e6cdf0428 -n not arp" Summary (see hwol-logs/01-a-pod2pod-sameNode.txt for full detail): Summary Ethtool results for 5e4c68e6cdf0428: RX Packets: 2111714 - 2111714 = 0 TX Packets: 1529765249 - 1529765249 = 0 Summary Tcpdump Output: dropped privs to tcpdump tcpdump: listening on 5e4c68e6cdf0428, link-type EN10MB (Ethernet), snapshot length 262144 bytes 0 packets captured 0 packets received by filter 0 packets dropped by kernel Summary Iperf Output: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-40.00 sec 144 GBytes 30.9 Gbits/sec 30382 sender [ 5] 0.00-40.00 sec 144 GBytes 30.9 Gbits/sec receiver Client Pod on Server Host VF Representor Results (Reverse): kubectl exec -n default ft-client-pod-sriov-7c7kw -- iperf3 -R -c 10.131.0.8 -p 5201 -t 40 kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "ethtool -S 5e4c68e6cdf0428 | sed -n 's/^\s\+//p'" kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i 5e4c68e6cdf0428 -n not arp" Summary (see hwol-logs/01-a-pod2pod-sameNode.txt for full detail): Summary Ethtool results for 5e4c68e6cdf0428: RX Packets: 2111810 - 2111810 = 0 TX Packets: 1529767130 - 1529767130 = 0 Summary Tcpdump Output: dropped privs to tcpdump tcpdump: listening on 5e4c68e6cdf0428, link-type EN10MB (Ethernet), snapshot length 262144 bytes 0 packets captured 0 packets received by filter 0 packets dropped by kernel Summary Iperf Output: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-40.00 sec 131 GBytes 28.2 Gbits/sec 25898 sender [ 5] 0.00-40.00 sec 131 GBytes 28.2 Gbits/sec receiver Server Pod on Server Host VF Representor Results (Reverse): kubectl exec -n default ft-client-pod-sriov-7c7kw -- iperf3 -R -c 10.131.0.8 -p 5201 -t 40 kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "ethtool -S e934c8ad11a9898 | sed -n 's/^\s\+//p'" kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i e934c8ad11a9898 -n not arp" Summary (see hwol-logs/01-a-pod2pod-sameNode.txt for full detail): Summary Ethtool results for 5e4c68e6cdf0428: RX Packets: 49241013 - 49241013 = 0 TX Packets: 256117663 - 256117663 = 0 Summary Tcpdump Output: dropped privs to tcpdump tcpdump: listening on e934c8ad11a9898, link-type EN10MB (Ethernet), snapshot length 262144 bytes 0 packets captured 0 packets received by filter 0 packets dropped by kernel Summary Iperf Output: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-40.00 sec 124 GBytes 26.7 Gbits/sec 16422 sender [ 5] 0.00-40.00 sec 124 GBytes 26.7 Gbits/sec receiver SUCCESS *** 1-b: Pod to Pod (Different Node) *** admin:worker-advnetlab49 -> admin:worker-advnetlab48 kubectl exec -it -n default ft-client-pod-sriov-5g7s5 -- curl -m 5 "http://10.131.0.7:8080/etc/httpserver/" SUCCESS admin:worker-advnetlab49 -> admin:worker-advnetlab48 Client Pod on Client Host VF Representor Results: kubectl exec -n default ft-client-pod-sriov-5g7s5 -- iperf3 -R -c 10.131.0.8 -p 5201 -t 40 kubectl exec -n "default" "ft-tools-tcdcf" -- /bin/sh -c "ethtool -S 71f6388f057f493 | sed -n 's/^\s\+//p'" kubectl exec -n "default" "ft-tools-tcdcf" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i 71f6388f057f493 -n not arp" Summary (see hwol-logs/01-b-pod2pod-diffNode.txt for full detail): Summary Ethtool results for 71f6388f057f493: RX Packets: 29709701 - 29709701 = 0 TX Packets: 16373115 - 16373115 = 0 Summary Tcpdump Output: dropped privs to tcpdump tcpdump: listening on 71f6388f057f493, link-type EN10MB (Ethernet), snapshot length 262144 bytes 0 packets captured 0 packets received by filter 0 packets dropped by kernel Summary Iperf Output: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-40.00 sec 80.1 GBytes 17.2 Gbits/sec 1886 sender [ 5] 0.00-40.00 sec 80.1 GBytes 17.2 Gbits/sec receiver Client Pod on Server Host VF Representor Results: kubectl exec -n default ft-client-pod-sriov-5g7s5 -- iperf3 -R -c 10.131.0.8 -p 5201 -t 40 kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "ethtool -S 5e4c68e6cdf0428 | sed -n 's/^\s\+//p'" kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i 5e4c68e6cdf0428 -n not arp" Summary (see hwol-logs/01-b-pod2pod-diffNode.txt for full detail): Summary Ethtool results for 71f6388f057f493: RX Packets: 2111927 - 2111927 = 0 TX Packets: 1529775109 - 1529775109 = 0 Summary Tcpdump Output: dropped privs to tcpdump tcpdump: listening on 5e4c68e6cdf0428, link-type EN10MB (Ethernet), snapshot length 262144 bytes 0 packets captured 0 packets received by filter 0 packets dropped by kernel Summary Iperf Output: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-40.00 sec 85.1 GBytes 18.3 Gbits/sec 2989 sender [ 5] 0.00-40.00 sec 85.1 GBytes 18.3 Gbits/sec receiver Server Pod on Server Host VF Representor Results: kubectl exec -n default ft-client-pod-sriov-5g7s5 -- iperf3 -R -c 10.131.0.8 -p 5201 -t 40 kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "ethtool -S e934c8ad11a9898 | sed -n 's/^\s\+//p'" kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i e934c8ad11a9898 -n not arp" Summary (see hwol-logs/01-b-pod2pod-diffNode.txt for full detail): Summary Ethtool results for 71f6388f057f493: RX Packets: 49246837 - 49246837 = 0 TX Packets: 256117890 - 256117890 = 0 Summary Tcpdump Output: dropped privs to tcpdump tcpdump: listening on e934c8ad11a9898, link-type EN10MB (Ethernet), snapshot length 262144 bytes 0 packets captured 0 packets received by filter 0 packets dropped by kernel Summary Iperf Output: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-40.00 sec 99.3 GBytes 21.3 Gbits/sec 6364 sender [ 5] 0.00-40.00 sec 99.3 GBytes 21.3 Gbits/sec receiver SUCCESS admin:worker-advnetlab49 -(Reverse)-> admin:worker-advnetlab48 Client Pod on Client Host VF Representor Results (Reverse): kubectl exec -n default ft-client-pod-sriov-5g7s5 -- iperf3 -R -c 10.131.0.8 -p 5201 -t 40 kubectl exec -n "default" "ft-tools-tcdcf" -- /bin/sh -c "ethtool -S 71f6388f057f493 | sed -n 's/^\s\+//p'" kubectl exec -n "default" "ft-tools-tcdcf" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i 71f6388f057f493 -n not arp" Summary (see hwol-logs/01-b-pod2pod-diffNode.txt for full detail): Summary Ethtool results for 71f6388f057f493: RX Packets: 29709735 - 29709735 = 0 TX Packets: 16373234 - 16373234 = 0 Summary Tcpdump Output: dropped privs to tcpdump tcpdump: listening on 71f6388f057f493, link-type EN10MB (Ethernet), snapshot length 262144 bytes 0 packets captured 0 packets received by filter 0 packets dropped by kernel Summary Iperf Output: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-40.00 sec 88.4 GBytes 19.0 Gbits/sec 5081 sender [ 5] 0.00-40.00 sec 88.4 GBytes 19.0 Gbits/sec receiver Client Pod on Server Host VF Representor Results (Reverse): kubectl exec -n default ft-client-pod-sriov-5g7s5 -- iperf3 -R -c 10.131.0.8 -p 5201 -t 40 kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "ethtool -S 5e4c68e6cdf0428 | sed -n 's/^\s\+//p'" kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i 5e4c68e6cdf0428 -n not arp" Summary (see hwol-logs/01-b-pod2pod-diffNode.txt for full detail): Summary Ethtool results for 71f6388f057f493: RX Packets: 2111927 - 2111927 = 0 TX Packets: 1529775109 - 1529775109 = 0 Summary Tcpdump Output: dropped privs to tcpdump tcpdump: listening on 5e4c68e6cdf0428, link-type EN10MB (Ethernet), snapshot length 262144 bytes 0 packets captured 0 packets received by filter 0 packets dropped by kernel Summary Iperf Output: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-40.00 sec 100 GBytes 21.6 Gbits/sec 7092 sender [ 5] 0.00-40.00 sec 100 GBytes 21.6 Gbits/sec receiver Server Pod on Server Host VF Representor Results (Reverse): kubectl exec -n default ft-client-pod-sriov-5g7s5 -- iperf3 -R -c 10.131.0.8 -p 5201 -t 40 kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "ethtool -S e934c8ad11a9898 | sed -n 's/^\s\+//p'" kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i e934c8ad11a9898 -n not arp" Summary (see hwol-logs/01-b-pod2pod-diffNode.txt for full detail): Summary Ethtool results for 71f6388f057f493: RX Packets: 49251327 - 49251327 = 0 TX Packets: 256118048 - 256118048 = 0 Summary Tcpdump Output: dropped privs to tcpdump tcpdump: listening on e934c8ad11a9898, link-type EN10MB (Ethernet), snapshot length 262144 bytes 0 packets captured 0 packets received by filter 0 packets dropped by kernel Summary Iperf Output: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-40.00 sec 101 GBytes 21.6 Gbits/sec 5579 sender [ 5] 0.00-40.00 sec 101 GBytes 21.6 Gbits/sec receiver SUCCESS =============================== Pod to external [north-south] =============================== [root@wsfd-advnetlab15 ~]# iperf3 -s -p 9999 on worker-advnetlab15 [root@wsfd-advnetlab15 ~]# kubectl exec -it -n default ft-client-pod-sriov-5g7s5 -- iperf3 -c wsfd-advnetlab15.anl.lab.eng.bos.redhat.com -p 9999 -t 30 Connecting to host wsfd-advnetlab15.anl.lab.eng.bos.redhat.com, port 9999 [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-30.00 sec 15.7 GBytes 4.50 Gbits/sec 1597 sender [ 5] 0.00-30.04 sec 15.7 GBytes 4.49 Gbits/sec receiver [root@wsfd-advnetlab15 ~]# kubectl exec -n "default" "ft-tools-tcdcf" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i 71f6388f057f493 -n not arp" 17:15:58.770045 IP (tos 0x0, ttl 62, id 33802, offset 0, flags [DF], proto TCP (6), length 52) 10.19.128.33.distinct > 10.128.2.7.42046: Flags [.], cksum 0x5cab (correct), ack 2740571658, win 24576, options [nop,nop,TS val 3409003707 ecr 3521667750], length 0 17:15:58.770046 IP (tos 0x0, ttl 62, id 33803, offset 0, flags [DF], proto TCP (6), length 52) 10.19.128.33.distinct > 10.128.2.7.42046: Flags [F.], cksum 0x5ca9 (correct), seq 1, ack 2740571658, win 24576, options [nop,nop,TS val 3409003708 ecr 3521667750], length 0 17:15:58.770048 IP (tos 0x0, ttl 62, id 54042, offset 0, flags [DF], proto TCP (6), length 52) 10.19.128.33.distinct > 10.128.2.7.42040: Flags [F.], cksum 0xb528 (correct), seq 5, ack 144, win 227, options [nop,nop,TS val 3409003708 ecr 3521667751], length 0 17:15:58.770078 IP (tos 0x0, ttl 62, id 60045, offset 0, flags [DF], proto TCP (6), length 64756) 192.168.111.34.42046 > 10.19.128.33.distinct: Flags [.], cksum 0xb6e6 (incorrect -> 0x21cc), seq 2740571620:2740636324, ack 1, win 213, options [nop,nop,TS val 3521667752 ecr 3409003707], length 64704 17:15:58.770122 IP (tos 0x0, ttl 62, id 60093, offset 0, flags [DF], proto TCP (6), length 64756) 192.168.111.34.42046 > 10.19.128.33.distinct: Flags [.], cksum 0xb6e6 (incorrect -> 0x690b), seq 2740636324:2740701028, ack 1, win 213, options [nop,nop,TS val 3521667752 ecr 3409003707], length 64704 17:15:58.770163 IP (tos 0x0, ttl 62, id 60141, offset 0, flags [DF], proto TCP (6), length 64756) 192.168.111.34.42046 > 10.19.128.33.distinct: Flags [.], cksum 0xb6e6 (incorrect -> 0x16a3), seq 2740701028:2740765732, ack 1, win 213, options [nop,nop,TS val 3521667752 ecr 3409003707], length 64704 CONCLUSION: HW Offload working is not working. =============================== External network traffic to nodePort/External IP service [north-south] =============================== [root@wsfd-advnetlab15 ~]# iperf3 -c 192.168.111.33 -p 30201 -t 30 [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-30.00 sec 678 MBytes 190 Mbits/sec 4392 sender [ 5] 0.00-30.03 sec 675 MBytes 189 Mbits/sec receiver [root@wsfd-advnetlab15 ~]# kubectl exec -n "default" "ft-tools-brxwg" -- /bin/sh -c "timeout --preserve-status 25 tcpdump -v -i e934c8ad11a9898 -n not arp" 17:31:24.358241 IP (tos 0x0, ttl 62, id 1723, offset 0, flags [DF], proto TCP (6), length 1400) 100.64.0.5.33426 > 10.131.0.8.targus-getdata1: Flags [.], cksum 0xbfed (correct), seq 272641088:272642436, ack 1, win 229, options [nop,nop,TS val 3910115808 ecr 189867650], length 1348 17:31:24.361371 IP (tos 0x0, ttl 62, id 1724, offset 0, flags [DF], proto TCP (6), length 1400) 100.64.0.5.33426 > 10.131.0.8.targus-getdata1: Flags [.], cksum 0xd9df (correct), seq 272642436:272643784, ack 1, win 229, options [nop,nop,TS val 3910115808 ecr 189867650], length 1348 17:31:24.361374 IP (tos 0x0, ttl 62, id 1725, offset 0, flags [DF], proto TCP (6), length 1400) 100.64.0.5.33426 > 10.131.0.8.targus-getdata1: Flags [.], cksum 0x02de (correct), seq 272643784:272645132, ack 1, win 229, options [nop,nop,TS val 3910115808 ecr 189867650], length 1348 17:31:24.361393 IP (tos 0x0, ttl 62, id 1726, offset 0, flags [DF], proto TCP (6), length 1400) 100.64.0.5.33426 > 10.131.0.8.targus-getdata1: Flags [.], cksum 0x9b6d (correct), seq 272645132:272646480, ack 1, win 229, options [nop,nop,TS val 3910115808 ecr 189867650], length 1348 17:31:24.361395 IP (tos 0x0, ttl 62, id 1727, offset 0, flags [DF], proto TCP (6), length 1400)