-
Bug
-
Resolution: Done
-
Undefined
-
None
-
None
-
None
-
2
-
False
-
-
False
-
None
-
rhel-net-ovs-dpdk
-
-
-
ssg_networking
-
OVS/DPDK - FDP 24.H
-
1
-
Important
When running OVN tests with the OVS submodule bumped to the current upstream OVS branch-3.3 tip:
0c9e932aa047 netdev-dpdk: Trigger port reconfiguration in main thread for resets.
ovs-vswitchd crashes as follows:
2024-01-22T10:04:32.102Z|00002|util(monitor12)|EMER|./lib/dp-packet.h:1390: assertion ip failed in dp_packet_ip_set_header_csum()
(gdb) bt
#0 0x00007f4459caeacf in raise () from /lib64/libc.so.6
#1 0x00007f4459c81ea5 in abort () from /lib64/libc.so.6
#2 0x0000000000609718 in ovs_abort_valist (err_no=0, format=0x71642c "%s: assertion %s failed in %s()",
args=0x7f44522bf060) at lib/util.c:447
#3 0x0000000000614dac in vlog_abort_valist (module_=0x9fd218 <this_module>,
message=0x71642c "%s: assertion %s failed in %s()", args=0x7f44522bf060) at lib/vlog.c:1293
#4 0x0000000000614e45 in vlog_abort (module=0x9fd218 <this_module>,
message=0x71642c "%s: assertion %s failed in %s()") at lib/vlog.c:1307
#5 0x0000000000608d1c in ovs_assert_failure (where=0x6e9755 "./lib/dp-packet.h:1390",
function=0x6e976c "dp_packet_ip_set_header_csum", condition=0x6f5e68 "ip") at lib/util.c:89
#6 0x00000000004b163a in dp_packet_ip_set_header_csum (p=0x1b52930, inner=true) at ./lib/dp-packet.h:1390
#7 0x00000000004b1842 in dp_packet_ol_send_prepare (p=0x1b52930, flags=0) at lib/dp-packet.c:593
#8 0x000000000051f70f in netdev_push_header (netdev=0x1b10cd0, batch=0x7f44522c0308, data=0x7f44522c066c)
at lib/netdev.c:1035
#9 0x00000000004c7623 in push_tnl_action (pmd=0x7f4456acc010, attr=0x7f44522c0668, batch=0x7f44522c0308)
at lib/dpif-netdev.c:9038
#10 0x00000000004c65aa in dp_execute_cb (aux_=0x7f44522c02a0, packets_=0x7f44522c0308, a=0x7f44522c0668,
should_steal=false) at lib/dpif-netdev.c:9197
#11 0x000000000052d570 in odp_execute_actions (dp=0x7f44522c02a0, batch=0x7f44522c0308, steal=false,
actions=0x7f44522c0668, actions_len=80, dp_execute_action=0x4c6460 <dp_execute_cb>) at lib/odp-execute.c:1008
#12 0x00000000004b2e7a in dp_netdev_execute_actions (pmd=0x7f4456acc010, packets=0x7f44522c0308, should_steal=false,
flow=0x7f44522c0ea0, actions=0x7f44522c0668, actions_len=80) at lib/dpif-netdev.c:9495
#13 0x00000000004cd1b0 in dpif_netdev_execute (dpif=0x1abf6f0, execute=0x7f44522c0540) at lib/dpif-netdev.c:4674
#14 0x00000000004b5231 in dpif_netdev_operate (dpif=0x1abf6f0, ops=0x7f44522c0590, n_ops=1,
--Type <RET> for more, q to quit, c to continue without paging--
UTO) at lib/dpif-netdev.c:4723
#15 0x00000000004d5faa in dpif_operate (dpif=0x1abf6f0, ops=0x7f44522c0590, n_ops=1, offload_type=DPIF_OFFLOAD_AUTO) at lib/dpif.c:1382
#16 0x00000000004d679b in dpif_execute (dpif=0x1abf6f0, execute=0x7f44522c05f0) at lib/dpif.c:1336
#17 0x0000000000438157 in ofproto_dpif_execute_actions__ (ofproto=0x1ace060, version=18446744073709551614, flow=0x7f44522c0ea0, rule=0x0, ofpacts=0x7f44522c1180, ofpacts_len=16, depth=0, resubmits=0, packet=0x7f44522c16c0)
at ofproto/ofproto-dpif.c:4290
#18 0x000000000043859c in ofproto_dpif_execute_actions (ofproto=0x1ace060, version=18446744073709551614, flow=0x7f44522c0ea0, rule=0x0, ofpacts=0x7f44522c1180, ofpacts_len=16, packet=0x7f44522c16c0) at ofproto/ofproto-dpif.c:4307
#19 0x0000000000471e79 in xlate_send_packet (ofport=0x1b0df10, oam=false, packet=0x7f44522c16c0) at ofproto/ofproto-dpif-xlate.c:8567
#20 0x000000000043906e in ofproto_dpif_send_packet (ofport=0x1b0df10, oam=false, packet=0x7f44522c16c0) at ofproto/ofproto-dpif.c:5331
#21 0x0000000000457e4a in monitor_mport_run (mport=0x1b46f00, packet=0x7f44522c16c0) at ofproto/ofproto-dpif-monitor.c:287
#22 0x0000000000457b89 in monitor_run () at ofproto/ofproto-dpif-monitor.c:227
#23 0x000000000045725e in monitor_main (args=0x0) at ofproto/ofproto-dpif-monitor.c:189
#24 0x00000000005ba319 in ovsthread_wrapper (aux_=0x1b38cc0) at lib/ovs-thread.c:423
#25 0x00007f445a91d1ca in start_thread () from /lib64/libpthread.so.0
#26 0x00007f4459c99e73 in clone () from /lib64/libc.so.6
(gdb) frame 6
#6 0x00000000004b163a in dp_packet_ip_set_header_csum (p=0x1b52930, inner=true) at ./lib/dp-packet.h:1390
1390 ovs_assert(ip);
(gdb) p ip
$1 = (struct ip_header *) 0x0
(gdb) p inner
$2 = true
(gdb) p /x p->ol_flags
$4 = 0x3b481
Sample run: https://github.com/dceara/ovn/actions/runs/7609294730/job/20720294903