Tracing mutex/sem lock events... Hit Ctrl-C to end Caller Avg Wait Count Max Wait Total Wait rtnetlink_rcv_msg+0x111 218.5 ms 2665 3.2 s 9.7 m __init_scratch_end+0x52703c1 rtnetlink_rcv_msg+0x5 netlink_rcv_skb+0x54 netlink_unicast+0x245 netlink_sendmsg+0x206 __sys_sendto+0x1dc __x64_sys_sendto+0x20 Max PID 1765403, COMM sriov, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETLINK Caller Avg Wait Count Max Wait Total Wait rtnl_dumpit+0x70 53.6 ms 905 3.2 s 48.5 s netlink_dump+0x1a3 __init_scratch_end+0x5276cf9 netlink_dump+0x5 __netlink_dump_start+0x1e1 rtnetlink_rcv_msg+0x2ba __init_scratch_end+0x52703c1 rtnetlink_rcv_msg+0x5 Max PID 1766443, COMM curl, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETADDR Caller Avg Wait Count Max Wait Total Wait rtnl_dumpit+0x70 2.3 ms 6876 3.1 s 15.7 s netlink_dump+0x1a3 __init_scratch_end+0x5276cf9 netlink_dump+0x5 netlink_recvmsg+0x1d1 sock_recvmsg+0x9c ____sys_recvmsg+0x87 ___sys_recvmsg+0x7c Max PID 6046, COMM node_exporter, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETLINK Caller Avg Wait Count Max Wait Total Wait ip6mr_sk_done+0x2e 349.1 ms 18 3.1 s 6.3 s rawv6_close+0x19 inet_release+0x3c sock_release+0x20 igmp6_net_exit+0x21 ops_exit_list+0x30 cleanup_net+0x269 process_one_work+0x194 Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Wait Count Max Wait Total Wait rtnl_dumpit+0x70 21.5 ms 743 2.9 s 16.0 s netlink_dump+0x1a3 __init_scratch_end+0x5276cf9 netlink_dump+0x5 netlink_recvmsg+0x1d1 sock_recvmsg+0x9c __sys_recvfrom+0xa8 __x64_sys_recvfrom+0x20 Max PID 5385, COMM corednsmonitor, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETLINK Caller Avg Wait Count Max Wait Total Wait ethnl_default_dumpit+0xb4 753.7 ms 11 2.8 s 8.3 s genl_dumpit+0x31 netlink_dump+0x1a3 __netlink_dump_start+0x1e1 genl_family_rcv_msg_dumpit+0x93 genl_family_rcv_msg+0xf1 genl_rcv_msg+0x47 netlink_rcv_skb+0x54 Max PID 6046, COMM node_exporter, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Wait Count Max Wait Total Wait rtnl_dumpit+0x70 150.6 ms 68 2.7 s 10.2 s netlink_dump+0x1a3 __init_scratch_end+0x5276cf9 netlink_dump+0x5 __netlink_dump_start+0x1e1 rtnetlink_rcv_msg+0x2ba __init_scratch_end+0x52703c1 rtnetlink_rcv_msg+0x5 Max PID 2453, COMM NetworkManager, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETROUTE Caller Avg Wait Count Max Wait Total Wait ethnl_default_dumpit+0xb4 19.6 ms 148 2.6 s 2.9 s genl_dumpit+0x31 netlink_dump+0x1a3 __init_scratch_end+0x5276cf9 netlink_dump+0x5 __netlink_dump_start+0x1e1 genl_family_rcv_msg_dumpit+0x93 genl_family_rcv_msg+0xf1 Max PID 6046, COMM node_exporter, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETADDR Caller Avg Wait Count Max Wait Total Wait rtnetlink_rcv_msg+0x111 312.5 us 40563 2.4 s 12.7 s __init_scratch_end+0x52703c1 rtnetlink_rcv_msg+0x5 netlink_rcv_skb+0x54 netlink_unicast+0x245 netlink_sendmsg+0x206 ____sys_sendmsg+0x38b ___sys_sendmsg+0x88 Max PID 2421, COMM ovs-vswitchd, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETLINK Caller Avg Wait Count Max Wait Total Wait dev_ethtool+0x93 364.7 ms 164 2.3 s 59.8 s dev_ioctl+0x1b5 sock_do_ioctl+0xab __init_scratch_end+0x5277609 sock_do_ioctl+0x5 sock_ioctl+0x1ce __x64_sys_ioctl+0x87 do_syscall_64+0x5f Max PID 1755434, COMM systemd-udevd, Lock rtnl_mutex (0xffffffffb9722a00) ioctl SIOCETHTOOL Caller Avg Wait Count Max Wait Total Wait linkwatch_event+0xa 242.1 ms 26 2.1 s 6.3 s process_one_work+0x194 worker_thread+0x2fe kthread+0xdd ret_from_fork+0x2c Max PID 1649857, COMM kworker/u418:3, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Wait Count Max Wait Total Wait ip6mr_sk_done+0x2e 126.0 ms 19 1.3 s 2.4 s rawv6_close+0x19 inet_release+0x3c sock_release+0x20 igmp6_net_exit+0x39 ops_exit_list+0x30 cleanup_net+0x269 process_one_work+0x194 Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Wait Count Max Wait Total Wait ip6mr_sk_done+0x2e 339.5 us 2496 156.5 ms 847.4 ms rawv6_close+0x19 inet_release+0x3c sock_release+0x20 icmpv6_sk_exit+0x4f ops_exit_list+0x30 cleanup_net+0x269 process_one_work+0x194 Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Wait Count Max Wait Total Wait ethnl_default_dumpit+0xb4 3.6 ms 267 153.3 ms 956.1 ms genl_dumpit+0x31 netlink_dump+0x1a3 __init_scratch_end+0x5276cf9 netlink_dump+0x5 netlink_recvmsg+0x1d1 sock_recvmsg+0x9c ____sys_recvmsg+0x87 Max PID 6046, COMM node_exporter, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETADDR Caller Avg Wait Count Max Wait Total Wait addrconf_dad_work+0x3e 1.9 ms 396 146.4 ms 749.8 ms process_one_work+0x194 worker_thread+0x2fe kthread+0xdd ret_from_fork+0x2c Max PID 1761397, COMM kworker/0:27, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Wait Count Max Wait Total Wait ip6mr_sk_done+0x2e 2.3 us 24 44.3 us 55.4 us rawv6_close+0x19 inet_release+0x3c sock_release+0x20 ops_exit_list+0x30 cleanup_net+0x269 process_one_work+0x194 worker_thread+0x2fe Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Wait Count Max Wait Total Wait ipmr_net_exit_batch+0xf 4.7 us 1 4.7 us 4.7 us cleanup_net+0x269 process_one_work+0x194 worker_thread+0x2fe kthread+0xdd ret_from_fork+0x2c Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Wait Count Max Wait Total Wait phys_port_name_show+0x60 911 ns 24 2.0 us 21.9 us dev_attr_show+0x16 sysfs_kf_seq_show+0x98 seq_read_iter+0x120 vfs_read+0x259 ksys_read+0x5f do_syscall_64+0x5f entry_SYSCALL_64_after_hwframe+0x78 Max PID 1755513, COMM systemd-udevd, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Wait Count Max Wait Total Wait phys_switch_id_show+0x60 499 ns 3 740 ns 1.5 us dev_attr_show+0x16 sysfs_kf_seq_show+0x98 seq_read_iter+0x120 vfs_read+0x259 ksys_read+0x5f do_syscall_64+0x5f entry_SYSCALL_64_after_hwframe+0x78 Max PID 6495, COMM sriov-network-c, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Wait Count Max Wait Total Wait speed_show+0x37 311 ns 6 600 ns 1.9 us dev_attr_show+0x16 sysfs_kf_seq_show+0x98 seq_read_iter+0x120 vfs_read+0x259 ksys_read+0x5f do_syscall_64+0x5f entry_SYSCALL_64_after_hwframe+0x78 Max PID 6495, COMM sriov-network-c, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Wait Count Max Wait Total Wait fib_net_exit_batch+0xf 523 ns 1 523 ns 523 ns cleanup_net+0x269 process_one_work+0x194 worker_thread+0x2fe kthread+0xdd ret_from_fork+0x2c Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Wait Count Max Wait Total Wait ip6mr_net_exit_batch+0xf 201 ns 1 201 ns 201 ns cleanup_net+0x269 process_one_work+0x194 worker_thread+0x2fe kthread+0xdd ret_from_fork+0x2c Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold linkwatch_event+0xa 19.9 ms 26 243.2 ms 517.5 ms process_one_work+0x194 worker_thread+0x2fe kthread+0xdd ret_from_fork+0x2c Max PID 1462205, COMM kworker/u417:0, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold rtnetlink_rcv_msg+0x111 4.0 ms 2665 99.5 ms 10.8 s __init_scratch_end+0x52703c1 rtnetlink_rcv_msg+0x5 netlink_rcv_skb+0x54 netlink_unicast+0x245 netlink_sendmsg+0x206 __sys_sendto+0x1dc __x64_sys_sendto+0x20 Max PID 1765259, COMM sriov, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_NEWLINK Caller Avg Hold Count Max Hold Total Hold ethnl_default_dumpit+0xb4 200.2 us 148 22.5 ms 29.6 ms genl_dumpit+0x31 netlink_dump+0x1a3 __init_scratch_end+0x5276cf9 netlink_dump+0x5 __netlink_dump_start+0x1e1 genl_family_rcv_msg_dumpit+0x93 genl_family_rcv_msg+0xf1 Max PID 6046, COMM node_exporter, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETADDR Caller Avg Hold Count Max Hold Total Hold rtnl_dumpit+0x70 70.4 us 6876 1.4 ms 484.1 ms netlink_dump+0x1a3 __init_scratch_end+0x5276cf9 netlink_dump+0x5 netlink_recvmsg+0x1d1 sock_recvmsg+0x9c ____sys_recvmsg+0x87 ___sys_recvmsg+0x7c Max PID 2453, COMM NetworkManager, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETLINK Caller Avg Hold Count Max Hold Total Hold rtnl_dumpit+0x70 72.0 us 68 1.2 ms 4.9 ms netlink_dump+0x1a3 __init_scratch_end+0x5276cf9 netlink_dump+0x5 __netlink_dump_start+0x1e1 rtnetlink_rcv_msg+0x2ba __init_scratch_end+0x52703c1 rtnetlink_rcv_msg+0x5 Max PID 2453, COMM NetworkManager, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETLINK Caller Avg Hold Count Max Hold Total Hold speed_show+0x37 309.0 us 6 1.1 ms 1.9 ms dev_attr_show+0x16 sysfs_kf_seq_show+0x98 seq_read_iter+0x120 vfs_read+0x259 ksys_read+0x5f do_syscall_64+0x5f entry_SYSCALL_64_after_hwframe+0x78 Max PID 6495, COMM sriov-network-c, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold addrconf_dad_work+0x3e 82.2 us 396 427.3 us 32.6 ms process_one_work+0x194 worker_thread+0x2fe kthread+0xdd ret_from_fork+0x2c Max PID 1761397, COMM kworker/0:27, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold ethnl_default_dumpit+0xb4 149.6 us 11 405.7 us 1.6 ms genl_dumpit+0x31 netlink_dump+0x1a3 __netlink_dump_start+0x1e1 genl_family_rcv_msg_dumpit+0x93 genl_family_rcv_msg+0xf1 genl_rcv_msg+0x47 netlink_rcv_skb+0x54 Max PID 6046, COMM node_exporter, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold dev_ethtool+0x93 12.3 us 164 205.9 us 2.0 ms dev_ioctl+0x1b5 sock_do_ioctl+0xab __init_scratch_end+0x5277609 sock_do_ioctl+0x5 sock_ioctl+0x1ce __x64_sys_ioctl+0x87 do_syscall_64+0x5f Max PID 1755899, COMM systemd-udevd, Lock rtnl_mutex (0xffffffffb9722a00) ioctl SIOCETHTOOL Caller Avg Hold Count Max Hold Total Hold rtnl_dumpit+0x70 59.6 us 743 181.2 us 44.3 ms netlink_dump+0x1a3 __init_scratch_end+0x5276cf9 netlink_dump+0x5 netlink_recvmsg+0x1d1 sock_recvmsg+0x9c __sys_recvfrom+0xa8 __x64_sys_recvfrom+0x20 Max PID 5385, COMM corednsmonitor, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETLINK Caller Avg Hold Count Max Hold Total Hold rtnl_dumpit+0x70 56.2 us 905 143.6 us 50.9 ms netlink_dump+0x1a3 __init_scratch_end+0x5276cf9 netlink_dump+0x5 __netlink_dump_start+0x1e1 rtnetlink_rcv_msg+0x2ba __init_scratch_end+0x52703c1 rtnetlink_rcv_msg+0x5 Max PID 2421, COMM ovs-vswitchd, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETLINK Caller Avg Hold Count Max Hold Total Hold rtnetlink_rcv_msg+0x111 12.8 us 40563 113.3 us 518.7 ms __init_scratch_end+0x52703c1 rtnetlink_rcv_msg+0x5 netlink_rcv_skb+0x54 netlink_unicast+0x245 netlink_sendmsg+0x206 ____sys_sendmsg+0x38b ___sys_sendmsg+0x88 Max PID 2453, COMM NetworkManager, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_NEWADDR Caller Avg Hold Count Max Hold Total Hold fib_net_exit_batch+0xf 63.3 us 1 63.3 us 63.3 us cleanup_net+0x269 process_one_work+0x194 worker_thread+0x2fe kthread+0xdd ret_from_fork+0x2c Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold ip6mr_sk_done+0x2e 218 ns 2496 60.4 us 546.5 us rawv6_close+0x19 inet_release+0x3c sock_release+0x20 icmpv6_sk_exit+0x4f ops_exit_list+0x30 cleanup_net+0x269 process_one_work+0x194 Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold ip6mr_net_exit_batch+0xf 49.1 us 1 49.1 us 49.1 us cleanup_net+0x269 process_one_work+0x194 worker_thread+0x2fe kthread+0xdd ret_from_fork+0x2c Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold ipmr_net_exit_batch+0xf 39.4 us 1 39.4 us 39.4 us cleanup_net+0x269 process_one_work+0x194 worker_thread+0x2fe kthread+0xdd ret_from_fork+0x2c Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold ethnl_default_dumpit+0xb4 740 ns 267 3.9 us 197.7 us genl_dumpit+0x31 netlink_dump+0x1a3 __init_scratch_end+0x5276cf9 netlink_dump+0x5 netlink_recvmsg+0x1d1 sock_recvmsg+0x9c ____sys_recvmsg+0x87 Max PID 6046, COMM node_exporter, Lock rtnl_mutex (0xffffffffb9722a00) nltype RTM_GETADDR Caller Avg Hold Count Max Hold Total Hold ip6mr_sk_done+0x2e 953 ns 18 2.2 us 17.2 us rawv6_close+0x19 inet_release+0x3c sock_release+0x20 igmp6_net_exit+0x21 ops_exit_list+0x30 cleanup_net+0x269 process_one_work+0x194 Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold ip6mr_sk_done+0x2e 844 ns 19 2.2 us 16.0 us rawv6_close+0x19 inet_release+0x3c sock_release+0x20 igmp6_net_exit+0x39 ops_exit_list+0x30 cleanup_net+0x269 process_one_work+0x194 Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold phys_switch_id_show+0x60 1.1 us 3 1.7 us 3.5 us dev_attr_show+0x16 sysfs_kf_seq_show+0x98 seq_read_iter+0x120 vfs_read+0x259 ksys_read+0x5f do_syscall_64+0x5f entry_SYSCALL_64_after_hwframe+0x78 Max PID 6495, COMM sriov-network-c, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold phys_port_name_show+0x60 942 ns 24 1.6 us 22.6 us dev_attr_show+0x16 sysfs_kf_seq_show+0x98 seq_read_iter+0x120 vfs_read+0x259 ksys_read+0x5f do_syscall_64+0x5f entry_SYSCALL_64_after_hwframe+0x78 Max PID 1755513, COMM systemd-udevd, Lock rtnl_mutex (0xffffffffb9722a00) Caller Avg Hold Count Max Hold Total Hold ip6mr_sk_done+0x2e 417 ns 24 510 ns 10.0 us rawv6_close+0x19 inet_release+0x3c sock_release+0x20 ops_exit_list+0x30 cleanup_net+0x269 process_one_work+0x194 worker_thread+0x2fe Max PID 1707118, COMM kworker/u416:13, Lock rtnl_mutex (0xffffffffb9722a00) Exiting trace of mutex/sem locks