-
Bug
-
Resolution: Done
-
Major
-
None
-
1.6.0.GA
-
None
The router is crashing with the following error from the daemon.log
Feb 17 05:15:02 rh-amq-prd04.trfw.nx.digikey.com qdrouterd: epoll proactor failure in /builddir/build/BUILD/qpid-proton-0.28.0/c/src/proactor/epoll.c:312: "EPOLL_CTL_DEL": No such file or directory Feb 17 05:15:02 rh-amq-prd04.trfw.nx.digikey.com systemd: qdrouterd.service: main process exited, code=killed, status=6/ABRT Feb 17 05:15:02 rh-amq-prd04.trfw.nx.digikey.com systemd: Unit qdrouterd.service entered failed state. Feb 17 05:15:02 rh-amq-prd04.trfw.nx.digikey.com systemd: qdrouterd.service failed.
We also have a backtrace which does not exactly line up with the above error but might be useful
(gdb) bt
#0 0x00007fac95cda377 in killpg () from /lib64/libc.so.6
#1 0x00007fac95cdba68 in abort () from /lib64/libc.so.6
#2 0x00007fac96c942b2 in rearm (p=<optimized out>, ee=<optimized out>) at /usr/src/debug/qpid-proton-0.28.0/c/src/proactor/epoll.c:715
#3 0x00007fac96c94ca6 in rearm (ee=0x4309, p=<optimized out>) at /usr/src/debug/qpid-proton-0.28.0/c/src/proactor/epoll.c:715
#4 0x00007fac00007fac in ?? ()
#5 0x0000000001af08a0 in ?? ()
#6 0xf60b914474d73800 in ?? ()
#7 0x00007fac5e605600 in ?? ()
#8 0x00007fac5e605a90 in ?? ()
#9 0x00007fac5e605600 in ?? ()
#10 0x00007fac96c95673 in lock (m=0x4) at /usr/src/debug/qpid-proton-0.28.0/c/src/proactor/epoll.c:112
#11 pconnection_cleanup (pc=0x7fac5e605b48) at /usr/src/debug/qpid-proton-0.28.0/c/src/proactor/epoll.c:901
#12 0x00007fac96c9852c in pconnection_done (pc=0x7fac5e6055f0) at /usr/src/debug/qpid-proton-0.28.0/c/src/proactor/epoll.c:1044
#13 pn_proactor_done (p=<optimized out>, batch=batch@entry=0x7fac5e605ba0) at /usr/src/debug/qpid-proton-0.28.0/c/src/proactor/epoll.c:2166
#14 0x00007fac9714abee in thread_run (arg=0x1ae2bd0) at /usr/src/debug/qpid-dispatch-1.9.0/src/server.c:1029
#15 0x00007fac96a7cea5 in start_thread () from /lib64/libpthread.so.0
#16 0x00007fac95da28cd in lseek64 () from /lib64/libc.so.6
#17 0x0000000000000000 in ?? ()
(gdb) info threads
Id Target Id Frame
10 LWP 17163 0x00007fac95da2ea3 in get_kernel_syms () from /lib64/libc.so.6
9 LWP 17157 0x00007fac96a80a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
8 LWP 17138 0x00007fac95da2ea3 in get_kernel_syms () from /lib64/libc.so.6
7 LWP 17159 0x00007fac95da2ea3 in get_kernel_syms () from /lib64/libc.so.6
6 LWP 17165 0x00007fac95da2ea3 in get_kernel_syms () from /lib64/libc.so.6
5 LWP 17164 0x00007fac95da2ea3 in get_kernel_syms () from /lib64/libc.so.6
4 LWP 17162 0x00007fac95da2ea3 in get_kernel_syms () from /lib64/libc.so.6
3 LWP 17160 0x00007fac95da2ea3 in get_kernel_syms () from /lib64/libc.so.6
2 LWP 17158 0x00007fac95d97c2d in ppoll () from /lib64/libc.so.6
* 1 LWP 17161 0x00007fac95cda377 in killpg () from /lib64/libc.so.6
(gdb) up
#1 0x00007fac95cdba68 in abort () from /lib64/libc.so.6
(gdb)
#2 0x00007fac96c942b2 in rearm (p=<optimized out>, ee=<optimized out>) at /usr/src/debug/qpid-proton-0.28.0/c/src/proactor/epoll.c:715
715 static void rearm(pn_proactor_t *p, epoll_extended_t *ee) {
(gdb)
#3 0x00007fac96c94ca6 in rearm (ee=0x4309, p=<optimized out>) at /usr/src/debug/qpid-proton-0.28.0/c/src/proactor/epoll.c:715
715 static void rearm(pn_proactor_t *p, epoll_extended_t *ee) {
(gdb)
#4 0x00007fac00007fac in ?? ()
(gdb)
#5 0x0000000001af08a0 in ?? ()
(gdb)
#6 0xf60b914474d73800 in ?? ()
(gdb)
#7 0x00007fac5e605600 in ?? ()
(gdb)
#8 0x00007fac5e605a90 in ?? ()
(gdb)
#9 0x00007fac5e605600 in ?? ()
(gdb)
#10 0x00007fac96c95673 in lock (m=0x4) at /usr/src/debug/qpid-proton-0.28.0/c/src/proactor/epoll.c:112
112 static inline void lock(pmutex *m) { pthread_mutex_lock(m); }
(gdb)
#11 pconnection_cleanup (pc=0x7fac5e605b48) at /usr/src/debug/qpid-proton-0.28.0/c/src/proactor/epoll.c:901
901 lock(&pc->context.mutex);
(gdb)
#12 0x00007fac96c9852c in pconnection_done (pc=0x7fac5e6055f0) at /usr/src/debug/qpid-proton-0.28.0/c/src/proactor/epoll.c:1044
1044 pconnection_cleanup(pc);
(gdb)
#13 pn_proactor_done (p=<optimized out>, batch=batch@entry=0x7fac5e605ba0) at /usr/src/debug/qpid-proton-0.28.0/c/src/proactor/epoll.c:2166
2166 pconnection_done(pc);
(gdb)
#14 0x00007fac9714abee in thread_run (arg=0x1ae2bd0) at /usr/src/debug/qpid-dispatch-1.9.0/src/server.c:1029
1029 pn_proactor_done(qd_server->proactor, events);
(gdb)
#15 0x00007fac96a7cea5 in start_thread () from /lib64/libpthread.so.0
(gdb)