Uploaded image for project: 'AMQ Interconnect'
  1. AMQ Interconnect
  2. ENTMQIC-51

deadlock in qdrouterd when link routing to just created destinations

XMLWordPrintable

    • Hide

      1) Have connector to qpid broker
      2) Link routing "pulp." to qpid broker
      3) Start qdrouterd
      4) In one terminal, create queues on the broker:

      for j in $(seq 0 10); do for i in $(seq 0 50); do qpid-config add queue pulp.pmoravec.${j}.${i}; done; echo $j; done

      5) In another terminal, run concurrently script that creates consumers for nodes being concurrently created in qpid broker (and - IMPORTANT - due to heartbeats enabled, the attach frame for the link is followed by close frame due to timeouted connection):

      #!/usr/bin/python

from time import sleep
from uuid import uuid4

from proton import ConnectionException, Timeout
from proton import SSLDomain, SSLException

from proton.utils import BlockingConnection

import random
import threading
import traceback

ROUTER_ADDRESS = "proton+amqp://pmoravec-sat61-rhel7.gsslab.brq.redhat.com:5648"
ADDRESS = "pulp.pmoravec"
HEARTBEAT = 2
SLEEP_MIN = 1.9
SLEEP_MAX = 2.1
THREADS = 10

class ReceiverThread(threading.Thread):
    def __init__(self, _id, address=ADDRESS, domain=None):
        super(ReceiverThread, self).__init__()
        self._id = _id
        self.address = address
        self.domain = domain
        self.running = True
        self.nr = 1

    def connect(self):
        self.conn = BlockingConnection(ROUTER_ADDRESS, ssl_domain=self.domain, heartbeat=HEARTBEAT)
        self.conn.create_receiver('%s.%s' %(self.address, 0), name=str(uuid4()), dynamic=False, options=None)

    def reconnect(self):
        print "(%s): something got broken, reconnecting.." % self._id
        try:
            self.conn.close()
        except:
            print "(%s): receiver thread: failed to close connection" % self._id
            pass
        self.connect()

    def run(self):
        self.connect()
        while self.running:
            sleep(random.uniform(SLEEP_MIN,SLEEP_MAX))
            self.nr += 1
            try:
                self.recv = self.conn.create_receiver('%s.%s' %(self.address, self.nr), name=str(uuid4()), dynamic=False, options=None)
            except Exception as e:
                print "(%s): receiver failed, retrying.." % self._id
                self.reconnect()

    def stop(self):
        self.running = False

threads = []
for i in range(THREADS):
  threads.append(ReceiverThread(i, '%s.%s' %(ADDRESS, i)))
  threads[i].start()

_in = raw_input("Press Enter to exit:")

for i in range(THREADS):
    threads[i].stop()
for i in range(THREADS):
    threads[i].join()

      6) Wait some time, qdrouterd will log "send: Connection reset by peer" and suddenly stop responding

      Show
      1) Have connector to qpid broker 2) Link routing "pulp." to qpid broker 3) Start qdrouterd 4) In one terminal, create queues on the broker: for j in $(seq 0 10); do for i in $(seq 0 50); do qpid-config add queue pulp.pmoravec.${j}.${i}; done; echo $j; done 5) In another terminal, run concurrently script that creates consumers for nodes being concurrently created in qpid broker (and - IMPORTANT - due to heartbeats enabled, the attach frame for the link is followed by close frame due to timeouted connection): #!/usr/bin/python from time import sleep from uuid import uuid4 from proton import ConnectionException, Timeout from proton import SSLDomain, SSLException from proton.utils import BlockingConnection import random import threading import traceback ROUTER_ADDRESS = "proton+amqp: //pmoravec-sat61-rhel7.gsslab.brq.redhat.com:5648" ADDRESS = "pulp.pmoravec" HEARTBEAT = 2 SLEEP_MIN = 1.9 SLEEP_MAX = 2.1 THREADS = 10 class ReceiverThread(threading. Thread ): def __init__(self, _id, address=ADDRESS, domain=None): super (ReceiverThread, self).__init__() self._id = _id self.address = address self.domain = domain self.running = True self.nr = 1 def connect(self): self.conn = BlockingConnection(ROUTER_ADDRESS, ssl_domain=self.domain, heartbeat=HEARTBEAT) self.conn.create_receiver( '%s.%s' %(self.address, 0), name=str(uuid4()), dynamic=False, options=None) def reconnect(self): print "(%s): something got broken, reconnecting.." % self._id try : self.conn.close() except: print "(%s): receiver thread: failed to close connection" % self._id pass self.connect() def run(self): self.connect() while self.running: sleep(random.uniform(SLEEP_MIN,SLEEP_MAX)) self.nr += 1 try : self.recv = self.conn.create_receiver( '%s.%s' %(self.address, self.nr), name=str(uuid4()), dynamic=False, options=None) except Exception as e: print "(%s): receiver failed, retrying.." % self._id self.reconnect() def stop(self): self.running = False threads = [] for i in range(THREADS): threads.append(ReceiverThread(i, '%s.%s' %(ADDRESS, i))) threads[i].start() _in = raw_input( "Press Enter to exit:" ) for i in range(THREADS): threads[i].stop() for i in range(THREADS): threads[i].join() 6) Wait some time, qdrouterd will log "send: Connection reset by peer" and suddenly stop responding

      qdrouterd becomes unresponsive (qdstat timeouts, any connection frozen,..) when trying to link route to a destination just being created on other side. See reproducer for details.

      Coredump (gcore generated) available on 10.34.84.149 (root/redhat) as core.24217 (big to attach), backtraces are:

      Thread 8 (Thread 0x7f8f42e171c0 (LWP 24217)):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f8f42570d32 in _L_lock_791 () from /lib64/libpthread.so.0
#2  0x00007f8f42570c38 in __GI___pthread_mutex_lock (mutex=0x143cb20) at pthread_mutex_lock.c:64
#3  0x00007f8f429f3715 in sys_mutex_lock (mutex=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/posix/threading.c:62
#4  0x00007f8f429f6b50 in router_writable_link_handler (context=0x13d4030, link=0x7f8f1c034130)
    at /usr/src/debug/qpid-dispatch-0.4/src/router_node.c:460
#5  0x00007f8f429ea4d6 in do_writable (pn_link=0x7f8f1c042fb0) at /usr/src/debug/qpid-dispatch-0.4/src/container.c:215
#6  process_handler (unused=<optimized out>, qd_conn=<optimized out>, container=0x13b8650)
    at /usr/src/debug/qpid-dispatch-0.4/src/container.c:446
#7  handler (handler_context=0x13b8650, conn_context=<optimized out>, event=event@entry=QD_CONN_EVENT_PROCESS, 
    qd_conn=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/container.c:496
#8  0x00007f8f429fc9ac in process_connector (cxtr=0x7f8f14048e30, qd_server=0x13dd5d0)
    at /usr/src/debug/qpid-dispatch-0.4/src/server.c:398
#9  thread_run (arg=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/server.c:626
#10 0x00007f8f429fd5c0 in qd_server_run (qd=0x119f030) at /usr/src/debug/qpid-dispatch-0.4/src/server.c:959
#11 0x0000000000401cd8 in main_process (config_path=config_path@entry=0x7ffff625a6be "/etc/qpid-dispatch/qdrouterd.conf", 
    python_pkgdir=python_pkgdir@entry=0x402401 "/usr/lib/qpid-dispatch/python", fd=fd@entry=2)
    at /usr/src/debug/qpid-dispatch-0.4/router/src/main.c:135
#12 0x0000000000401950 in main (argc=3, argv=0x7ffff62594a8) at /usr/src/debug/qpid-dispatch-0.4/router/src/main.c:335

Thread 7 (Thread 0x7f8f2effd700 (LWP 24224)):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007f8f429fc819 in thread_run (arg=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/server.c:508
#2  0x00007f8f4256edf5 in start_thread (arg=0x7f8f2effd700) at pthread_create.c:308
#3  0x00007f8f41aca1ad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113

Thread 6 (Thread 0x7f8f2f7fe700 (LWP 24223)):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f8f42570d32 in _L_lock_791 () from /lib64/libpthread.so.0
#2  0x00007f8f42570c38 in __GI___pthread_mutex_lock (mutex=0x143cb20) at pthread_mutex_lock.c:64
#3  0x00007f8f429f3715 in sys_mutex_lock (mutex=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/posix/threading.c:62
#4  0x00007f8f429fa1c2 in qd_router_send (qd=qd@entry=0x119f030, address=address@entry=0x7f8f2000c0a0, msg=msg@entry=0x7f8f2000faa0)
    at /usr/src/debug/qpid-dispatch-0.4/src/router_node.c:2140
#5  0x00007f8f429fa438 in qd_router_send2 (qd=0x119f030, address=<optimized out>, msg=msg@entry=0x7f8f2000faa0)
    at /usr/src/debug/qpid-dispatch-0.4/src/router_node.c:2208
#6  0x00007f8f429f4425 in qd_python_send (self=<dispatch.IoAdapter at remote 0x7f8f36265ee0>, args=<optimized out>)
    at /usr/src/debug/qpid-dispatch-0.4/src/python_embedded.c:608
#7  0x00007f8f41e75b94 in call_function (oparg=<optimized out>, pp_stack=0x7f8f2f7fcc90)
    at /usr/src/debug/Python-2.7.5/Python/ceval.c:4098
#8  PyEval_EvalFrameEx (
    f=f@entry=Frame 0x7f8f240010b0, for file /usr/lib/qpid-dispatch/python/qpid_dispatch_internal/router/engine.py, line 234, in send (self=<RouterEngine(node_tracker=<NodeTracker(neighbor_max_age=3, next_maskbit=1, container=<...>, nodes_by_link_id={}, maskbits=[True, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ...(truncated), 
    throwflag=throwflag@entry=0) at /usr/src/debug/Python-2.7.5/Python/ceval.c:2740
#9  0x00007f8f41e75950 in fast_function (nk=<optimized out>, na=3, n=3, pp_stack=0x7f8f2f7fcdf0, 
    func=<function at remote 0x7f8f42cb3848>) at /usr/src/debug/Python-2.7.5/Python/ceval.c:4184
#10 call_function (oparg=<optimized out>, pp_stack=0x7f8f2f7fcdf0) at /usr/src/debug/Python-2.7.5/Python/ceval.c:4119
#11 PyEval_EvalFrameEx (
    f=f@entry=Frame 0x7f8f24000ad0, for file /usr/lib/qpid-dispatch/python/qpid_dispatch_internal/router/hello.py, line 42, in tick (self=<HelloProtocol(container=<RouterEngine(node_tracker=<NodeTracker(neighbor_max_age=3, next_maskbit=1, container=<...>, nodes_by_link_id={}, maskbits=[True, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, N...(truncated), 
    throwflag=throwflag@entry=0) at /usr/src/debug/Python-2.7.5/Python/ceval.c:2740
#12 0x00007f8f41e75950 in fast_function (nk=<optimized out>, na=2, n=2, pp_stack=0x7f8f2f7fcf50, 
    func=<function at remote 0x7f8f42d1dde8>) at /usr/src/debug/Python-2.7.5/Python/ceval.c:4184
#13 call_function (oparg=<optimized out>, pp_stack=0x7f8f2f7fcf50) at /usr/src/debug/Python-2.7.5/Python/ceval.c:4119
#14 PyEval_EvalFrameEx (
    f=f@entry=Frame 0x7f8f240008e0, for file /usr/lib/qpid-dispatch/python/qpid_dispatch_internal/router/engine.py, line 121, in handleTimerTick (self=<RouterEngine(node_tracker=<NodeTracker(neighbor_max_age=3, next_maskbit=1, container=<...>, nodes_by_link_id={}, maskbits=[True, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, N...(truncated), 
    throwflag=throwflag@entry=0) at /usr/src/debug/Python-2.7.5/Python/ceval.c:2740
#15 0x00007f8f41e771ad in PyEval_EvalCodeEx (co=<optimized out>, globals=<optimized out>, locals=locals@entry=0x0, 
    args=args@entry=0x7f8f369ecf28, argcount=1, kws=kws@entry=0x0, kwcount=kwcount@entry=0, defs=defs@entry=0x0, 
    defcount=defcount@entry=0, closure=0x0) at /usr/src/debug/Python-2.7.5/Python/ceval.c:3330
#16 0x00007f8f41e04098 in function_call (func=<function at remote 0x7f8f42cb3488>, 
    arg=(<RouterEngine(node_tracker=<NodeTracker(neighbor_max_age=3, next_maskbit=1, container=<...>, nodes_by_link_id={}, maskbits=[True, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None,...(truncated), kw=0x0)
    at /usr/src/debug/Python-2.7.5/Objects/funcobject.c:526
#17 0x00007f8f41ddf073 in PyObject_Call (func=func@entry=<function at remote 0x7f8f42cb3488>, 
    arg=arg@entry=(<RouterEngine(node_tracker=<NodeTracker(neighbor_max_age=3, next_maskbit=1, container=<...>, nodes_by_link_id={}, maskbits=[True, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None,...(truncated), kw=kw@entry=0x0)
    at /usr/src/debug/Python-2.7.5/Objects/abstract.c:2529
#18 0x00007f8f41dee085 in instancemethod_call (func=<function at remote 0x7f8f42cb3488>, 
    arg=(<RouterEngine(node_tracker=<NodeTracker(neighbor_max_age=3, next_maskbit=1, container=<...>, nodes_by_link_id={}, maskbits=[True, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None,...(truncated), kw=0x0)
    at /usr/src/debug/Python-2.7.5/Objects/classobject.c:2602
#19 0x00007f8f41ddf073 in PyObject_Call (func=func@entry=<instancemethod at remote 0x7f8f369f0af0>, arg=arg@entry=(), 
    kw=<optimized out>) at /usr/src/debug/Python-2.7.5/Objects/abstract.c:2529
#20 0x00007f8f41e70ff7 in PyEval_CallObjectWithKeywords (func=<instancemethod at remote 0x7f8f369f0af0>, arg=arg@entry=(), 
    kw=kw@entry=0x0) at /usr/src/debug/Python-2.7.5/Python/ceval.c:3967
#21 0x00007f8f41ddf027 in PyObject_CallObject (o=<optimized out>, a=a@entry=()) at /usr/src/debug/Python-2.7.5/Objects/abstract.c:2517
#22 0x00007f8f429fb601 in qd_pyrouter_tick (router=router@entry=0x13d4030) at /usr/src/debug/qpid-dispatch-0.4/src/router_pynode.c:706
#23 0x00007f8f429f5ed9 in qd_router_timer_handler (context=0x13d4030) at /usr/src/debug/qpid-dispatch-0.4/src/router_node.c:1913
#24 0x00007f8f429fc337 in thread_run (arg=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/server.c:490
#25 0x00007f8f4256edf5 in start_thread (arg=0x7f8f2f7fe700) at pthread_create.c:308
#26 0x00007f8f41aca1ad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113

Thread 5 (Thread 0x7f8f2ffff700 (LWP 24222)):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f8f42570d32 in _L_lock_791 () from /lib64/libpthread.so.0
#2  0x00007f8f42570c38 in __GI___pthread_mutex_lock (mutex=0x143cb20) at pthread_mutex_lock.c:64
#3  0x00007f8f429f3715 in sys_mutex_lock (mutex=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/posix/threading.c:62
#4  0x00007f8f429f8f2e in router_outgoing_link_handler (context=0x13d4030, link=0x7f8f1c034230)
    at /usr/src/debug/qpid-dispatch-0.4/src/router_node.c:1508
#5  0x00007f8f429ea1a0 in setup_outgoing_link (pn_link=0x7f8f1c032f50, container=0x13b8650)
    at /usr/src/debug/qpid-dispatch-0.4/src/container.c:144
#6  process_handler (unused=<optimized out>, qd_conn=0x7f8f20028ea0, container=0x13b8650)
    at /usr/src/debug/qpid-dispatch-0.4/src/container.c:385
#7  handler (handler_context=0x13b8650, conn_context=<optimized out>, event=event@entry=QD_CONN_EVENT_PROCESS, qd_conn=0x7f8f20028ea0)
    at /usr/src/debug/qpid-dispatch-0.4/src/container.c:496
#8  0x00007f8f429fc9ac in process_connector (cxtr=0x7f8f2001bfa0, qd_server=0x13dd5d0)
    at /usr/src/debug/qpid-dispatch-0.4/src/server.c:398
#9  thread_run (arg=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/server.c:626
#10 0x00007f8f4256edf5 in start_thread (arg=0x7f8f2ffff700) at pthread_create.c:308
#11 0x00007f8f41aca1ad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113

Thread 4 (Thread 0x7f8f34807700 (LWP 24221)):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007f8f429fc819 in thread_run (arg=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/server.c:508
#2  0x00007f8f4256edf5 in start_thread (arg=0x7f8f34807700) at pthread_create.c:308
#3  0x00007f8f41aca1ad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113

Thread 3 (Thread 0x7f8f35008700 (LWP 24220)):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f8f42570d32 in _L_lock_791 () from /lib64/libpthread.so.0
#2  0x00007f8f42570c38 in __GI___pthread_mutex_lock (mutex=0x143cb20) at pthread_mutex_lock.c:64
#3  0x00007f8f429f3715 in sys_mutex_lock (mutex=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/posix/threading.c:62
#4  0x00007f8f429f6b50 in router_writable_link_handler (context=0x13d4030, link=0x7f8f3001bdd0)
    at /usr/src/debug/qpid-dispatch-0.4/src/router_node.c:460
#5  0x00007f8f429ea4d6 in do_writable (pn_link=0x7f8f3002d7c0) at /usr/src/debug/qpid-dispatch-0.4/src/container.c:215
#6  process_handler (unused=<optimized out>, qd_conn=<optimized out>, container=0x13b8650)
    at /usr/src/debug/qpid-dispatch-0.4/src/container.c:446
#7  handler (handler_context=0x13b8650, conn_context=<optimized out>, event=event@entry=QD_CONN_EVENT_PROCESS, 
    qd_conn=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/container.c:496
#8  0x00007f8f429fc9ac in process_connector (cxtr=0x7f8f1c012f30, qd_server=0x13dd5d0)
    at /usr/src/debug/qpid-dispatch-0.4/src/server.c:398
#9  thread_run (arg=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/server.c:626
#10 0x00007f8f4256edf5 in start_thread (arg=0x7f8f35008700) at pthread_create.c:308
#11 0x00007f8f41aca1ad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113

Thread 2 (Thread 0x7f8f35809700 (LWP 24219)):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f8f42570d32 in _L_lock_791 () from /lib64/libpthread.so.0
#2  0x00007f8f42570c38 in __GI___pthread_mutex_lock (mutex=0x7f8f1c033b80) at pthread_mutex_lock.c:64
#3  0x00007f8f429f3715 in sys_mutex_lock (mutex=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/posix/threading.c:62
#4  0x00007f8f429fda0e in qd_connection_invoke_deferred (conn=conn@entry=0x7f8f1c0208a0, call=call@entry=
    0x7f8f429f8a00 <qd_router_open_routed_link>, context=0x7f8f2001d600) at /usr/src/debug/qpid-dispatch-0.4/src/server.c:1135
#5  0x00007f8f429f8943 in router_link_attach_handler (context=0x13d4030, link=<optimized out>)
    at /usr/src/debug/qpid-dispatch-0.4/src/router_node.c:1632
#6  0x00007f8f429ea395 in handle_link_open (container=<optimized out>, pn_link=0x7f8f1c043990)
    at /usr/src/debug/qpid-dispatch-0.4/src/container.c:201
#7  process_handler (unused=<optimized out>, qd_conn=0x7f8f3000cb10, container=0x13b8650)
    at /usr/src/debug/qpid-dispatch-0.4/src/container.c:389
#8  handler (handler_context=0x13b8650, conn_context=<optimized out>, event=event@entry=QD_CONN_EVENT_PROCESS, qd_conn=0x7f8f3000cb10)
    at /usr/src/debug/qpid-dispatch-0.4/src/container.c:496
#9  0x00007f8f429fc9ac in process_connector (cxtr=0x7f8f30010250, qd_server=0x13dd5d0)
    at /usr/src/debug/qpid-dispatch-0.4/src/server.c:398
#10 thread_run (arg=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/server.c:626
#11 0x00007f8f4256edf5 in start_thread (arg=0x7f8f35809700) at pthread_create.c:308
#12 0x00007f8f41aca1ad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113

Thread 1 (Thread 0x7f8f3600a700 (LWP 24218)):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f8f42570d32 in _L_lock_791 () from /lib64/libpthread.so.0
#2  0x00007f8f42570c38 in __GI___pthread_mutex_lock (mutex=0x143cb20) at pthread_mutex_lock.c:64
#3  0x00007f8f429f3715 in sys_mutex_lock (mutex=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/posix/threading.c:62
#4  0x00007f8f429f8f2e in router_outgoing_link_handler (context=0x13d4030, link=0x7f8f280419a0)
    at /usr/src/debug/qpid-dispatch-0.4/src/router_node.c:1508
#5  0x00007f8f429ea1a0 in setup_outgoing_link (pn_link=0x7f8f28040380, container=0x13b8650)
    at /usr/src/debug/qpid-dispatch-0.4/src/container.c:144
#6  process_handler (unused=<optimized out>, qd_conn=0x7f8f20028dd0, container=0x13b8650)
    at /usr/src/debug/qpid-dispatch-0.4/src/container.c:385
#7  handler (handler_context=0x13b8650, conn_context=<optimized out>, event=event@entry=QD_CONN_EVENT_PROCESS, qd_conn=0x7f8f20028dd0)
    at /usr/src/debug/qpid-dispatch-0.4/src/container.c:496
#8  0x00007f8f429fc9ac in process_connector (cxtr=0x7f8f2001be10, qd_server=0x13dd5d0)
    at /usr/src/debug/qpid-dispatch-0.4/src/server.c:398
#9  thread_run (arg=<optimized out>) at /usr/src/debug/qpid-dispatch-0.4/src/server.c:626
#10 0x00007f8f4256edf5 in start_thread (arg=0x7f8f3600a700) at pthread_create.c:308
#11 0x00007f8f41aca1ad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113

              tross1@redhat.com Ted Ross
              rhn-support-pmoravec Pavel Moravec
              Matej Lesko Matej Lesko (Inactive)
              Votes:
              0 Vote for this issue
              Watchers:
              1 Start watching this issue

                Created:
                Updated:
                Resolved: