It seems like there are two code paths that have the potential to get into deadlock due to locking order.Found one Java-level deadlock:
=============================
"pool-43-thread-1":
waiting to lock monitor 0x00007f54dc00d888 (object 0x00000000e2f84b60, a io.fabric8.mq.fabric.ActiveMQServiceFactory),
which is held by "pool-11-thread-1"
"pool-11-thread-1":
waiting to lock monitor 0x00007f54d8cbf048 (object 0x00000000e2f84ca8, a io.fabric8.mq.fabric.ActiveMQServiceFactory$ClusteredConfiguration),
which is held by "pool-43-thread-1"Java stack information for the threads listed above:
===================================================
"pool-43-thread-1":
at io.fabric8.mq.fabric.ActiveMQServiceFactory.return_pool(ActiveMQServiceFactory.java:232)waiting to lock <0x00000000e2f84b60> (a io.fabric8.mq.fabric.ActiveMQServiceFactory)
at io.fabric8.mq.fabric.ActiveMQServiceFactory$ClusteredConfiguration.updateCurator(ActiveMQServiceFactory.java:649)locked <0x00000000e2f84ca8> (a io.fabric8.mq.fabric.ActiveMQServiceFactory$ClusteredConfiguration)
at io.fabric8.mq.fabric.ActiveMQServiceFactory$ClusteredConfiguration$2.run(ActiveMQServiceFactory.java:523)
at org.apache.activemq.broker.BrokerService.stop(BrokerService.java:838)locked <0x00000000e3213ec8> (a java.util.ArrayList)
at org.apache.activemq.xbean.XBeanBrokerService.stop(XBeanBrokerService.java:122)
at io.fabric8.mq.fabric.ActiveMQServiceFactory$ClusteredConfiguration.doStop(ActiveMQServiceFactory.java:595)
at io.fabric8.mq.fabric.ActiveMQServiceFactory$ClusteredConfiguration.access$1300(ActiveMQServiceFactory.java:317)
at io.fabric8.mq.fabric.ActiveMQServiceFactory$ClusteredConfiguration$3.run(ActiveMQServiceFactory.java:585)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
"pool-11-thread-1":
at io.fabric8.mq.fabric.ActiveMQServiceFactory$ClusteredConfiguration.close(ActiveMQServiceFactory.java:562)waiting to lock <0x00000000e2f84ca8> (a io.fabric8.mq.fabric.ActiveMQServiceFactory$ClusteredConfiguration)
at io.fabric8.mq.fabric.ActiveMQServiceFactory.deleted(ActiveMQServiceFactory.java:270)locked <0x00000000e2f84b60> (a io.fabric8.mq.fabric.ActiveMQServiceFactory)
at io.fabric8.mq.fabric.StandaloneBrokerDeploymentManager.deleted(StandaloneBrokerDeploymentManager.java:69)
at io.fabric8.mq.fabric.StandaloneBrokerDeployment.deactivate(StandaloneBrokerDeployment.java:79)
at io.fabric8.mq.fabric.ClusteredBrokerDeployment.deactivate(ClusteredBrokerDeployment.java:65)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.felix.scr.impl.helper.BaseMethod.invokeMethod(BaseMethod.java:231)
at org.apache.felix.scr.impl.helper.BaseMethod.access$500(BaseMethod.java:39)
at org.apache.felix.scr.impl.helper.BaseMethod$Resolved.invoke(BaseMethod.java:624)
at org.apache.felix.scr.impl.helper.BaseMethod.invoke(BaseMethod.java:508)
at org.apache.felix.scr.impl.helper.ActivateMethod.invoke(ActivateMethod.java:149)
at org.apache.felix.scr.impl.manager.SingleComponentManager.disposeImplementationObject(SingleComponentManager.java:355)
at org.apache.felix.scr.impl.manager.SingleComponentManager.deleteComponent(SingleComponentManager.java:170)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.doDeactivate(AbstractComponentManager.java:908)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.deactivateInternal(AbstractComponentManager.java:883)
at org.apache.felix.scr.impl.manager.DependencyManager$SingleStaticCustomizer.removedService(DependencyManager.java:974)
at org.apache.felix.scr.impl.manager.DependencyManager$SingleStaticCustomizer.removedService(DependencyManager.java:895)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.customizerRemoved(ServiceTracker.java:1506)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.customizerRemoved(ServiceTracker.java:1401)
at org.apache.felix.scr.impl.manager.ServiceTracker$AbstractTracked.untrack(ServiceTracker.java:1261)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.serviceChanged(ServiceTracker.java:1440)
at org.apache.felix.framework.util.EventDispatcher.invokeServiceListenerCallback(EventDispatcher.java:943)
at org.apache.felix.framework.util.EventDispatcher.fireEventImmediately(EventDispatcher.java:794)
at org.apache.felix.framework.util.EventDispatcher.fireServiceEvent(EventDispatcher.java:544)
at org.apache.felix.framework.Felix.fireServiceEvent(Felix.java:4445)
at org.apache.felix.framework.Felix.access$000(Felix.java:77)
at org.apache.felix.framework.Felix$1.serviceChanged(Felix.java:404)
at org.apache.felix.framework.ServiceRegistry.unregisterService(ServiceRegistry.java:153)
at org.apache.felix.framework.ServiceRegistrationImpl.unregister(ServiceRegistrationImpl.java:128)
at org.apache.felix.scr.impl.manager.AbstractComponentManager$3.unregister(AbstractComponentManager.java:1011)
at org.apache.felix.scr.impl.manager.AbstractComponentManager$3.unregister(AbstractComponentManager.java:992)
at org.apache.felix.scr.impl.manager.RegistrationManager.changeRegistration(RegistrationManager.java:141)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.unregisterService(AbstractComponentManager.java:1054)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.doDeactivate(AbstractComponentManager.java:900)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.deactivateInternal(AbstractComponentManager.java:883)
at org.apache.felix.scr.impl.manager.DependencyManager$SingleStaticCustomizer.removedService(DependencyManager.java:974)
at org.apache.felix.scr.impl.manager.DependencyManager$SingleStaticCustomizer.removedService(DependencyManager.java:895)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.customizerRemoved(ServiceTracker.java:1506)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.customizerRemoved(ServiceTracker.java:1401)
at org.apache.felix.scr.impl.manager.ServiceTracker$AbstractTracked.untrack(ServiceTracker.java:1261)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.serviceChanged(ServiceTracker.java:1440)
at org.apache.felix.framework.util.EventDispatcher.invokeServiceListenerCallback(EventDispatcher.java:943)
at org.apache.felix.framework.util.EventDispatcher.fireEventImmediately(EventDispatcher.java:794)
at org.apache.felix.framework.util.EventDispatcher.fireServiceEvent(EventDispatcher.java:544)
at org.apache.felix.framework.Felix.fireServiceEvent(Felix.java:4445)
at org.apache.felix.framework.Felix.access$000(Felix.java:77)
at org.apache.felix.framework.Felix$1.serviceChanged(Felix.java:404)
at org.apache.felix.framework.ServiceRegistry.unregisterService(ServiceRegistry.java:153)
at org.apache.felix.framework.ServiceRegistrationImpl.unregister(ServiceRegistrationImpl.java:128)
at org.apache.felix.scr.impl.manager.AbstractComponentManager$3.unregister(AbstractComponentManager.java:1011)
at org.apache.felix.scr.impl.manager.AbstractComponentManager$3.unregister(AbstractComponentManager.java:992)
at org.apache.felix.scr.impl.manager.RegistrationManager.changeRegistration(RegistrationManager.java:141)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.unregisterService(AbstractComponentManager.java:1054)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.doDeactivate(AbstractComponentManager.java:900)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.deactivateInternal(AbstractComponentManager.java:883)
at org.apache.felix.scr.impl.manager.DependencyManager$SingleStaticCustomizer.removedService(DependencyManager.java:974)
at org.apache.felix.scr.impl.manager.DependencyManager$SingleStaticCustomizer.removedService(DependencyManager.java:895)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.customizerRemoved(ServiceTracker.java:1506)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.customizerRemoved(ServiceTracker.java:1401)
at org.apache.felix.scr.impl.manager.ServiceTracker$AbstractTracked.untrack(ServiceTracker.java:1261)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.serviceChanged(ServiceTracker.java:1440)
at org.apache.felix.framework.util.EventDispatcher.invokeServiceListenerCallback(EventDispatcher.java:943)
at org.apache.felix.framework.util.EventDispatcher.fireEventImmediately(EventDispatcher.java:794)
at org.apache.felix.framework.util.EventDispatcher.fireServiceEvent(EventDispatcher.java:544)
at org.apache.felix.framework.Felix.fireServiceEvent(Felix.java:4445)
at org.apache.felix.framework.Felix.access$000(Felix.java:77)
at org.apache.felix.framework.Felix$1.serviceChanged(Felix.java:404)
at org.apache.felix.framework.ServiceRegistry.unregisterService(ServiceRegistry.java:153)
at org.apache.felix.framework.ServiceRegistrationImpl.unregister(ServiceRegistrationImpl.java:128)
at org.apache.felix.scr.impl.manager.AbstractComponentManager$3.unregister(AbstractComponentManager.java:1011)
at org.apache.felix.scr.impl.manager.AbstractComponentManager$3.unregister(AbstractComponentManager.java:992)
at org.apache.felix.scr.impl.manager.RegistrationManager.changeRegistration(RegistrationManager.java:141)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.unregisterService(AbstractComponentManager.java:1054)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.doDeactivate(AbstractComponentManager.java:900)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.deactivateInternal(AbstractComponentManager.java:883)
at org.apache.felix.scr.impl.manager.DependencyManager$SingleStaticCustomizer.removedService(DependencyManager.java:974)
at org.apache.felix.scr.impl.manager.DependencyManager$SingleStaticCustomizer.removedService(DependencyManager.java:895)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.customizerRemoved(ServiceTracker.java:1506)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.customizerRemoved(ServiceTracker.java:1401)
at org.apache.felix.scr.impl.manager.ServiceTracker$AbstractTracked.untrack(ServiceTracker.java:1261)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.serviceChanged(ServiceTracker.java:1440)
at org.apache.felix.framework.util.EventDispatcher.invokeServiceListenerCallback(EventDispatcher.java:943)
at org.apache.felix.framework.util.EventDispatcher.fireEventImmediately(EventDispatcher.java:794)
at org.apache.felix.framework.util.EventDispatcher.fireServiceEvent(EventDispatcher.java:544)
at org.apache.felix.framework.Felix.fireServiceEvent(Felix.java:4445)
at org.apache.felix.framework.Felix.access$000(Felix.java:77)
at org.apache.felix.framework.Felix$1.serviceChanged(Felix.java:404)
at org.apache.felix.framework.ServiceRegistry.unregisterService(ServiceRegistry.java:153)
at org.apache.felix.framework.ServiceRegistrationImpl.unregister(ServiceRegistrationImpl.java:128)
at org.apache.felix.scr.impl.manager.AbstractComponentManager$3.unregister(AbstractComponentManager.java:1011)
at org.apache.felix.scr.impl.manager.AbstractComponentManager$3.unregister(AbstractComponentManager.java:992)
at org.apache.felix.scr.impl.manager.RegistrationManager.changeRegistration(RegistrationManager.java:141)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.unregisterService(AbstractComponentManager.java:1054)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.doDeactivate(AbstractComponentManager.java:900)
at org.apache.felix.scr.impl.manager.AbstractComponentManager.deactivateInternal(AbstractComponentManager.java:883)
at org.apache.felix.scr.impl.manager.DependencyManager$SingleStaticCustomizer.removedService(DependencyManager.java:974)
at org.apache.felix.scr.impl.manager.DependencyManager$SingleStaticCustomizer.removedService(DependencyManager.java:895)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.customizerRemoved(ServiceTracker.java:1506)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.customizerRemoved(ServiceTracker.java:1401)
at org.apache.felix.scr.impl.manager.ServiceTracker$AbstractTracked.untrack(ServiceTracker.java:1261)
at org.apache.felix.scr.impl.manager.ServiceTracker$Tracked.serviceChanged(ServiceTracker.java:1440)
at org.apache.felix.framework.util.EventDispatcher.invokeServiceListenerCallback(EventDispatcher.java:943)
at org.apache.felix.framework.util.EventDispatcher.fireEventImmediately(EventDispatcher.java:794)
at org.apache.felix.framework.util.EventDispatcher.fireServiceEvent(EventDispatcher.java:544)
at org.apache.felix.framework.Felix.fireServiceEvent(Felix.java:4445)
at org.apache.felix.framework.Felix.access$000(Felix.java:77)
at org.apache.felix.framework.Felix$1.serviceChanged(Felix.java:404)
at org.apache.felix.framework.ServiceRegistry.unregisterService(ServiceRegistry.java:153)
at org.apache.felix.framework.ServiceRegistrationImpl.unregister(ServiceRegistrationImpl.java:128)
at io.fabric8.zookeeper.curator.ManagedCuratorFramework$State.run(ManagedCuratorFramework.java:116)
at io.fabric8.zookeeper.curator.ManagedCuratorFramework$State.stateChanged(ManagedCuratorFramework.java:159)
at org.apache.curator.framework.state.ConnectionStateManager$2.apply(ConnectionStateManager.java:259)
at org.apache.curator.framework.state.ConnectionStateManager$2.apply(ConnectionStateManager.java:255)
at org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:92)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)Found 1 deadlock.