Major We had a recent incident involving a Fuse instance that failed to shutdown, even after the 300 second timeout to finish exchanges. From the thread dumps taken, it appears that the FelixShutdown thread is hung, waiting for the jetty container to stop:
"FelixShutdown" #5721 daemon prio=5 os_prio=0 tid=0x000000001c4ca000 nid=0x1010 waiting on condition [0x0000000060d7d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e4563f60> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at org.eclipse.jetty.io.ManagedSelector.doStop(ManagedSelector.java:136) at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:93) - locked <0x00000000e2f589d8> (a java.lang.Object) at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180) at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:201) at org.eclipse.jetty.io.SelectorManager.doStop(SelectorManager.java:281) at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:93) - locked <0x00000000e2f588c0> (a java.lang.Object) at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180) at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:201) at org.eclipse.jetty.server.AbstractConnector.doStop(AbstractConnector.java:374) at org.eclipse.jetty.server.AbstractNetworkConnector.doStop(AbstractNetworkConnector.java:88) at org.eclipse.jetty.server.ServerConnector.doStop(ServerConnector.java:246) at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:93) - locked <0x00000000e2f6ca20> (a java.lang.Object) at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180) at org.eclipse.jetty.util.component.ContainerLifeCycle.remove(ContainerLifeCycle.java:642) at org.eclipse.jetty.util.component.ContainerLifeCycle.removeBean(ContainerLifeCycle.java:618) at org.eclipse.jetty.server.Server.removeConnector(Server.java:240) at org.apache.camel.component.jetty.JettyHttpComponent.disconnect(JettyHttpComponent.java:516) - locked <0x00000000e2f5f7e0> (a java.util.HashMap) at org.apache.camel.http.common.HttpCommonEndpoint.disconnect(HttpCommonEndpoint.java:183) at org.apache.camel.http.common.HttpConsumer.doStop(HttpConsumer.java:68) at org.apache.camel.support.ServiceSupport.stop(ServiceSupport.java:119) at org.apache.camel.util.ServiceHelper.stopService(ServiceHelper.java:142) at org.apache.camel.impl.DefaultShutdownStrategy.shutdownNow(DefaultShutdownStrategy.java:381) at org.apache.camel.impl.DefaultShutdownStrategy.shutdownRoutesNow(DefaultShutdownStrategy.java:355) at org.apache.camel.impl.DefaultShutdownStrategy.doShutdown(DefaultShutdownStrategy.java:238) at org.apache.camel.impl.DefaultShutdownStrategy.shutdownForced(DefaultShutdownStrategy.java:145) at org.apache.camel.impl.DefaultCamelContext.doStop(DefaultCamelContext.java:3505) - locked <0x00000000e2f12548> (a org.apache.camel.blueprint.BlueprintCamelContext) at org.apache.camel.support.ServiceSupport.stop(ServiceSupport.java:119) at org.apache.camel.impl.DefaultCamelContext.stop(DefaultCamelContext.java:3215) at org.apache.camel.blueprint.BlueprintCamelContext.destroy(BlueprintCamelContext.java:141) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.aries.blueprint.utils.ReflectionUtils.invoke(ReflectionUtils.java:337) at org.apache.aries.blueprint.container.BeanRecipe.invoke(BeanRecipe.java:835) at org.apache.aries.blueprint.container.BeanRecipe.destroy(BeanRecipe.java:742) at org.apache.aries.blueprint.container.BlueprintRepository.destroy(BlueprintRepository.java:434) at org.apache.aries.blueprint.container.BlueprintContainerImpl.destroyComponents(BlueprintContainerImpl.java:778) at org.apache.aries.blueprint.container.BlueprintContainerImpl.tidyupComponents(BlueprintContainerImpl.java:987) at org.apache.aries.blueprint.container.BlueprintContainerImpl.destroy(BlueprintContainerImpl.java:923) at org.apache.aries.blueprint.container.BlueprintExtender$3.run(BlueprintExtender.java:336) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.aries.blueprint.container.BlueprintExtender.destroyContainer(BlueprintExtender.java:357) at org.apache.aries.blueprint.container.BlueprintExtender.stop(BlueprintExtender.java:191) at org.apache.aries.blueprint.container.BlueprintExtender.modifiedBundle(BlueprintExtender.java:240) at org.apache.aries.util.tracker.hook.BundleHookBundleTracker$Tracked.customizerModified(BundleHookBundleTracker.java:500) at org.apache.aries.util.tracker.hook.BundleHookBundleTracker$Tracked.customizerModified(BundleHookBundleTracker.java:433) at org.apache.aries.util.tracker.hook.BundleHookBundleTracker$AbstractTracked.track(BundleHookBundleTracker.java:725) at org.apache.aries.util.tracker.hook.BundleHookBundleTracker$Tracked.bundleChanged(BundleHookBundleTracker.java:463) at org.apache.aries.util.tracker.hook.BundleHookBundleTracker$BundleEventHook.event(BundleHookBundleTracker.java:422) at org.apache.felix.framework.util.SecureAction.invokeBundleEventHook(SecureAction.java:1179) at org.apache.felix.framework.EventDispatcher.createWhitelistFromHooks(EventDispatcher.java:730) at org.apache.felix.framework.EventDispatcher.fireBundleEvent(EventDispatcher.java:485) at org.apache.felix.framework.Felix.fireBundleEvent(Felix.java:4579) at org.apache.felix.framework.Felix.stopBundle(Felix.java:2626) at org.apache.felix.framework.Felix$2.run(Felix.java:1034)
But there is no obvious cause, as the qtp threads seem all to be in timed waits or periodic polls:
"qtp1424286270-5287" #5287 prio=5 os_prio=0 tid=0x000000001c4d6000 nid=0x3310 runnable [0x000000007677e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:314) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:293) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000e2f5c7d8> (a sun.nio.ch.Util$3) - locked <0x00000000e2f5c7c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000e2f5c668> (a sun.nio.ch.WindowsSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149) at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:375) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.lang.Thread.run(Thread.java:748)
There are several undertow threads waiting for the blueprint context held by the FelixShutdownThread:
"XNIO-3 task-40" #168 prio=5 os_prio=0 tid=0x0000000017a5e000 nid=0x2e54 waiting for monitor entry [0x000000002b09a000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.camel.support.ServiceSupport.getVersion(ServiceSupport.java:341) - waiting to lock <0x00000000e2f12548> (a org.apache.camel.blueprint.BlueprintCamelContext) at org.apache.camel.management.mbean.ManagedCamelContext.getCamelVersion(ManagedCamelContext.java:108) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:72) at sun.reflect.GeneratedMethodAccessor1153.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:276) at javax.management.modelmbean.RequiredModelMBean$4.run(RequiredModelMBean.java:1252) at java.security.AccessController.doPrivileged(Native Method) at java.security.ProtectionDomain$JavaSecurityAccessImpl.doIntersectionPrivilege(ProtectionDomain.java:74) at javax.management.modelmbean.RequiredModelMBean.invokeMethod(RequiredModelMBean.java:1246) at javax.management.modelmbean.RequiredModelMBean.invoke(RequiredModelMBean.java:1085) at org.apache.camel.management.MixinRequiredModelMBean.invoke(MixinRequiredModelMBean.java:70) at javax.management.modelmbean.RequiredModelMBean.getAttribute(RequiredModelMBean.java:1562) at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:647) at com.sun.jmx.mbeanserver.JmxMBeanServer.getAttribute(JmxMBeanServer.java:678) at org.apache.karaf.management.internal.EventAdminMBeanServerWrapper.getAttribute(EventAdminMBeanServerWrapper.java:304) at org.jolokia.handler.ReadHandler.doHandleRequest(ReadHandler.java:106) at org.jolokia.handler.ReadHandler.doHandleRequest(ReadHandler.java:37) at org.jolokia.handler.JsonRequestHandler.handleRequest(JsonRequestHandler.java:89) at org.jolokia.backend.MBeanServerExecutorLocal.handleRequest(MBeanServerExecutorLocal.java:109) at org.jolokia.backend.MBeanServerHandler.dispatchRequest(MBeanServerHandler.java:161) at org.jolokia.backend.LocalRequestDispatcher.dispatchRequest(LocalRequestDispatcher.java:99) at org.jolokia.backend.BackendManager.callRequestDispatcher(BackendManager.java:429) at org.jolokia.backend.BackendManager.handleRequest(BackendManager.java:158) at org.jolokia.http.HttpRequestHandler.executeRequest(HttpRequestHandler.java:197) at org.jolokia.http.HttpRequestHandler.handlePostRequest(HttpRequestHandler.java:137) at org.jolokia.http.AgentServlet$3.handleRequest(AgentServlet.java:460) at org.jolokia.http.AgentServlet.handleSecurely(AgentServlet.java:350) at org.jolokia.http.AgentServlet.handle(AgentServlet.java:321) at org.jolokia.http.AgentServlet.doPost(AgentServlet.java:284) at javax.servlet.http.HttpServlet.service(HttpServlet.java:665) at javax.servlet.http.HttpServlet.service(HttpServlet.java:750) at io.undertow.servlet.handlers.ServletHandler.handleRequest(ServletHandler.java:74) at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:129) at io.hawt.web.filters.HttpHeaderFilter.doFilter(HttpHeaderFilter.java:43) at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61) at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131) at io.hawt.web.filters.HttpHeaderFilter.doFilter(HttpHeaderFilter.java:43) at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61) at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131) at io.hawt.web.filters.HttpHeaderFilter.doFilter(HttpHeaderFilter.java:43) at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61) at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131) at io.hawt.web.filters.HttpHeaderFilter.doFilter(HttpHeaderFilter.java:43) at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61) at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131) at io.hawt.web.filters.HttpHeaderFilter.doFilter(HttpHeaderFilter.java:43) at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61) at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131) at io.hawt.web.auth.SessionExpiryFilter.process(SessionExpiryFilter.java:107) at io.hawt.web.auth.SessionExpiryFilter.doFilter(SessionExpiryFilter.java:60) at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61) at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131) at io.hawt.web.filters.HttpHeaderFilter.doFilter(HttpHeaderFilter.java:43) at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61) at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131) at io.hawt.web.auth.LoginRedirectFilter.doFilter(LoginRedirectFilter.java:59) at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61) at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131) at io.hawt.web.filters.HttpHeaderFilter.doFilter(HttpHeaderFilter.java:43) at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61) at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131) at io.hawt.web.filters.HttpHeaderFilter.doFilter(HttpHeaderFilter.java:43) at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61) at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131) at io.hawt.web.auth.AuthenticationFilter.lambda$executeAs$1(AuthenticationFilter.java:106) at io.hawt.web.auth.AuthenticationFilter$$Lambda$297/76946120.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at io.hawt.web.auth.AuthenticationFilter.executeAs(AuthenticationFilter.java:105) at io.hawt.web.auth.AuthenticationFilter.lambda$doFilter$0(AuthenticationFilter.java:81) at io.hawt.web.auth.AuthenticationFilter$$Lambda$295/1547615887.accept(Unknown Source) at io.hawt.system.Authenticator.authenticate(Authenticator.java:119) at io.hawt.system.Authenticator.authenticate(Authenticator.java:96) at io.hawt.web.auth.AuthenticationFilter.doFilter(AuthenticationFilter.java:79) at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61) at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131) at io.undertow.servlet.handlers.FilterHandler.handleRequest(FilterHandler.java:84) at io.undertow.servlet.handlers.security.ServletSecurityRoleHandler.handleRequest(ServletSecurityRoleHandler.java:62) at io.undertow.servlet.handlers.ServletChain$1.handleRequest(ServletChain.java:68) at io.undertow.servlet.handlers.ServletDispatchingHandler.handleRequest(ServletDispatchingHandler.java:36) at org.ops4j.pax.web.service.undertow.internal.Context$1.lambda$wrap$0(Context.java:615) at org.ops4j.pax.web.service.undertow.internal.Context$1$$Lambda$189/1299436836.handleRequest(Unknown Source) at io.undertow.servlet.handlers.RedirectDirHandler.handleRequest(RedirectDirHandler.java:68) at io.undertow.servlet.handlers.security.SSLInformationAssociationHandler.handleRequest(SSLInformationAssociationHandler.java:132) at io.undertow.servlet.handlers.security.ServletAuthenticationCallHandler.handleRequest(ServletAuthenticationCallHandler.java:57) at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43) at io.undertow.security.handlers.AbstractConfidentialityHandler.handleRequest(AbstractConfidentialityHandler.java:46) at io.undertow.servlet.handlers.security.ServletConfidentialityConstraintHandler.handleRequest(ServletConfidentialityConstraintHandler.java:64) at io.undertow.security.handlers.AuthenticationMechanismsHandler.handleRequest(AuthenticationMechanismsHandler.java:60) at io.undertow.servlet.handlers.security.CachedAuthenticatedSessionHandler.handleRequest(CachedAuthenticatedSessionHandler.java:77) at io.undertow.security.handlers.AbstractSecurityContextAssociationHandler.handleRequest(AbstractSecurityContextAssociationHandler.java:43) at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43) at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43) at io.undertow.servlet.handlers.SessionRestoringHandler.handleRequest(SessionRestoringHandler.java:119) at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:269) at io.undertow.servlet.handlers.ServletInitialHandler.access$100(ServletInitialHandler.java:78) at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:133) at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:130) at io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:48) at io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43) at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:249) at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:78) at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:99) at io.undertow.server.Connectors.executeRootHandler(Connectors.java:390) at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:830) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
This was running on Windows Server and the task failed to exit even when taskkill was invoked.
