Uploaded image for project: 'AMQ Broker'
  1. AMQ Broker
  2. ENTMQBR-10140

jgroups scaledown race with shared channel

XMLWordPrintable

    • Icon: Bug Bug
    • Resolution: Unresolved
    • Icon: Undefined Undefined
    • None
    • None
    • None

      AMQ222181: Unable to scaleDown messages

      logs:
      {{
      org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd

      {refCount=1, channel=managed-scaledown51a-ss-1-24689 (activemq_broadcast_channel), channelName='activemq_broadcast_channel', connected=true}

      ::removeReceiver: org.apache.activemq.artemis.api.core.jgroups.JGroupsReceiver@642857e0 on activemq_broadcast_channel
      java.lang.Exception: Trace
      at org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.removeReceiver(JChannelWrapper.java:107)
      at org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.close(JGroupsBroadcastEndpoint.java:132)
      at org.apache.activemq.artemis.core.cluster.DiscoveryGroup.stop(DiscoveryGroup.java:168)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.doClose(ServerLocatorImpl.java:1426)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.close(ServerLocatorImpl.java:1402)
      at org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.closeLocator(ClusterConnectionImpl.java:517)
      at org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.lambda$stop$0(ClusterConnectionImpl.java:506)
      at org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:59)
      at org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:32)
      at org.apache.activemq.artemis.utils.actors.ProcessorBase.executePendingTasks(ProcessorBase.java:69)
      at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
      at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
      at org.apache.activemq.artemis.utils.ActiveMQThreadFactory$1.run(ActiveMQThreadFactory.java:120)
      org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd

      {refCount=0, channel=managed-scaledown51a-ss-1-24689 (activemq_broadcast_channel), channelName='activemq_broadcast_channel', connected=true}

      ::RefCount-- 0 on channel activemq_broadcast_channel
      java.lang.Exception: Trace
      at org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.close(JChannelWrapper.java:87)
      at org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.internalCloseChannel(JGroupsBroadcastEndpoint.java:143)
      at org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.close(JGroupsBroadcastEndpoint.java:135)
      at org.apache.activemq.artemis.core.cluster.DiscoveryGroup.stop(DiscoveryGroup.java:168)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.doClose(ServerLocatorImpl.java:1426)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.close(ServerLocatorImpl.java:1402)
      at org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.closeLocator(ClusterConnectionImpl.java:517)
      at org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.lambda$stop$0(ClusterConnectionImpl.java:506)
      at org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:59)
      at org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:32)
      at org.apache.activemq.artemis.utils.actors.ProcessorBase.executePendingTasks(ProcessorBase.java:69)
      at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
      at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
      at org.apache.activemq.artemis.utils.ActiveMQThreadFactory$1.run(ActiveMQThreadFactory.java:120)
      org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd

      {refCount=0, channel=managed-scaledown51a-ss-1-24689 (activemq_broadcast_channel), channelName='activemq_broadcast_channel', connected=false}

      ::Closing Channel: activemq_broadcast_channel
      java.lang.Exception: Trace
      at org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.closeChannel(JChannelWrapper.java:100)
      at org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.close(JChannelWrapper.java:90)
      at org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.internalCloseChannel(JGroupsBroadcastEndpoint.java:143)
      at org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.close(JGroupsBroadcastEndpoint.java:135)
      at org.apache.activemq.artemis.core.cluster.DiscoveryGroup.stop(DiscoveryGroup.java:168)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.doClose(ServerLocatorImpl.java:1426)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.close(ServerLocatorImpl.java:1402)
      at org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.closeLocator(ClusterConnectionImpl.java:517)
      at org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.lambda$stop$0(ClusterConnectionImpl.java:506)
      at org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:59)
      at org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:32)
      at org.apache.activemq.artemis.utils.actors.ProcessorBase.executePendingTasks(ProcessorBase.java:69)
      at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
      at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
      at org.apache.activemq.artemis.utils.ActiveMQThreadFactory$1.run(ActiveMQThreadFactory.java:120)
      org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd

      {refCount=1, channel=managed-scaledown51a-ss-1-24689 (activemq_broadcast_channel), channelName='activemq_broadcast_channel', connected=false}

      ::RefCount++ = 1 on channel activemq_broadcast_channel
      AMQ221029: Stopped bridge $.artemis.internal.sf.my-cluster.c9cfdf48-9f87-11f0-bd48-e24e2a7d3973
      org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd

      {refCount=1, channel=org.jgroups.JChannel@574a89e2, channelName='activemq_broadcast_channel', connected=false}

      :: Connecting activemq_broadcast_channel
      java.lang.Exception: Trace
      at org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.connect(JChannelWrapper.java:115)
      at org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.internalOpen(JGroupsBroadcastEndpoint.java:124)
      at org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.openClient(JGroupsBroadcastEndpoint.java:102)
      at org.apache.activemq.artemis.core.cluster.DiscoveryGroup.start(DiscoveryGroup.java:110)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.startDiscovery(ServerLocatorImpl.java:300)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.initialize(ServerLocatorImpl.java:284)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.createSessionFactory(ServerLocatorImpl.java:677)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:570)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:549)
      at org.apache.activemq.artemis.core.server.NodeLocator.connectToCluster(NodeLocator.java:85)
      at org.apache.activemq.artemis.core.server.impl.PrimaryOnlyActivation.connectToScaleDownTarget(PrimaryOnlyActivation.java:148)
      at org.apache.activemq.artemis.core.server.impl.PrimaryOnlyActivation.freezeConnections(PrimaryOnlyActivation.java:116)
      at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.freezeConnections(ActiveMQServerImpl.java:1595)
      at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:1334)
      at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:1251)
      at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.fail(ActiveMQServerImpl.java:1230)
      at org.apache.activemq.artemis.core.management.impl.ActiveMQServerControlImpl.scaleDown(ActiveMQServerControlImpl.java:4209)
      at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      ...
      at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:678)
      at org.eclipse.jetty.server.internal.HttpConnection.onFillable(HttpConnection.java:416)
      at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
      at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1211)
      at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
      at java.base/java.lang.Thread.run(Thread.java:840)
      AMQ222181: Unable to scaleDown messages
      org.apache.activemq.artemis.api.core.ActiveMQInternalErrorException: AMQ219004: Failed to initialise session factory
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.initialize(ServerLocatorImpl.java:288)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.createSessionFactory(ServerLocatorImpl.java:677)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:570)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:549)
      at org.apache.activemq.artemis.core.server.NodeLocator.connectToCluster(NodeLocator.java:85)
      at org.apache.activemq.artemis.core.server.impl.PrimaryOnlyActivation.connectToScaleDownTarget(PrimaryOnlyActivation.java:148)
      at org.apache.activemq.artemis.core.server.impl.PrimaryOnlyActivation.freezeConnections(PrimaryOnlyActivation.java:116)
      at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.freezeConnections(ActiveMQServerImpl.java:1595)
      at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:1334)
      at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:1251)
      at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.fail(ActiveMQServerImpl.java:1230)
      at org.apache.activemq.artemis.core.management.impl.ActiveMQServerControlImpl.scaleDown(ActiveMQServerControlImpl.java:4209)
      ...
      at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
      at java.base/java.lang.Thread.run(Thread.java:840)
      Caused by: org.apache.activemq.artemis.api.core.ActiveMQInternalErrorException: channel is closed
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.startDiscovery(ServerLocatorImpl.java:302)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.initialize(ServerLocatorImpl.java:284)
      ... 135 more
      Caused by: java.lang.IllegalStateException: channel is closed
      at org.jgroups.JChannel.checkClosed(JChannel.java:998)
      at org.jgroups.JChannel._preConnect(JChannel.java:809)
      at org.jgroups.JChannel.connect(JChannel.java:328)
      at org.jgroups.JChannel.connect(JChannel.java:321)
      at org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.connect(JChannelWrapper.java:126)
      at org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.internalOpen(JGroupsBroadcastEndpoint.java:124)
      at org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.openClient(JGroupsBroadcastEndpoint.java:102)
      at org.apache.activemq.artemis.core.cluster.DiscoveryGroup.start(DiscoveryGroup.java:110)
      at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.startDiscovery(ServerLocatorImpl.java:300)
      ... 136 more
      }}

              gtully@redhat.com Gary Tully
              rh-messaging-ci Messaging CI
              Votes:
              0 Vote for this issue
              Watchers:
              1 Start watching this issue

                Created:
                Updated: