2019/01/16 19:56:16,791+05:30 WARN [org.apache.activemq.artemis.service.extensions.xa.recovery] (Thread-5 (ActiveMQ-client-global-threads-15284004)) AMQ122014: Notified of connection failure in xa recovery connectionFactory for provider org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl@6c2daeb2 will attempt reconnect on next pass: ActiveMQConnectionTimedOutException[errorType=CONNECTION_TIMEDOUT message=AMQ119011: Did not receive data from server for org.apache.activemq.artemis.core.remoting.impl.netty.NettyConnection@4a529cc6[local= /135.250.138.197:37738, remote=/135.250.138.172:12080]] at org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl$PingRunnable.run(ClientSessionFactoryImpl.java:1197) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl$ActualScheduledPinger.run(ClientSessionFactoryImpl.java:1169) [artemis-core-client-1.1.0.wildfly-017.jar:] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [rt.jar:1.8.0_161] at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [rt.jar:1.8.0_161] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [rt.jar:1.8.0_161] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [rt.jar:1.8.0_161] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [rt.jar:1.8.0_161] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [rt.jar:1.8.0_161] at java.lang.Thread.run(Thread.java:748) [rt.jar:1.8.0_161] 2019/01/16 19:56:21,115+05:30 WARN [com.arjuna.ats.jta] (Thread-22 (ActiveMQ-client-global-threads-15284004)) ARJUNA016086: TransactionImple.enlistResource setTransactionTimeout on XAResource < formatId=131077, gtrid_length=29, bqual_length=36, tx_uid=0:ffff87fa8ac5:-1666fc8:5c3a30c3:a212610, node_name=1, branch_uid=0:ffff87fa8ac5:-1666fc8:5c3a30c3:a212611, subordinatenodename=null, eis_name=java:/HAConsumerConnectionFactory NodeId:75f7fe28-0fd8-11e9-a447-b3f2c1b520a3 > threw: XAException.XAER_RMFAIL: javax.transaction.xa.XAException at org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.setTransactionTimeout(ClientSessionImpl.java:1183) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.service.extensions.xa.ActiveMQXAResourceWrapperImpl.setTransactionTimeout(ActiveMQXAResourceWrapperImpl.java:116) at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.enlistResource(TransactionImple.java:637) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.enlistResource(TransactionImple.java:423) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at org.jboss.as.ejb3.inflow.MessageEndpointInvocationHandler.beforeDelivery(MessageEndpointInvocationHandler.java:109) [wildfly-ejb3-10.1.0.Final.jar:10.1.0.Final] at sun.reflect.GeneratedMethodAccessor383.invoke(Unknown Source) [:1.8.0_161] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_161] at java.lang.reflect.Method.invoke(Method.java:498) [rt.jar:1.8.0_161] at org.jboss.as.ejb3.inflow.AbstractInvocationHandler.handle(AbstractInvocationHandler.java:60) [wildfly-ejb3-10.1.0.Final.jar:10.1.0.Final] at org.jboss.as.ejb3.inflow.MessageEndpointInvocationHandler.doInvoke(MessageEndpointInvocationHandler.java:135) [wildfly-ejb3-10.1.0.Final.jar:10.1.0.Final] at org.jboss.as.ejb3.inflow.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:73) [wildfly-ejb3-10.1.0.Final.jar:10.1.0.Final] at com.nokia.aor.pm.common.jms.in.PmEventReceptor$$$endpoint4.beforeDelivery(Unknown Source) [aor-platform-persistancemanager-mba-18.05.jar:] at org.apache.activemq.artemis.ra.inflow.ActiveMQMessageHandler.onMessage(ActiveMQMessageHandler.java:300) at org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl.callOnMessage(ClientConsumerImpl.java:1018) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl.access$400(ClientConsumerImpl.java:48) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl$Runner.run(ClientConsumerImpl.java:1145) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.utils.OrderedExecutorFactory$OrderedExecutor$ExecutorTask.run(OrderedExecutorFactory.java:103) [artemis-core-client-1.1.0.wildfly-017.jar:] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [rt.jar:1.8.0_161] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [rt.jar:1.8.0_161] at java.lang.Thread.run(Thread.java:748) [rt.jar:1.8.0_161] Caused by: ActiveMQUnBlockedException[errorType=UNBLOCKED message=AMQ119016: Connection failure detected. Unblocking a blocking call that will never get a response] at org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:406) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.sendBlocking(ChannelImpl.java:304) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.protocol.core.impl.ActiveMQSessionContext.configureTransactionTimeout(ActiveMQSessionContext.java:512) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.setTransactionTimeout(ClientSessionImpl.java:1178) [artemis-core-client-1.1.0.wildfly-017.jar:] ... 19 more Caused by: ActiveMQNotConnectedException[errorType=NOT_CONNECTED message=AMQ119006: Channel disconnected] at org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl.connectionDestroyed(ClientSessionFactoryImpl.java:341) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.remoting.impl.netty.NettyConnector$Listener$1.run(NettyConnector.java:915) [artemis-core-client-1.1.0.wildfly-017.jar:] ... 4 more 2019/01/16 19:56:21,126+05:30 WARN [com.arjuna.ats.jta] (Thread-22 (ActiveMQ-client-global-threads-15284004)) ARJUNA016041: prepare on < formatId=131077, gtrid_length=29, bqual_length=36, tx_uid=0:ffff87fa8ac5:-1666fc8:5c3a30c3:a212610, node_name=1, branch_uid=0:ffff87fa8ac5:-1666fc8:5c3a30c3:a212611, subordinatenodename=null, eis_name=java:/HAConsumerConnectionFactory NodeId:75f7fe28-0fd8-11e9-a447-b3f2c1b520a3 > (org.jboss.activemq.artemis.wildfly.integration.WildFlyActiveMQXAResourceWrapper@329bcc1) failed with exception XAException.XAER_RMFAIL: javax.transaction.xa.XAException at org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.end(ClientSessionImpl.java:1110) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.service.extensions.xa.ActiveMQXAResourceWrapperImpl.end(ActiveMQXAResourceWrapperImpl.java:81) [artemis-service-extensions-1.1.0.wildfly-017.jar:] at com.arjuna.ats.internal.jta.resources.arjunacore.XAResourceRecord.topLevelPrepare(XAResourceRecord.java:213) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.arjuna.coordinator.BasicAction.doPrepare(BasicAction.java:2662) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.arjuna.coordinator.BasicAction.doPrepare(BasicAction.java:2612) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.arjuna.coordinator.BasicAction.prepare(BasicAction.java:2155) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.arjuna.coordinator.BasicAction.End(BasicAction.java:1501) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:96) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:162) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1200) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.jbossatx.BaseTransactionManagerDelegate.commit(BaseTransactionManagerDelegate.java:89) at org.jboss.as.ejb3.inflow.MessageEndpointInvocationHandler.afterDelivery(MessageEndpointInvocationHandler.java:71) at sun.reflect.GeneratedMethodAccessor404.invoke(Unknown Source) [:1.8.0_161] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_161] at java.lang.reflect.Method.invoke(Method.java:498) [rt.jar:1.8.0_161] at org.jboss.as.ejb3.inflow.AbstractInvocationHandler.handle(AbstractInvocationHandler.java:60) at org.jboss.as.ejb3.inflow.MessageEndpointInvocationHandler.doInvoke(MessageEndpointInvocationHandler.java:135) at org.jboss.as.ejb3.inflow.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:73) at com.nokia.aor.pm.common.jms.in.PmEventReceptor$$$endpoint4.afterDelivery(Unknown Source) at org.apache.activemq.artemis.ra.inflow.ActiveMQMessageHandler.onMessage(ActiveMQMessageHandler.java:321) [artemis-ra-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl.callOnMessage(ClientConsumerImpl.java:1018) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl.access$400(ClientConsumerImpl.java:48) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl$Runner.run(ClientConsumerImpl.java:1145) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.utils.OrderedExecutorFactory$OrderedExecutor$ExecutorTask.run(OrderedExecutorFactory.java:103) [artemis-core-client-1.1.0.wildfly-017.jar:] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [rt.jar:1.8.0_161] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [rt.jar:1.8.0_161] at java.lang.Thread.run(Thread.java:748) [rt.jar:1.8.0_161] 2019/01/16 19:56:21,131+05:30 WARN [com.arjuna.ats.arjuna] (Thread-22 (ActiveMQ-client-global-threads-15284004)) ARJUNA012073: BasicAction.End() - prepare phase of action-id 0:ffff87fa8ac5:-1666fc8:5c3a30c3:a212610 failed. 2019/01/16 19:56:21,131+05:30 WARN [com.arjuna.ats.arjuna] (Thread-22 (ActiveMQ-client-global-threads-15284004)) ARJUNA012075: Action Aborting 2019/01/16 19:56:21,135+05:30 WARN [com.arjuna.ats.jta] (Thread-22 (ActiveMQ-client-global-threads-15284004)) ARJUNA016045: attempted rollback of < formatId=131077, gtrid_length=29, bqual_length=36, tx_uid=0:ffff87fa8ac5:-1666fc8:5c3a30c3:a212610, node_name=1, branch_uid=0:ffff87fa8ac5:-1666fc8:5c3a30c3:a212611, subordinatenodename=null, eis_name=java:/HAConsumerConnectionFactory NodeId:75f7fe28-0fd8-11e9-a447-b3f2c1b520a3 > (org.jboss.activemq.artemis.wildfly.integration.WildFlyActiveMQXAResourceWrapper@329bcc1) failed with exception code XAException.XAER_PROTO: javax.transaction.xa.XAException at org.apache.activemq.artemis.core.protocol.core.impl.ActiveMQSessionContext.xaEnd(ActiveMQSessionContext.java:349) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientSessionImpl.end(ClientSessionImpl.java:1118) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.service.extensions.xa.ActiveMQXAResourceWrapperImpl.end(ActiveMQXAResourceWrapperImpl.java:81) [artemis-service-extensions-1.1.0.wildfly-017.jar:] at com.arjuna.ats.internal.jta.resources.arjunacore.XAResourceRecord.topLevelAbort(XAResourceRecord.java:319) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.arjuna.coordinator.BasicAction.doAbort(BasicAction.java:3017) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.arjuna.coordinator.BasicAction.doAbort(BasicAction.java:2996) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.arjuna.coordinator.BasicAction.phase2Abort(BasicAction.java:1979) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.arjuna.coordinator.BasicAction.End(BasicAction.java:1517) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:96) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:162) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1200) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.jbossatx.BaseTransactionManagerDelegate.commit(BaseTransactionManagerDelegate.java:89) at org.jboss.as.ejb3.inflow.MessageEndpointInvocationHandler.afterDelivery(MessageEndpointInvocationHandler.java:71) at sun.reflect.GeneratedMethodAccessor404.invoke(Unknown Source) [:1.8.0_161] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_161] at java.lang.reflect.Method.invoke(Method.java:498) [rt.jar:1.8.0_161] at org.jboss.as.ejb3.inflow.AbstractInvocationHandler.handle(AbstractInvocationHandler.java:60) at org.jboss.as.ejb3.inflow.MessageEndpointInvocationHandler.doInvoke(MessageEndpointInvocationHandler.java:135) at org.jboss.as.ejb3.inflow.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:73) at com.nokia.aor.pm.common.jms.in.PmEventReceptor$$$endpoint4.afterDelivery(Unknown Source) at org.apache.activemq.artemis.ra.inflow.ActiveMQMessageHandler.onMessage(ActiveMQMessageHandler.java:321) [artemis-ra-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl.callOnMessage(ClientConsumerImpl.java:1018) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl.access$400(ClientConsumerImpl.java:48) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl$Runner.run(ClientConsumerImpl.java:1145) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.utils.OrderedExecutorFactory$OrderedExecutor$ExecutorTask.run(OrderedExecutorFactory.java:103) [artemis-core-client-1.1.0.wildfly-017.jar:] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [rt.jar:1.8.0_161] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [rt.jar:1.8.0_161] at java.lang.Thread.run(Thread.java:748) [rt.jar:1.8.0_161] 2019/01/16 19:56:21,139+05:30 WARN [com.arjuna.ats.arjuna] (Thread-22 (ActiveMQ-client-global-threads-15284004)) ARJUNA012091: Top-level abort of action 0:ffff87fa8ac5:-1666fc8:5c3a30c3:a212610 received TwoPhaseOutcome.FINISH_ERROR from com.arjuna.ats.internal.jta.resources.arjunacore.XAResourceRecord 2019/01/16 19:56:21,142+05:30 WARN [org.apache.activemq.artemis.ra] (Thread-22 (ActiveMQ-client-global-threads-15284004)) AMQ152006: Unable to call after delivery: javax.resource.spi.LocalTransactionException: javax.transaction.RollbackException: ARJUNA016053: Could not commit transaction. at org.jboss.as.ejb3.inflow.MessageEndpointInvocationHandler.afterDelivery(MessageEndpointInvocationHandler.java:87) at sun.reflect.GeneratedMethodAccessor404.invoke(Unknown Source) [:1.8.0_161] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_161] at java.lang.reflect.Method.invoke(Method.java:498) [rt.jar:1.8.0_161] at org.jboss.as.ejb3.inflow.AbstractInvocationHandler.handle(AbstractInvocationHandler.java:60) at org.jboss.as.ejb3.inflow.MessageEndpointInvocationHandler.doInvoke(MessageEndpointInvocationHandler.java:135) at org.jboss.as.ejb3.inflow.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:73) at com.nokia.aor.pm.common.jms.in.PmEventReceptor$$$endpoint4.afterDelivery(Unknown Source) at org.apache.activemq.artemis.ra.inflow.ActiveMQMessageHandler.onMessage(ActiveMQMessageHandler.java:321) [artemis-ra-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl.callOnMessage(ClientConsumerImpl.java:1018) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl.access$400(ClientConsumerImpl.java:48) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.core.client.impl.ClientConsumerImpl$Runner.run(ClientConsumerImpl.java:1145) [artemis-core-client-1.1.0.wildfly-017.jar:] at org.apache.activemq.artemis.utils.OrderedExecutorFactory$OrderedExecutor$ExecutorTask.run(OrderedExecutorFactory.java:103) [artemis-core-client-1.1.0.wildfly-017.jar:] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [rt.jar:1.8.0_161] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [rt.jar:1.8.0_161] at java.lang.Thread.run(Thread.java:748) [rt.jar:1.8.0_161] Caused by: javax.transaction.RollbackException: ARJUNA016053: Could not commit transaction. at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1212) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126) [narayana-jts-idlj-5.3.3.Final.jar:5.3.3.Final (revision: c5912)] at com.arjuna.ats.jbossatx.BaseTransactionManagerDelegate.commit(BaseTransactionManagerDelegate.java:89) at org.jboss.as.ejb3.inflow.MessageEndpointInvocationHandler.afterDelivery(MessageEndpointInvocationHandler.java:71) ... 15 more 2019/01/16 19:56:21,144+05:30 WARN [org.apache.activemq.artemis.core.client] (Thread-22 (ActiveMQ-client-global-threads-15284004)) AMQ212009: resetting session after failure