Server to server transaction ejb over http calls fails with `Authentication failed` error

2019-06-26 11:35:13,751 WARN  [com.arjuna.ats.jta] (pool-9-thread-1) ARJUNA016039: onePhaseCommit on < formatId=131077, gtrid_length=29, bqual_length=36, tx_uid=0:ffff0a2804c5:5db496d0:5d133c49:e, node_name=1, branch_uid=0:ffff0a2804c5:5db496d0:5d133c49:14, subordinatenodename=null, eis_name=unknown eis name > (Subordinate XAResource at http://127.0.0.1:8180/wildfly-services) failed with exception XAException.XAER_RMERR: javax.transaction.xa.XAException
 at org.wildfly.httpclient.transaction.HttpSubordinateTransactionHandle.processOperation(HttpSubordinateTransactionHandle.java:145)
 at org.wildfly.httpclient.transaction.HttpSubordinateTransactionHandle.processOperation(HttpSubordinateTransactionHandle.java:102)
 at org.wildfly.httpclient.transaction.HttpSubordinateTransactionHandle.commit(HttpSubordinateTransactionHandle.java:69)
 at org.wildfly.transaction.client.SubordinateXAResource.commit(SubordinateXAResource.java:160)
 at com.arjuna.ats.internal.jta.resources.arjunacore.XAResourceRecord.topLevelOnePhaseCommit(XAResourceRecord.java:702)
 at com.arjuna.ats.arjuna.coordinator.BasicAction.onePhaseCommit(BasicAction.java:2386)
 at com.arjuna.ats.arjuna.coordinator.BasicAction.End(BasicAction.java:1497)
 at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:96)
 at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:162)
 at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1288)
 at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126)
 at com.arjuna.ats.jbossatx.BaseTransactionManagerDelegate.commit(BaseTransactionManagerDelegate.java:89)
 at org.wildfly.transaction.client.LocalTransaction.commitAndDissociate(LocalTransaction.java:75)
 at org.wildfly.transaction.client.ContextTransactionManager.commit(ContextTransactionManager.java:71)
 at org.jboss.as.ejb3.tx.CMTTxInterceptor.endTransaction(CMTTxInterceptor.java:88)
 at org.jboss.as.ejb3.tx.CMTTxInterceptor.invokeInOurTx(CMTTxInterceptor.java:261)
 at org.jboss.as.ejb3.tx.CMTTxInterceptor.required(CMTTxInterceptor.java:362)
 at org.jboss.as.ejb3.tx.CMTTxInterceptor.processInvocation(CMTTxInterceptor.java:144)
 at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
 at org.jboss.invocation.InterceptorContext$Invocation.proceed(InterceptorContext.java:509)
 at org.jboss.weld.module.ejb.AbstractEJBRequestScopeActivationInterceptor.aroundInvoke(AbstractEJBRequestScopeActivationInterceptor.java:72)
 at org.jboss.as.weld.ejb.EjbRequestScopeActivationInterceptor.processInvocation(EjbRequestScopeActivationInterceptor.java:89)
 at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
 at org.jboss.as.ejb3.component.interceptors.CurrentInvocationContextInterceptor.processInvocation(CurrentInvocationContextInterceptor.java:41)
 at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
 at org.jboss.as.ejb3.component.invocationmetrics.WaitTimeInterceptor.processInvocation(WaitTimeInterceptor.java:47)
 at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
 at org.jboss.as.ejb3.security.SecurityContextInterceptor.processInvocation(SecurityContextInterceptor.java:100)
 at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
 at org.jboss.as.ejb3.deployment.processors.StartupAwaitInterceptor.processInvocation(StartupAwaitInterceptor.java:22)
 at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
 at org.jboss.as.ejb3.component.interceptors.ShutDownInterceptorFactory$1.processInvocation(ShutDownInterceptorFactory.java:64)
 at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
 at org.jboss.as.ejb3.component.interceptors.LoggingInterceptor.processInvocation(LoggingInterceptor.java:67)
 at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
 at org.jboss.as.ee.component.NamespaceContextInterceptor.processInvocation(NamespaceContextInterceptor.java:50)
 at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
 at org.jboss.invocation.ContextClassLoaderInterceptor.processInvocation(ContextClassLoaderInterceptor.java:60)
 at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
 at org.jboss.invocation.InterceptorContext.run(InterceptorContext.java:438)
 at org.wildfly.security.manager.WildFlySecurityManager.doChecked(WildFlySecurityManager.java:618)
 at org.jboss.invocation.AccessCheckingInterceptor.processInvocation(AccessCheckingInterceptor.java:57)
 at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
 at org.jboss.invocation.ChainedInterceptor.processInvocation(ChainedInterceptor.java:53)
 at org.jboss.as.ee.component.ViewService$View.invoke(ViewService.java:198)
 at org.jboss.as.ee.component.ViewDescription$1.processInvocation(ViewDescription.java:185)
 at org.jboss.as.ee.component.ProxyInvocationHandler.invoke(ProxyInvocationHandler.java:81)
 at org.jboss.as.test.multinode.transaction.nooutbound.ClientStatelessBean$$$view3.call(Unknown Source)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.jboss.weld.util.reflection.Reflections.invokeAndUnwrap(Reflections.java:410)
 at org.jboss.weld.module.ejb.EnterpriseBeanProxyMethodHandler.invoke(EnterpriseBeanProxyMethodHandler.java:134)
 at org.jboss.weld.bean.proxy.EnterpriseTargetBeanInstance.invoke(EnterpriseTargetBeanInstance.java:56)
 at org.jboss.weld.module.ejb.InjectionPointPropagatingEnterpriseTargetBeanInstance.invoke(InjectionPointPropagatingEnterpriseTargetBeanInstance.java:68)
 at org.jboss.weld.bean.proxy.ProxyMethodHandler.invoke(ProxyMethodHandler.java:106)
 at org.jboss.as.test.multinode.transaction.nooutbound.ClientStatelessBean$Proxy$_$$_Weld$EnterpriseProxy$.call(Unknown Source)
 at org.jboss.as.test.multinode.transaction.nooutbound.TransactionContextRemoteCallTestCase.httpMandatoryBean(TransactionContextRemoteCallTestCase.java:99)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
 at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
 at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
 at org.jboss.arquillian.junit.Arquillian$8$1.invokeMethod(Arquillian.java:325)
 at org.jboss.arquillian.junit.MethodInvoker$1.invoke(MethodInvoker.java:18)
 at org.jboss.arquillian.container.test.impl.execution.LocalTestExecuter.execute(LocalTestExecuter.java:57)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
 at org.jboss.arquillian.core.impl.EventContextImpl.invokeObservers(EventContextImpl.java:103)
 at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:90)
 at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:133)
 at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:105)
 at org.jboss.arquillian.core.impl.EventImpl.fire(EventImpl.java:62)
 at org.jboss.arquillian.container.test.impl.execution.ContainerTestExecuter.execute(ContainerTestExecuter.java:36)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
 at org.jboss.arquillian.core.impl.EventContextImpl.invokeObservers(EventContextImpl.java:103)
 at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:90)
 at org.jboss.arquillian.test.impl.TestContextHandler.createTestContext(TestContextHandler.java:116)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
 at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:95)
 at org.jboss.arquillian.test.impl.TestContextHandler.createClassContext(TestContextHandler.java:83)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
 at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:95)
 at org.jboss.arquillian.test.impl.TestContextHandler.createSuiteContext(TestContextHandler.java:69)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
 at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:95)
 at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:133)
 at org.jboss.arquillian.test.impl.EventTestRunnerAdaptor.test(EventTestRunnerAdaptor.java:139)
 at org.jboss.arquillian.junit.MethodInvoker.invoke(MethodInvoker.java:15)
 at org.jboss.arquillian.junit.Arquillian$8.evaluate(Arquillian.java:332)
 at org.jboss.arquillian.junit.Arquillian$4.evaluate(Arquillian.java:204)
 at org.jboss.arquillian.junit.Arquillian.multiExecute(Arquillian.java:350)
 at org.jboss.arquillian.junit.Arquillian.access$200(Arquillian.java:54)
 at org.jboss.arquillian.junit.Arquillian$5.evaluate(Arquillian.java:215)
 at org.jboss.arquillian.junit.Arquillian$7$1.invoke(Arquillian.java:279)
 at org.jboss.arquillian.container.test.impl.execution.BeforeLifecycleEventExecuter.on(BeforeLifecycleEventExecuter.java:34)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
 at org.jboss.arquillian.core.impl.EventContextImpl.invokeObservers(EventContextImpl.java:103)
 at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:90)
 at org.jboss.arquillian.test.impl.TestContextHandler.createTestContext(TestContextHandler.java:116)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
 at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:95)
 at org.jboss.arquillian.test.impl.TestContextHandler.createClassContext(TestContextHandler.java:83)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
 at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:95)
 at org.jboss.arquillian.test.impl.TestContextHandler.createSuiteContext(TestContextHandler.java:69)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:86)
 at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:95)
 at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:133)
 at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:105)
 at org.jboss.arquillian.test.impl.EventTestRunnerAdaptor.fireCustomLifecycle(EventTestRunnerAdaptor.java:159)
 at org.jboss.arquillian.junit.Arquillian$7.evaluate(Arquillian.java:273)
 at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
 at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
 at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
 at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
 at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
 at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
 at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
 at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
 at org.jboss.arquillian.junit.Arquillian$2.evaluate(Arquillian.java:166)
 at org.jboss.arquillian.junit.Arquillian.multiExecute(Arquillian.java:350)
 at org.jboss.arquillian.junit.Arquillian.access$200(Arquillian.java:54)
 at org.jboss.arquillian.junit.Arquillian$3.evaluate(Arquillian.java:177)
 at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
 at org.jboss.arquillian.junit.Arquillian.run(Arquillian.java:115)
 at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
 at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
 at org.jboss.arquillian.junit.container.JUnitTestRunner.execute(JUnitTestRunner.java:61)
 at org.jboss.arquillian.protocol.jmx.JMXTestRunner.doRunTestMethod(JMXTestRunner.java:153)
 at org.jboss.as.arquillian.service.ArquillianService$ExtendedJMXTestRunner.doRunTestMethod(ArquillianService.java:200)
 at org.jboss.arquillian.protocol.jmx.JMXTestRunner.runTestMethodInternal(JMXTestRunner.java:137)
 at org.jboss.arquillian.protocol.jmx.JMXTestRunner.runTestMethod(JMXTestRunner.java:119)
 at org.jboss.as.arquillian.service.ArquillianService$ExtendedJMXTestRunner.runTestMethod(ArquillianService.java:176)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:71)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:275)
 at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:112)
 at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:46)
 at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:237)
 at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:138)
 at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:252)
 at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:819)
 at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:801)
 at org.jboss.as.jmx.PluggableMBeanServerImpl$TcclMBeanServer.invoke(PluggableMBeanServerImpl.java:1475)
 at org.jboss.as.jmx.PluggableMBeanServerImpl.invoke(PluggableMBeanServerImpl.java:724)
 at org.jboss.as.jmx.BlockingNotificationMBeanServer.invoke(BlockingNotificationMBeanServer.java:168)
 at org.jboss.as.jmx.AuthorizingMBeanServer.invoke(AuthorizingMBeanServer.java:258)
 at org.jboss.remotingjmx.protocol.v2.ServerProxy$InvokeHandler.handle(ServerProxy.java:950)
 at org.jboss.remotingjmx.protocol.v2.ServerCommon$MessageReciever$1$1.run(ServerCommon.java:153)
 at org.jboss.as.jmx.ServerInterceptorFactory$Interceptor$1.run(ServerInterceptorFactory.java:71)
 at org.jboss.as.jmx.ServerInterceptorFactory$Interceptor$1.run(ServerInterceptorFactory.java:66)
 at org.wildfly.security.auth.server.SecurityIdentity.runAs(SecurityIdentity.java:313)
 at org.wildfly.security.auth.server.SecurityIdentity.runAs(SecurityIdentity.java:270)
 at org.jboss.as.controller.AccessAuditContext.doAs(AccessAuditContext.java:254)
 at org.jboss.as.controller.AccessAuditContext.doAs(AccessAuditContext.java:225)
 at org.jboss.as.jmx.ServerInterceptorFactory$Interceptor.handleEvent(ServerInterceptorFactory.java:66)
 at org.jboss.remotingjmx.protocol.v2.ServerCommon$MessageReciever$1.run(ServerCommon.java:149)
 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
 at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.SecurityException: WFHTTP000008: Authentication failed
 at org.wildfly.httpclient.common.HttpTargetContext$1$1.lambda$null$0(HttpTargetContext.java:187)
 at org.wildfly.httpclient.common.HttpConnectionPool.runPending(HttpConnectionPool.java:134)
 at org.wildfly.httpclient.common.HttpConnectionPool.getConnection(HttpConnectionPool.java:83)
 at org.wildfly.httpclient.common.HttpTargetContext$1$1.lambda$null$1(HttpTargetContext.java:182)
 at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:92)
 at org.xnio.ChannelListeners$DrainListener.handleEvent(ChannelListeners.java:1145)
 at org.xnio.ChannelListeners$DrainListener.handleEvent(ChannelListeners.java:1125)
 at org.wildfly.httpclient.common.HttpTargetContext$1$1.lambda$completed$4(HttpTargetContext.java:192)
 at org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35)
 at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1982)
 at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1486)
 at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1377)
 ... 1 more