[GSS](7.1.z) ISPN-8893 - Stack trace of a primary request of Infinispan is lost, damaging supportability severely

Configure a domain with two hosts: cp -a $EAP_HOME/domain $EAP_HOME/host1 cp -a $EAP_HOME/domain $EAP_HOME/host2 Download the host-slave files: download host1-slave.xml in $EAP_HOME/host1/configuration/ download host2-slave.xml in $EAP_HOME/host2/configuration/ Start the EAP instances: $EAP_HOME/bin/domain.sh --host-config=host-master.xml $EAP_HOME/bin/domain.sh -Djboss.domain.base.dir=$EAP_HOME/host1 --host-config=host1-slave.xml -Djboss.domain.master.address=127.0.0.1 $EAP_HOME/bin/domain.sh -Djboss.domain.base.dir=$EAP_HOME/host2 --host-config=host2-slave.xml -Djboss.domain.master.address=127.0.0.1 Configure the infinispan cache used by the reproducer /profile=full-ha/subsystem=infinispan/cache-container=test_cache_container:add(jndi-name=infinispan/test_cache_container) /profile=full-ha/subsystem=infinispan/cache-container=test_cache_container/transport=TRANSPORT/:add(lock-timeout=60000) /profile=full-ha/subsystem=infinispan/cache-container=test_cache_container/replicated-cache=test_cache:add(jndi-name=infinispan/test_cache_container/test_cache, mode=SYNC) Deploy the reproducer application to other-server-group server group Navigate to http://localhost:8180/infinispan-distributed (host1) or http://localhost:8280/infinispan-distributed (host2) and click on "put value in the cache", that action will return an error which only contains the error trace retrieved from the server which is deserializing the invalid object. The stack trace of the asking thread is lost. Sample error trace: javax.ejb.EJBException: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from host2:server-two, see cause for remote stack trace at org.jboss.as.ejb3.tx.CMTTxInterceptor.handleExceptionInOurTx(CMTTxInterceptor.java:188) at org.jboss.as.ejb3.tx.CMTTxInterceptor.invokeInOurTx(CMTTxInterceptor.java:277) at org.jboss.as.ejb3.tx.CMTTxInterceptor.required(CMTTxInterceptor.java:332) at org.jboss.as.ejb3.tx.CMTTxInterceptor.processInvocation(CMTTxInterceptor.java:240) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422) at org.jboss.invocation.InterceptorContext$Invocation.proceed(InterceptorContext.java:509) at org.jboss.weld.ejb.AbstractEJBRequestScopeActivationInterceptor.aroundInvoke(AbstractEJBRequestScopeActivationInterceptor.java:64) at org.jboss.as.weld.ejb.EjbRequestScopeActivationInterceptor.processInvocation(EjbRequestScopeActivationInterceptor.java:89) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422) at org.jboss.as.ejb3.component.interceptors.CurrentInvocationContextInterceptor.processInvocation(CurrentInvocationContextInterceptor.java:41) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422) at org.jboss.as.ejb3.component.invocationmetrics.WaitTimeInterceptor.processInvocation(WaitTimeInterceptor.java:47) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422) at org.jboss.as.ejb3.security.SecurityContextInterceptor.processInvocation(SecurityContextInterceptor.java:100) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422) at org.jboss.as.ejb3.deployment.processors.StartupAwaitInterceptor.processInvocation(StartupAwaitInterceptor.java:22) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422) at org.jboss.as.ejb3.component.interceptors.ShutDownInterceptorFactory$1.processInvocation(ShutDownInterceptorFactory.java:64) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422) at org.jboss.as.ejb3.component.interceptors.LoggingInterceptor.processInvocation(LoggingInterceptor.java:67) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422) at org.jboss.as.ee.component.NamespaceContextInterceptor.processInvocation(NamespaceContextInterceptor.java:50) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422) at org.jboss.as.ejb3.component.interceptors.AdditionalSetupInterceptor.processInvocation(AdditionalSetupInterceptor.java:54) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422) at org.jboss.invocation.ContextClassLoaderInterceptor.processInvocation(ContextClassLoaderInterceptor.java:60) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422) at org.jboss.invocation.InterceptorContext.run(InterceptorContext.java:438) at org.wildfly.security.manager.WildFlySecurityManager.doChecked(WildFlySecurityManager.java:609) at org.jboss.invocation.AccessCheckingInterceptor.processInvocation(AccessCheckingInterceptor.java:57) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422) at org.jboss.invocation.ChainedInterceptor.processInvocation(ChainedInterceptor.java:53) at org.jboss.as.ee.component.ViewService$View.invoke(ViewService.java:198) at org.jboss.as.ee.component.ViewDescription$1.processInvocation(ViewDescription.java:185) at org.jboss.as.ee.component.ProxyInvocationHandler.invoke(ProxyInvocationHandler.java:81) at local.test.Backend$$$view1.actionPutInCache(Unknown Source) at local.test.Front.actionPutInCache(Front.java:23) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at javax.el.ELUtil.invokeMethod(ELUtil.java:311) at javax.el.BeanELResolver.invoke(BeanELResolver.java:415) at javax.el.CompositeELResolver.invoke(CompositeELResolver.java:256) at com.sun.el.parser.AstValue.invoke(AstValue.java:285) at com.sun.el.MethodExpressionImpl.invoke(MethodExpressionImpl.java:304) at org.jboss.weld.util.el.ForwardingMethodExpression.invoke(ForwardingMethodExpression.java:40) at org.jboss.weld.el.WeldMethodExpression.invoke(WeldMethodExpression.java:50) at org.jboss.weld.util.el.ForwardingMethodExpression.invoke(ForwardingMethodExpression.java:40) at org.jboss.weld.el.WeldMethodExpression.invoke(WeldMethodExpression.java:50) at com.sun.faces.facelets.el.TagMethodExpression.invoke(TagMethodExpression.java:105) at javax.faces.event.MethodExpressionActionListener.processAction(MethodExpressionActionListener.java:147) at javax.faces.event.ActionEvent.processListener(ActionEvent.java:88) at javax.faces.component.UIComponentBase.broadcast(UIComponentBase.java:814) at javax.faces.component.UICommand.broadcast(UICommand.java:300) at javax.faces.component.UIViewRoot.broadcastEvents(UIViewRoot.java:790) at javax.faces.component.UIViewRoot.processApplication(UIViewRoot.java:1282) at com.sun.faces.lifecycle.InvokeApplicationPhase.execute(InvokeApplicationPhase.java:81) at com.sun.faces.lifecycle.Phase.doPhase(Phase.java:101) at com.sun.faces.lifecycle.LifecycleImpl.execute(LifecycleImpl.java:198) at javax.faces.webapp.FacesServlet.service(FacesServlet.java:658) at io.undertow.servlet.handlers.ServletHandler.handleRequest(ServletHandler.java:74) at io.undertow.servlet.handlers.security.ServletSecurityRoleHandler.handleRequest(ServletSecurityRoleHandler.java:62) at io.undertow.servlet.handlers.ServletChain$1.handleRequest(ServletChain.java:64) at io.undertow.servlet.handlers.ServletDispatchingHandler.handleRequest(ServletDispatchingHandler.java:36) at org.wildfly.extension.undertow.security.SecurityContextAssociationHandler.handleRequest(SecurityContextAssociationHandler.java:78) at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43) at io.undertow.servlet.handlers.security.SSLInformationAssociationHandler.handleRequest(SSLInformationAssociationHandler.java:131) at io.undertow.servlet.handlers.security.ServletAuthenticationCallHandler.handleRequest(ServletAuthenticationCallHandler.java:57) at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43) at io.undertow.security.handlers.AbstractConfidentialityHandler.handleRequest(AbstractConfidentialityHandler.java:46) at io.undertow.servlet.handlers.security.ServletConfidentialityConstraintHandler.handleRequest(ServletConfidentialityConstraintHandler.java:64) at io.undertow.security.handlers.AuthenticationMechanismsHandler.handleRequest(AuthenticationMechanismsHandler.java:60) at io.undertow.servlet.handlers.security.CachedAuthenticatedSessionHandler.handleRequest(CachedAuthenticatedSessionHandler.java:77) at io.undertow.security.handlers.NotificationReceiverHandler.handleRequest(NotificationReceiverHandler.java:50) at io.undertow.security.handlers.AbstractSecurityContextAssociationHandler.handleRequest(AbstractSecurityContextAssociationHandler.java:43) at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43) at org.wildfly.extension.undertow.security.jacc.JACCContextIdHandler.handleRequest(JACCContextIdHandler.java:61) at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43) at org.wildfly.extension.undertow.deployment.GlobalRequestControllerHandler.handleRequest(GlobalRequestControllerHandler.java:68) at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43) at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:292) at io.undertow.servlet.handlers.ServletInitialHandler.access$100(ServletInitialHandler.java:81) at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:138) at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:135) at io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:48) at io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43) at org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction.lambda$create$0(SecurityContextThreadSetupAction.java:105) at org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) at org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) at org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) at org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:272) at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:81) at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:104) at io.undertow.server.Connectors.executeRootHandler(Connectors.java:330) at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:812) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from host2:server-two, see cause for remote stack trace at org.infinispan.remoting.transport.AbstractTransport.checkResponse(AbstractTransport.java:44) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:821) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$1(JGroupsTransport.java:648) at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602) at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) at org.infinispan.remoting.transport.jgroups.RspListFuture.futureDone(RspListFuture.java:40) at org.jgroups.blocks.Request.checkCompletion(Request.java:152) at org.jgroups.blocks.GroupRequest.receiveResponse(GroupRequest.java:115) at org.jgroups.blocks.RequestCorrelator.dispatch(RequestCorrelator.java:427) at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:357) at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:245) at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:664) at org.jgroups.JChannel.up(JChannel.java:738) at org.jgroups.fork.ForkProtocolStack.up(ForkProtocolStack.java:120) at org.jgroups.stack.Protocol.up(Protocol.java:380) at org.jgroups.protocols.FORK.up(FORK.java:114) at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) at org.jgroups.protocols.FlowControl.up(FlowControl.java:390) at org.jgroups.protocols.FlowControl.up(FlowControl.java:390) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1037) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1077) at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:792) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:433) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:649) at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:155) at org.jgroups.protocols.FD_ALL.up(FD_ALL.java:200) at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:325) at org.jgroups.protocols.MERGE3.up(MERGE3.java:292) at org.jgroups.protocols.Discovery.up(Discovery.java:296) at org.jgroups.protocols.TP.passMessageUp(TP.java:1657) at org.jgroups.protocols.TP$SingleMessageHandler.run(TP.java:1872) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at org.jboss.as.clustering.jgroups.ClassLoaderThreadFactory.lambda$newThread$0(ClassLoaderThreadFactory.java:52) ... 1 more Caused by: org.infinispan.commons.CacheException: Problems invoking command. at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:183) at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:455) at org.jgroups.blocks.RequestCorrelator.dispatch(RequestCorrelator.java:406) at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:357) at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:245) at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:664) at org.jgroups.JChannel.up(JChannel.java:738) at org.jgroups.fork.ForkProtocolStack.up(ForkProtocolStack.java:120) at org.jgroups.stack.Protocol.up(Protocol.java:380) at org.jgroups.protocols.FORK.up(FORK.java:114) at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) at org.jgroups.protocols.FlowControl.up(FlowControl.java:374) at org.jgroups.protocols.FlowControl.up(FlowControl.java:390) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1037) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:442) at org.jgroups.protocols.pbcast.NAKACK2.deliver(NAKACK2.java:968) at org.jgroups.protocols.pbcast.NAKACK2.handleMessage(NAKACK2.java:848) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:611) ... 11 more Caused by: java.io.EOFException at local.test.BrokenDeserializationPojo.readExternal(BrokenDeserializationPojo.java:18) at org.jboss.marshalling.river.RiverUnmarshaller.doReadNewObject(RiverUnmarshaller.java:1420) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:275) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:208) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.commands.write.PutKeyValueCommand.readFrom(PutKeyValueCommand.java:131) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:113) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:89) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:41) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:479) at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:236) at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:149) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:353) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:208) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.commands.remote.SingleRpcCommand.readFrom(SingleRpcCommand.java:46) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:113) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:173) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:68) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:479) at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:236) at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:149) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:353) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:208) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.commons.marshall.jboss.AbstractJBossMarshaller.objectFromObjectStream(AbstractJBossMarshaller.java:134) at org.infinispan.marshall.core.VersionAwareMarshaller.objectFromByteBuffer(VersionAwareMarshaller.java:101) at org.infinispan.commons.marshall.AbstractDelegatingMarshaller.objectFromByteBuffer(AbstractDelegatingMarshaller.java:80) at org.infinispan.remoting.transport.jgroups.MarshallerAdapter.objectFromBuffer(MarshallerAdapter.java:28) at org.jboss.as.clustering.infinispan.ChannelFactoryTransport$1.objectFromBuffer(ChannelFactoryTransport.java:81) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:163) ... 29 more Caused by: an exception which occurred: in object of type local.test.BrokenDeserializationPojo