Blocker We are see occasional broken pipe errors in the logs with broken pipe errors writing the response back to the client using undertow server. This happens sporadically and causing transactions to fail.
It seems to be similar to a REST issue on EAP -https://access.redhat.com/solutions/4850891
The below stacktrace is for a SOAP service but we also see it happening with REST apis created using cxf:rsServer with pax-web-undertow as well.
Here is the stacktrace:
2020-07-08 18:20:45,742 | WARN | XNIO-8 task-4 | o.a.c.p.PhaseInterceptorChain | 83 - org.apache.cxf.cxf-core - 3.2.7.fuse-760026-redhat-00001 | Interceptor for { [http://www.asdf.com/user/location/userprofile/1/|http://www.asdf.com/user/location/userprofile/1/] }ProfileLocator#{ [http://www.asdf.com/user/location/userprofile/1/|http://www.asdf.com/user/location/userprofile/1/] }locateProfile has thrown exception, unwinding now org.apache.cxf.interceptor.Fault: Could not send Message. at org.apache.cxf.interceptor.MessageSenderInterceptor$MessageSenderEndingInterceptor.handleMessage(MessageSenderInterceptor.java:67) ~[!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.interceptor.OutgoingChainInterceptor.handleMessage(OutgoingChainInterceptor.java:90) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.phase.PhaseInterceptorChain.resume(PhaseInterceptorChain.java:278) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:78) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:267) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.http_undertow.UndertowHTTPDestination.doService(UndertowHTTPDestination.java:200) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.http_undertow.CxfUndertowServlet.invoke(CxfUndertowServlet.java:52) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.servlet.AbstractHTTPServlet.handleRequest(AbstractHTTPServlet.java:301) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.servlet.AbstractHTTPServlet.doPost(AbstractHTTPServlet.java:220) [!/:3.2.7.fuse-760026-redhat-00001] at javax.servlet.http.HttpServlet.service(HttpServlet.java:665) [!/:4.0.1] at org.apache.cxf.transport.servlet.AbstractHTTPServlet.service(AbstractHTTPServlet.java:276) [!/:3.2.7.fuse-760026-redhat-00001] at io.undertow.servlet.handlers.ServletHandler.handleRequest(ServletHandler.java:74) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.security.ServletSecurityRoleHandler.handleRequest(ServletSecurityRoleHandler.java:62) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletChain$1.handleRequest(ServletChain.java:68) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletDispatchingHandler.handleRequest(ServletDispatchingHandler.java:36) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.RedirectDirHandler.handleRequest(RedirectDirHandler.java:68) [!/:2.0.26.SP3-redhat-00001] at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43) [!/:2.0.26.SP3-redhat-00001] at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:269) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletInitialHandler.access$100(ServletInitialHandler.java:78) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:133) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:130) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:48) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:249) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletInitialHandler.dispatchToPath(ServletInitialHandler.java:186) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.spec.AsyncContextImpl$2$1.handleRequest(AsyncContextImpl.java:194) [!/:2.0.26.SP3-redhat-00001] at io.undertow.server.Connectors.executeRootHandler(Connectors.java:376) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.spec.AsyncContextImpl$2.run(AsyncContextImpl.java:191) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.spec.AsyncContextImpl$6.run(AsyncContextImpl.java:465) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.spec.AsyncContextImpl$TaskDispatchRunnable.run(AsyncContextImpl.java:584) [!/:2.0.26.SP3-redhat-00001] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_241] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_241] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_241] Caused by: [java.io|https://java.io/] .IOException: Broken pipe at [sun.nio.ch|https://sun.nio.ch/] .FileDispatcherImpl.write0(Native Method) ~[?:1.8.0_241] at [sun.nio.ch|https://sun.nio.ch/] .SocketDispatcher.write(SocketDispatcher.java:47) ~[?:1.8.0_241] at [sun.nio.ch|https://sun.nio.ch/] .IOUtil.writeFromNativeBuffer(IOUtil.java:93) ~[?:1.8.0_241] at [sun.nio.ch|https://sun.nio.ch/] .IOUtil.write(IOUtil.java:65) ~[?:1.8.0_241] at [sun.nio.ch|https://sun.nio.ch/] .SocketChannelImpl.write(SocketChannelImpl.java:471) ~[?:1.8.0_241] at org.xnio.nio.NioSocketConduit.write(NioSocketConduit.java:164) ~[?:?] at io.undertow.conduits.IdleTimeoutConduit.write(IdleTimeoutConduit.java:123) ~[?:?] at io.undertow.server.protocol.http.HttpResponseConduit.write(HttpResponseConduit.java:609) ~[?:?] at io.undertow.conduits.ChunkedStreamSinkConduit.flush(ChunkedStreamSinkConduit.java:270) ~[?:?] at org.xnio.conduits.ConduitStreamSinkChannel.flush(ConduitStreamSinkChannel.java:162) ~[?:?] at io.undertow.channels.DetachableStreamSinkChannel.flush(DetachableStreamSinkChannel.java:119) ~[?:?] at org.xnio.channels.Channels.flushBlocking(Channels.java:63) ~[?:?] at io.undertow.servlet.spec.ServletOutputStreamImpl.close(ServletOutputStreamImpl.java:618) ~[?:?] at org.apache.cxf.transport.http.AbstractHTTPDestination$WrappedOutputStream.close(AbstractHTTPDestination.java:825) ~[!/:3.2.7.fuse-760026-redhat-00001] at [org.apache.cxf.io|https://org.apache.cxf.io/] .CacheAndWriteOutputStream.postClose(CacheAndWriteOutputStream.java:56) ~[!/:3.2.7.fuse-760026-redhat-00001] at [org.apache.cxf.io|https://org.apache.cxf.io/] .CachedOutputStream.close(CachedOutputStream.java:225) ~[!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.AbstractConduit.close(AbstractConduit.java:56) ~[!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.http.AbstractHTTPDestination$BackChannelConduit.close(AbstractHTTPDestination.java:756) ~[!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.interceptor.MessageSenderInterceptor$MessageSenderEndingInterceptor.handleMessage(MessageSenderInterceptor.java:63) ~[!/:3.2.7.fuse-760026-redhat-00001] ... 35 more 2020-07-08 18:20:45,753 | WARN | XNIO-8 task-4 | o.a.c.p.PhaseInterceptorChain | 83 - org.apache.cxf.cxf-core - 3.2.7.fuse-760026-redhat-00001 | Interceptor for { [http://www.asdf.com/user/location/userprofile/1/|http://www.asdf.com/user/location/userprofile/1/] }ProfileLocator#{ [http://www.asdf.com/user/location/userprofile/1/|http://www.asdf.com/user/location/userprofile/1/] }locateProfile has thrown exception, unwinding now org.apache.cxf.binding.soap.SoapFault: Error writing to XMLStreamWriter. at org.apache.cxf.binding.soap.interceptor.SoapOutInterceptor$SoapOutEndingInterceptor.handleMessage(SoapOutInterceptor.java:318) ~[!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.binding.soap.interceptor.SoapOutInterceptor$SoapOutEndingInterceptor.handleMessage(SoapOutInterceptor.java:294) ~[!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) ~[!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.interceptor.AbstractFaultChainInitiatorObserver.onMessage(AbstractFaultChainInitiatorObserver.java:112) ~[!/:3.2.7.fuse-760026-redhat-00001] at org.apache.camel.component.cxf.CxfConsumer.lambda$createServer$0(CxfConsumer.java:88) ~[?:?] at org.apache.cxf.phase.PhaseInterceptorChain.wrapExceptionAsFault(PhaseInterceptorChain.java:374) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:332) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.interceptor.OutgoingChainInterceptor.handleMessage(OutgoingChainInterceptor.java:90) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.phase.PhaseInterceptorChain.resume(PhaseInterceptorChain.java:278) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:78) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:267) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.http_undertow.UndertowHTTPDestination.doService(UndertowHTTPDestination.java:200) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.http_undertow.CxfUndertowServlet.invoke(CxfUndertowServlet.java:52) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.servlet.AbstractHTTPServlet.handleRequest(AbstractHTTPServlet.java:301) [!/:3.2.7.fuse-760026-redhat-00001] at org.apache.cxf.transport.servlet.AbstractHTTPServlet.doPost(AbstractHTTPServlet.java:220) [!/:3.2.7.fuse-760026-redhat-00001] at javax.servlet.http.HttpServlet.service(HttpServlet.java:665) [!/:4.0.1] at org.apache.cxf.transport.servlet.AbstractHTTPServlet.service(AbstractHTTPServlet.java:276) [!/:3.2.7.fuse-760026-redhat-00001] at io.undertow.servlet.handlers.ServletHandler.handleRequest(ServletHandler.java:74) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.security.ServletSecurityRoleHandler.handleRequest(ServletSecurityRoleHandler.java:62) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletChain$1.handleRequest(ServletChain.java:68) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletDispatchingHandler.handleRequest(ServletDispatchingHandler.java:36) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.RedirectDirHandler.handleRequest(RedirectDirHandler.java:68) [!/:2.0.26.SP3-redhat-00001] at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43) [!/:2.0.26.SP3-redhat-00001] at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:269) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletInitialHandler.access$100(ServletInitialHandler.java:78) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:133) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:130) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:48) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:249) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.handlers.ServletInitialHandler.dispatchToPath(ServletInitialHandler.java:186) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.spec.AsyncContextImpl$2$1.handleRequest(AsyncContextImpl.java:194) [!/:2.0.26.SP3-redhat-00001] at io.undertow.server.Connectors.executeRootHandler(Connectors.java:376) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.spec.AsyncContextImpl$2.run(AsyncContextImpl.java:191) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.spec.AsyncContextImpl$6.run(AsyncContextImpl.java:465) [!/:2.0.26.SP3-redhat-00001] at io.undertow.servlet.spec.AsyncContextImpl$TaskDispatchRunnable.run(AsyncContextImpl.java:584) [!/:2.0.26.SP3-redhat-00001] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_241] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_241] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_241] Caused by: javax.xml.stream.XMLStreamException: No open start element, when trying to write end element at com.ctc.wstx.sw.BaseStreamWriter.throwOutputError(BaseStreamWriter.java:1537) ~[!/:5.0.3.redhat-1] at com.ctc.wstx.sw.BaseStreamWriter.reportNwfStructure(BaseStreamWriter.java:1566) ~[!/:5.0.3.redhat-1] at com.ctc.wstx.sw.BaseNsStreamWriter.doWriteEndTag(BaseNsStreamWriter.java:648) ~[!/:5.0.3.redhat-1] at com.ctc.wstx.sw.BaseNsStreamWriter.writeEndElement(BaseNsStreamWriter.java:280) ~[!/:5.0.3.redhat-1] at org.apache.cxf.binding.soap.interceptor.SoapOutInterceptor$SoapOutEndingInterceptor.handleMessage(SoapOutInterceptor.java:304) ~[!/:3.2.7.fuse-760026-redhat-00001] ... 40 more
