Major Rarely our CI will fail with some tests failing to shutdown. This was tracked down to an issue where UNICAST3 can throw a ConcurrentModificationException during a view change if a concurrent transmit is also happening.
Stack trace is as follows:
15:12:18,906 WARN (jgroups-8,RehashWithSharedStoreTest-NodeB:[]) [o.j.p.UNICAST3] JGRP000039: RehashWithSharedStoreTest-NodeB: failed to deliver OOB message [RehashWithSharedStoreTest-NodeC to RehashWithSharedStoreTest-NodeB, 0 bytes, flags=OOB]: java.util.ConcurrentModificationException java.util.ConcurrentModificationException: null at java.base/java.util.HashMap$HashIterator.nextNode(HashMap.java:1605) ~[?:?] at java.base/java.util.HashMap$KeyIterator.next(HashMap.java:1628) ~[?:?] at java.base/java.util.AbstractCollection.retainAll(AbstractCollection.java:405) ~[?:?] at org.jgroups.protocols.UNICAST3.down(UNICAST3.java:628) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.pbcast.STABLE.down(STABLE.java:260) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.pbcast.GMS.installView(GMS.java:676) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.pbcast.ServerGmsImpl.handleViewChange(ServerGmsImpl.java:66) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.pbcast.GMS.castViewChangeAndSendJoinRsps(GMS.java:554) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.pbcast.CoordGmsImpl.handleMembershipChange(CoordGmsImpl.java:195) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.pbcast.GMS.process(GMS.java:1272) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.pbcast.ViewHandler.process(ViewHandler.java:264) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.pbcast.ViewHandler.add(ViewHandler.java:64) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.pbcast.GMS.handle(GMS.java:956) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:846) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:226) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1075) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.UNICAST3.addMessage(UNICAST3.java:814) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:796) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:447) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:672) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.Discovery.up(Discovery.java:294) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.stack.Protocol.up(Protocol.java:340) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.protocols.TP.passMessageUp(TP.java:1184) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:107) ~[jgroups-5.3.0.Final.jar:5.3.0.Final] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?] at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
