Concurrent startup of JBoss nodes fails with exceptions

Since this is a known issue, I am creating this issue and in case it cannot be fixed to make sure that:

• people know about it and save themselves some trouble,
• this gets to the documentation,
• try to make it fail less bad.

Starting 2 nodes concurrently or within very short time (few seconds)
./bin/run.sh -c production1 -b localhost -Djboss.messaging.ServerPeerID=1
./bin/run.sh -c production2 -b rhusar.englab.brq.redhat.com -Djboss.messaging.ServerPeerID=2
results in the stack traces below.

The second VM cannot be terminated with the SIGINT signal, which is quite nasty.

Node 1:

...
---------------------------------------------------------
GMS: address is 127.0.0.1:34177 (cluster=DefaultPartition-HAPartitionCache)
---------------------------------------------------------
14:45:31,387 INFO [DefaultPartition] Number of cluster members: 2
14:45:31,387 INFO [DefaultPartition] Other members: 1
14:45:31,489 INFO [RPCManagerImpl] Received new cluster view: [10.34.32.216:43284|1] [10.34.32.216:43284, 127.0.0.1:34177]
14:45:31,607 INFO [LegacyStateTransferIntegrator] Using version 4096
14:45:31,619 INFO [RPCManagerImpl] Cache local address is 127.0.0.1:34177
14:45:31,619 INFO [RPCManagerImpl] Disconnecting and closing the Channel
14:45:31,651 ERROR [AbstractKernelController] Error installing to Start: name=HAPartition state=Create
org.jboss.cache.CacheException: java.lang.reflect.InvocationTargetException
at org.jboss.cache.util.reflect.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:148)
at org.jboss.cache.factories.ComponentRegistry$PrioritizedMethod.invoke(ComponentRegistry.java:1012)
at org.jboss.cache.factories.ComponentRegistry.internalStart(ComponentRegistry.java:778)
at org.jboss.cache.factories.ComponentRegistry.start(ComponentRegistry.java:632)
at org.jboss.cache.invocation.CacheInvocationDelegate.start(CacheInvocationDelegate.java:345)
at org.jboss.ha.cachemanager.CacheManagerManagedCache.start(CacheManagerManagedCache.java:98)
at org.jboss.ha.framework.server.HAPartitionCacheHandlerImpl.startCache(HAPartitionCacheHandlerImpl.java:66)
at org.jboss.ha.framework.server.ClusterPartition.startService(ClusterPartition.java:397)
at org.jboss.system.ServiceMBeanSupport.jbossInternalStart(ServiceMBeanSupport.java:376)
at org.jboss.system.ServiceMBeanSupport.pojoStart(ServiceMBeanSupport.java:216)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.jboss.reflect.plugins.introspection.ReflectionUtils.invoke(ReflectionUtils.java:59)
at org.jboss.reflect.plugins.introspection.ReflectMethodInfoImpl.invoke(ReflectMethodInfoImpl.java:150)
at org.jboss.joinpoint.plugins.BasicMethodJoinPoint.dispatch(BasicMethodJoinPoint.java:66)
at org.jboss.kernel.plugins.dependency.KernelControllerContextAction$JoinpointDispatchWrapper.execute(KernelControllerContextAction.java:243)
at org.jboss.kernel.plugins.dependency.ExecutionWrapper.execute(ExecutionWrapper.java:47)
at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchExecutionWrapper(KernelControllerContextAction.java:111)
at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchJoinPoint(KernelControllerContextAction.java:72)
at org.jboss.kernel.plugins.dependency.LifecycleAction.installActionInternal(LifecycleAction.java:221)
at org.jboss.kernel.plugins.dependency.InstallsAwareAction.installAction(InstallsAwareAction.java:54)
at org.jboss.kernel.plugins.dependency.InstallsAwareAction.installAction(InstallsAwareAction.java:42)
at org.jboss.dependency.plugins.action.SimpleControllerContextAction.simpleInstallAction(SimpleControllerContextAction.java:62)
at org.jboss.dependency.plugins.action.AccessControllerContextAction.install(AccessControllerContextAction.java:71)
at org.jboss.dependency.plugins.AbstractControllerContextActions.install(AbstractControllerContextActions.java:51)
at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:348)
at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1632)
at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:935)
at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:1083)
at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:985)
at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:775)
at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:540)
at org.jboss.deployers.vfs.deployer.kernel.BeanMetaDataDeployer.deploy(BeanMetaDataDeployer.java:121)
at org.jboss.deployers.vfs.deployer.kernel.BeanMetaDataDeployer.deploy(BeanMetaDataDeployer.java:51)
at org.jboss.deployers.spi.deployer.helpers.AbstractSimpleRealDeployer.internalDeploy(AbstractSimpleRealDeployer.java:62)
at org.jboss.deployers.spi.deployer.helpers.AbstractRealDeployer.deploy(AbstractRealDeployer.java:50)
at org.jboss.deployers.plugins.deployers.DeployerWrapper.deploy(DeployerWrapper.java:171)
at org.jboss.deployers.plugins.deployers.DeployersImpl.doDeploy(DeployersImpl.java:1440)
at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:1158)
at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:1179)
at org.jboss.deployers.plugins.deployers.DeployersImpl.install(DeployersImpl.java:1099)
at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:348)
at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1632)
at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:935)
at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:1083)
at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:985)
at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:823)
at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:553)
at org.jboss.deployers.plugins.deployers.DeployersImpl.process(DeployersImpl.java:782)
at org.jboss.deployers.plugins.main.MainDeployerImpl.process(MainDeployerImpl.java:702)
at org.jboss.system.server.profileservice.repository.MainDeployerAdapter.process(MainDeployerAdapter.java:117)
at org.jboss.system.server.profileservice.repository.ProfileDeployAction.install(ProfileDeployAction.java:70)
at org.jboss.system.server.profileservice.repository.AbstractProfileAction.install(AbstractProfileAction.java:53)
at org.jboss.system.server.profileservice.repository.AbstractProfileService.install(AbstractProfileService.java:403)
at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:348)
at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1632)
at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:935)
at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:1083)
at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:985)
at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:775)
at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:540)
at org.jboss.system.server.profileservice.repository.AbstractProfileService.registerProfile(AbstractProfileService.java:308)
at org.jboss.system.server.profileservice.ProfileServiceBootstrap.start(ProfileServiceBootstrap.java:256)
at org.jboss.bootstrap.AbstractServerImpl.start(AbstractServerImpl.java:461)
at org.jboss.Main.boot(Main.java:221)
at org.jboss.Main$1.run(Main.java:556)
at java.lang.Thread.run(Thread.java:619)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.jboss.cache.util.reflect.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:144)
... 68 more
Caused by: org.jboss.cache.CacheException: Unable to fetch state on startup
at org.jboss.cache.RPCManagerImpl.start(RPCManagerImpl.java:396)
... 73 more
Caused by: org.jboss.cache.CacheException: java.lang.ClassCastException: org.jboss.cache.marshall.NodeDataExceptionMarker cannot be cast to java.util.List
at org.jboss.cache.statetransfer.LegacyStateTransferIntegrator.integrateTransientState(LegacyStateTransferIntegrator.java:118)
at org.jboss.cache.statetransfer.LegacyStateTransferIntegrator.integrateState(LegacyStateTransferIntegrator.java:88)
at org.jboss.cache.statetransfer.LegacyStateTransferManager.setState(LegacyStateTransferManager.java:155)
at org.jboss.cache.statetransfer.DefaultStateTransferManager.setState(DefaultStateTransferManager.java:163)
at org.jboss.cache.remoting.jgroups.ChannelMessageListener.setState(ChannelMessageListener.java:190)
at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.handleUpEvent(MessageDispatcher.java:676)
at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:776)
at org.jgroups.JChannel.up(JChannel.java:1251)
at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:462)
at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:486)
at org.jgroups.protocols.pbcast.STATE_TRANSFER.handleStateRsp(STATE_TRANSFER.java:430)
at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:124)
at org.jgroups.protocols.FRAG2.up(FRAG2.java:188)
at org.jgroups.protocols.FC.up(FC.java:473)
at org.jgroups.protocols.pbcast.GMS.up(GMS.java:821)
at org.jgroups.protocols.VIEW_SYNC.up(VIEW_SYNC.java:192)
at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:233)
at org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:616)
at org.jgroups.protocols.UNICAST.up(UNICAST.java:282)
at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:758)
at org.jgroups.protocols.BARRIER.up(BARRIER.java:136)
at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167)
at org.jgroups.protocols.FD.up(FD.java:284)
at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:309)
at org.jgroups.protocols.MERGE2.up(MERGE2.java:144)
at org.jgroups.protocols.Discovery.up(Discovery.java:264)
at org.jgroups.protocols.PING.up(PING.java:273)
at org.jgroups.protocols.TP$ProtocolAdapter.up(TP.java:2327)
at org.jgroups.protocols.TP.passMessageUp(TP.java:1261)
at org.jgroups.protocols.TP.access$100(TP.java:49)
at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1838)
at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1817)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
... 1 more
Caused by: java.lang.ClassCastException: org.jboss.cache.marshall.NodeDataExceptionMarker cannot be cast to java.util.List
at org.jboss.cache.statetransfer.LegacyStateTransferIntegrator.readNodesAsList(LegacyStateTransferIntegrator.java:278)
at org.jboss.cache.statetransfer.LegacyStateTransferIntegrator.integrateTransientState(LegacyStateTransferIntegrator.java:217)
at org.jboss.cache.statetransfer.LegacyStateTransferIntegrator.integrateTransientState(LegacyStateTransferIntegrator.java:105)
... 34 more

and node 2:

14:45:31,469 INFO [RPCManagerImpl] Received new cluster view: [10.34.32.216:43284|1] [10.34.32.216:43284, 127.0.0.1:34177]
14:45:31,504 ERROR [ChannelMessageListener] Caught java.lang.NullPointerException while responding to state transfer request
java.lang.NullPointerException
at org.jboss.cache.statetransfer.LegacyStateTransferGenerator.generateNodeDataList(LegacyStateTransferGenerator.java:175)
at org.jboss.cache.statetransfer.LegacyStateTransferGenerator.marshallTransientState(LegacyStateTransferGenerator.java:169)
at org.jboss.cache.statetransfer.LegacyStateTransferGenerator.generateState(LegacyStateTransferGenerator.java:82)
at org.jboss.cache.statetransfer.LegacyStateTransferManager.getState(LegacyStateTransferManager.java:83)
at org.jboss.cache.remoting.jgroups.ChannelMessageListener.getState(ChannelMessageListener.java:163)
at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.handleUpEvent(MessageDispatcher.java:658)
at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:776)
at org.jgroups.JChannel.up(JChannel.java:1338)
at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:462)
at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:486)
at org.jgroups.protocols.pbcast.STATE_TRANSFER.requestApplicationStates(STATE_TRANSFER.java:241)
at org.jgroups.protocols.pbcast.STATE_TRANSFER.handleStateReq(STATE_TRANSFER.java:374)
at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:117)
at org.jgroups.protocols.FRAG2.up(FRAG2.java:188)
at org.jgroups.protocols.FC.up(FC.java:473)
at org.jgroups.protocols.pbcast.GMS.up(GMS.java:821)
at org.jgroups.protocols.VIEW_SYNC.up(VIEW_SYNC.java:192)
at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:233)
at org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:616)
at org.jgroups.protocols.UNICAST.up(UNICAST.java:282)
at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:758)
at org.jgroups.protocols.BARRIER.up(BARRIER.java:136)
at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167)
at org.jgroups.protocols.FD.up(FD.java:284)
at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:309)
at org.jgroups.protocols.MERGE2.up(MERGE2.java:144)
at org.jgroups.protocols.Discovery.up(Discovery.java:264)
at org.jgroups.protocols.PING.up(PING.java:273)
at org.jgroups.protocols.TP$ProtocolAdapter.up(TP.java:2327)
at org.jgroups.protocols.TP.passMessageUp(TP.java:1261)
at org.jgroups.protocols.TP.access$100(TP.java:49)
at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1838)
at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1817)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
14:45:31,613 INFO [RPCManagerImpl] Cache local address is 10.34.32.216:43284
14:45:31,625 INFO [RPCManagerImpl] Received new cluster view: [10.34.32.216:43284|2] [10.34.32.216:43284]
14:45:31,720 INFO [DefaultPartition] New cluster view for partition DefaultPartition: 2 ([10.34.32.216:1099] delta: -1)