BasicConnectionTable.java in JGroups-2.4.1-sp4.src/src/org/jgroups/blocks does not use readFully to read the cookie and so during high load, runs into cookie related error. Changing the line to readFully fixes the issue.

if(in != null) {
initCookie(input_cookie);

// read the cookie first
in.read(input_cookie, 0, input_cookie.length);
if(!matchCookie(input_cookie))
throw new SocketException("ConnectionTable.Connection.readPeerAddress(): cookie sent by " +
client_peer_addr + " does not match own cookie; terminating connection");
// then read the version
version=in.readShort();

doSend when it fails, removes the connection in the exception block and so the second try in _send always fails with the error "2nd attempt to send data failed too" as the connection is already closed.

void doSend(byte[] data, int offset, int length) throws Exception {
try {
// we're using 'double-writes', sending the buffer to the destination in 2 pieces. this would
// ensure that, if the peer closed the connection while we were idle, we would get an exception.
// this won't happen if we use a single write (see Stevens, ch. 5.13).
if(out != null)

{ out.writeInt(length); // write the length of the data buffer first Util.doubleWrite(data, offset, length, out); out.flush(); // may not be very efficient (but safe) }

}
catch(Exception ex)

{ remove(peer_addr); throw ex; }

}

private void _send(byte[] data, int offset, int length) {
synchronized(send_mutex) {
try

{ doSend(data, offset, length); updateLastAccessed(); }

catch(IOException io_ex) {
if(log.isWarnEnabled())
log.warn("peer closed connection, trying to re-send msg");
try

{ doSend(data, offset, length); updateLastAccessed(); }

catch(IOException io_ex2)

{ if(log.isErrorEnabled()) log.error("2nd attempt to send data failed too"); }