18881 com.hp.mwtests.ts.jts.remote.servers.HammerServer value_1 -Dqa.debug=true -Djava.naming.provider.url=file:///tmp -Djava.naming.factory.initial=com.sun.jndi.fscontext.RefFSContextFactory -Dperformanceprofilestore.dir=config/perf_profiles/ -Djdbcprofilestore.dir=config/jdbc_profiles -Dmemorytestprofilestore.dir=config/memory_profiles/ -Dots.server.bindname=value_1 -DCoordinatorEnvironmentBean.maintainHeuristics=NO -DRecoveryEnvironmentBean.recoveryBackoffPeriod=5 -DCoreEnvironmentBean.timeoutFactor=1 -DCoordinatorEnvironmentBean.defaultTimeout=120 -Demma.coverage.out.file=./testoutput/jtsremote/JTSRemote_DistributedHammerTest2/server1-coverage.ec -DportOffsetId=2 -DObjectStoreBaseDir=/home/mmusgrov/source/forks/narayana/narayana/qa/testoutput/jtsremote/JTSRemote_DistributedHammerTest2/server1 -DRecoveryEnvironmentBean.recoveryListener=true 2013-03-18 17:26:15 Full thread dump Java HotSpot(TM) 64-Bit Server VM (20.4-b02 mixed mode): "Attach Listener" daemon prio=10 tid=0x00007f7788001000 nid=0x4aa0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "p: default-threadpool; w: Idle" daemon prio=10 tid=0x00007f777000f800 nid=0x4a0a waiting for monitor entry [0x00007f778d9d8000] java.lang.Thread.State: BLOCKED (on object monitor) at com.arjuna.ats.txoj.LockManager.propagate(LockManager.java:188) - waiting to lock <0x00000007d93a0938> (a java.lang.Object) - locked <0x00000007d93a06e8> (a java.util.concurrent.locks.ReentrantLock) at com.arjuna.ats.internal.txoj.abstractrecords.LockRecord.nestedCommit(LockRecord.java:137) at com.arjuna.ats.arjuna.coordinator.BasicAction.doCommit(BasicAction.java:2821) at com.arjuna.ats.arjuna.coordinator.BasicAction.doCommit(BasicAction.java:2669) at com.arjuna.ats.arjuna.coordinator.BasicAction.phase2Commit(BasicAction.java:1813) - locked <0x00000007d83e5ce8> (a com.arjuna.ats.internal.jts.orbspecific.interposition.coordinator.ServerTransaction) at com.arjuna.ats.arjuna.coordinator.BasicAction.End(BasicAction.java:1504) - locked <0x00000007d83e5ce8> (a com.arjuna.ats.internal.jts.orbspecific.interposition.coordinator.ServerTransaction) at com.arjuna.ats.internal.jts.orbspecific.coordinator.ArjunaTransactionImple.commit(ArjunaTransactionImple.java:375) at com.arjuna.ats.internal.jts.orbspecific.interposition.resources.arjuna.ServerNestedAction.commit_subtransaction(ServerNestedAction.java:153) at org.omg.CosTransactions.SubtransactionAwareResourcePOATie.commit_subtransaction(SubtransactionAwareResourcePOATie.java:39) at org.omg.CosTransactions.SubtransactionAwareResourcePOA._invoke(SubtransactionAwareResourcePOA.java:43) at com.sun.corba.se.impl.protocol.CorbaServerRequestDispatcherImpl.dispatchToServant(CorbaServerRequestDispatcherImpl.java:637) at com.sun.corba.se.impl.protocol.CorbaServerRequestDispatcherImpl.dispatch(CorbaServerRequestDispatcherImpl.java:189) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequestRequest(CorbaMessageMediatorImpl.java:1682) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequest(CorbaMessageMediatorImpl.java:1540) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleInput(CorbaMessageMediatorImpl.java:922) at com.sun.corba.se.impl.protocol.giopmsgheaders.RequestMessage_1_2.callback(RequestMessage_1_2.java:181) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequest(CorbaMessageMediatorImpl.java:694) at com.sun.corba.se.impl.transport.SocketOrChannelConnectionImpl.dispatch(SocketOrChannelConnectionImpl.java:451) at com.sun.corba.se.impl.transport.SocketOrChannelConnectionImpl.doWork(SocketOrChannelConnectionImpl.java:1213) at com.sun.corba.se.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.performWork(ThreadPoolImpl.java:471) at com.sun.corba.se.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:500) "p: default-threadpool; w: Idle" daemon prio=10 tid=0x00007f777000a000 nid=0x4a09 waiting for monitor entry [0x00007f778dada000] java.lang.Thread.State: BLOCKED (on object monitor) at com.arjuna.ats.arjuna.coordinator.BasicAction.Abort(BasicAction.java:1582) - waiting to lock <0x00000007d85dd1f8> (a com.arjuna.ats.internal.jts.orbspecific.coordinator.ArjunaTransactionImple) at com.arjuna.ats.arjuna.coordinator.BasicAction.checkChildren(BasicAction.java:3281) at com.arjuna.ats.arjuna.coordinator.BasicAction.Abort(BasicAction.java:1618) - locked <0x00000007d85a4440> (a com.arjuna.ats.internal.jts.orbspecific.interposition.coordinator.ServerTransaction) at com.arjuna.ats.internal.jts.orbspecific.coordinator.ArjunaTransactionImple.rollback(ArjunaTransactionImple.java:481) at com.arjuna.ats.internal.jts.orbspecific.interposition.coordinator.ServerTransaction.rollback(ServerTransaction.java:410) at com.arjuna.ats.internal.jts.orbspecific.interposition.resources.arjuna.ServerNestedAction.rollback_subtransaction(ServerNestedAction.java:234) at org.omg.CosTransactions.SubtransactionAwareResourcePOATie.rollback_subtransaction(SubtransactionAwareResourcePOATie.java:44) at org.omg.CosTransactions.SubtransactionAwareResourcePOA._invoke(SubtransactionAwareResourcePOA.java:50) at com.sun.corba.se.impl.protocol.CorbaServerRequestDispatcherImpl.dispatchToServant(CorbaServerRequestDispatcherImpl.java:637) at com.sun.corba.se.impl.protocol.CorbaServerRequestDispatcherImpl.dispatch(CorbaServerRequestDispatcherImpl.java:189) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequestRequest(CorbaMessageMediatorImpl.java:1682) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequest(CorbaMessageMediatorImpl.java:1540) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleInput(CorbaMessageMediatorImpl.java:922) at com.sun.corba.se.impl.protocol.giopmsgheaders.RequestMessage_1_2.callback(RequestMessage_1_2.java:181) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequest(CorbaMessageMediatorImpl.java:694) at com.sun.corba.se.impl.transport.SocketOrChannelConnectionImpl.dispatch(SocketOrChannelConnectionImpl.java:451) at com.sun.corba.se.impl.transport.SocketOrChannelConnectionImpl.doWork(SocketOrChannelConnectionImpl.java:1213) at com.sun.corba.se.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.performWork(ThreadPoolImpl.java:471) at com.sun.corba.se.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:500) "p: default-threadpool; w: Idle" daemon prio=10 tid=0x00007f7770007800 nid=0x4a02 waiting for monitor entry [0x00007f778dbdb000] java.lang.Thread.State: BLOCKED (on object monitor) at com.arjuna.ats.arjuna.StateManager.activate(StateManager.java:232) - waiting to lock <0x00000007d93a06e8> (a java.util.concurrent.locks.ReentrantLock) - locked <0x00000007d934f300> (a com.hp.mwtests.ts.jts.orbspecific.resources.HammerObject) at com.arjuna.ats.arjuna.StateManager.activate(StateManager.java:182) - locked <0x00000007d934f300> (a com.hp.mwtests.ts.jts.orbspecific.resources.HammerObject) at com.arjuna.ats.arjuna.StateManager.activate(StateManager.java:161) at com.arjuna.ats.txoj.LockManager.setlock(LockManager.java:408) - locked <0x00000007d93a0938> (a java.lang.Object) - locked <0x00000007d85dd1f8> (a com.arjuna.ats.internal.jts.orbspecific.coordinator.ArjunaTransactionImple) at com.arjuna.ats.txoj.LockManager.setlock(LockManager.java:323) at com.hp.mwtests.ts.jts.orbspecific.resources.HammerObject.get(HammerObject.java:199) at com.hp.mwtests.ts.jts.TestModule.HammerPOATie.get(HammerPOATie.java:49) at com.hp.mwtests.ts.jts.TestModule.HammerPOA._invoke(HammerPOA.java:64) at com.sun.corba.se.impl.protocol.CorbaServerRequestDispatcherImpl.dispatchToServant(CorbaServerRequestDispatcherImpl.java:637) at com.sun.corba.se.impl.protocol.CorbaServerRequestDispatcherImpl.dispatch(CorbaServerRequestDispatcherImpl.java:189) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequestRequest(CorbaMessageMediatorImpl.java:1682) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequest(CorbaMessageMediatorImpl.java:1540) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleInput(CorbaMessageMediatorImpl.java:922) at com.sun.corba.se.impl.protocol.giopmsgheaders.RequestMessage_1_2.callback(RequestMessage_1_2.java:181) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequest(CorbaMessageMediatorImpl.java:694) at com.sun.corba.se.impl.transport.SocketOrChannelConnectionImpl.dispatch(SocketOrChannelConnectionImpl.java:451) at com.sun.corba.se.impl.transport.SocketOrChannelConnectionImpl.doWork(SocketOrChannelConnectionImpl.java:1213) at com.sun.corba.se.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.performWork(ThreadPoolImpl.java:471) at com.sun.corba.se.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:500) "p: default-threadpool; w: Idle" daemon prio=10 tid=0x00007f7770003000 nid=0x49fc in Object.wait() [0x00007f77b0108000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007d9348118> (a com.sun.corba.se.impl.orbutil.threadpool.WorkQueueImpl) at com.sun.corba.se.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:121) - locked <0x00000007d9348118> (a com.sun.corba.se.impl.orbutil.threadpool.WorkQueueImpl) at com.sun.corba.se.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:484) "Transaction Reaper Worker 0" daemon prio=10 tid=0x00007f77b4364000 nid=0x49d4 in Object.wait() [0x00007f77b0209000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007d93401a8> (a java.util.LinkedList) at java.lang.Object.wait(Object.java:485) at com.arjuna.ats.arjuna.coordinator.TransactionReaper.waitForCancellations(TransactionReaper.java:321) - locked <0x00000007d93401a8> (a java.util.LinkedList) at com.arjuna.ats.internal.arjuna.coordinator.ReaperWorkerThread.run(ReaperWorkerThread.java:65) "Transaction Reaper" daemon prio=10 tid=0x00007f77b435a000 nid=0x49d3 in Object.wait() [0x00007f77b030a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007d93401c0> (a com.arjuna.ats.arjuna.coordinator.TransactionReaper) at com.arjuna.ats.internal.arjuna.coordinator.ReaperThread.run(ReaperThread.java:90) - locked <0x00000007d93401c0> (a com.arjuna.ats.arjuna.coordinator.TransactionReaper) "Listener:55164" daemon prio=10 tid=0x00007f77b4338000 nid=0x49d2 runnable [0x00007f77b040b000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:408) - locked <0x00000007d9340320> (a java.net.SocksSocketImpl) at java.net.ServerSocket.implAccept(ServerSocket.java:462) at java.net.ServerSocket.accept(ServerSocket.java:430) at com.arjuna.ats.internal.arjuna.recovery.Listener.run(Listener.java:122) "SelectorThread" daemon prio=10 tid=0x00007f77b42b7800 nid=0x49d1 runnable [0x00007f77b050c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00000007d9340560> (a sun.nio.ch.Util$2) - locked <0x00000007d9340570> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007d9340518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at com.sun.corba.se.impl.transport.SelectorImpl.run(SelectorImpl.java:256) "Low Memory Detector" daemon prio=10 tid=0x00007f77b4093000 nid=0x49cf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00007f77b4091000 nid=0x49ce waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x00007f77b408e000 nid=0x49cd waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00007f77b408c000 nid=0x49cc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00007f77b406f000 nid=0x49cb in Object.wait() [0x00007f77b0dfc000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007d9390230> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x00000007d9390230> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00007f77b406d800 nid=0x49ca in Object.wait() [0x00007f77b0efd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007d93481d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x00000007d93481d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x00007f77b4009000 nid=0x49c2 in Object.wait() [0x00007f77b9997000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007d938b1a0> (a java.lang.Object) at java.lang.Object.wait(Object.java:485) at com.sun.corba.se.impl.orb.ORBImpl.run(ORBImpl.java:1220) - locked <0x00000007d938b1a0> (a java.lang.Object) at com.arjuna.orbportability.internal.orbspecific.oa.implementations.POABase.run(POABase.java:123) at com.arjuna.orbportability.oa.core.OA.run(OA.java:135) at com.arjuna.orbportability.OA.run(OA.java:381) at com.hp.mwtests.ts.jts.remote.servers.HammerServer.main(HammerServer.java:78) "VM Thread" prio=10 tid=0x00007f77b4066800 nid=0x49c9 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x00007f77b401c000 nid=0x49c5 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x00007f77b401d800 nid=0x49c6 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x00007f77b401f800 nid=0x49c7 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x00007f77b4021800 nid=0x49c8 runnable "VM Periodic Task Thread" prio=10 tid=0x00007f77b40a6000 nid=0x49d0 waiting on condition JNI global references: 1417 Found one Java-level deadlock: ============================= "p: default-threadpool; w: Idle": waiting to lock monitor 0x00007f776400a338 (object 0x00000007d93a0938, a java.lang.Object), which is held by "p: default-threadpool; w: Idle" "p: default-threadpool; w: Idle": waiting to lock monitor 0x00007f7764009c00 (object 0x00000007d93a06e8, a java.util.concurrent.locks.ReentrantLock), which is held by "p: default-threadpool; w: Idle" Java stack information for the threads listed above: =================================================== "p: default-threadpool; w: Idle": at com.arjuna.ats.txoj.LockManager.propagate(LockManager.java:188) - waiting to lock <0x00000007d93a0938> (a java.lang.Object) - locked <0x00000007d93a06e8> (a java.util.concurrent.locks.ReentrantLock) at com.arjuna.ats.internal.txoj.abstractrecords.LockRecord.nestedCommit(LockRecord.java:137) at com.arjuna.ats.arjuna.coordinator.BasicAction.doCommit(BasicAction.java:2821) at com.arjuna.ats.arjuna.coordinator.BasicAction.doCommit(BasicAction.java:2669) at com.arjuna.ats.arjuna.coordinator.BasicAction.phase2Commit(BasicAction.java:1813) - locked <0x00000007d83e5ce8> (a com.arjuna.ats.internal.jts.orbspecific.interposition.coordinator.ServerTransaction) at com.arjuna.ats.arjuna.coordinator.BasicAction.End(BasicAction.java:1504) - locked <0x00000007d83e5ce8> (a com.arjuna.ats.internal.jts.orbspecific.interposition.coordinator.ServerTransaction) at com.arjuna.ats.internal.jts.orbspecific.coordinator.ArjunaTransactionImple.commit(ArjunaTransactionImple.java:375) at com.arjuna.ats.internal.jts.orbspecific.interposition.resources.arjuna.ServerNestedAction.commit_subtransaction(ServerNestedAction.java:153) at org.omg.CosTransactions.SubtransactionAwareResourcePOATie.commit_subtransaction(SubtransactionAwareResourcePOATie.java:39) at org.omg.CosTransactions.SubtransactionAwareResourcePOA._invoke(SubtransactionAwareResourcePOA.java:43) at com.sun.corba.se.impl.protocol.CorbaServerRequestDispatcherImpl.dispatchToServant(CorbaServerRequestDispatcherImpl.java:637) at com.sun.corba.se.impl.protocol.CorbaServerRequestDispatcherImpl.dispatch(CorbaServerRequestDispatcherImpl.java:189) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequestRequest(CorbaMessageMediatorImpl.java:1682) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequest(CorbaMessageMediatorImpl.java:1540) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleInput(CorbaMessageMediatorImpl.java:922) at com.sun.corba.se.impl.protocol.giopmsgheaders.RequestMessage_1_2.callback(RequestMessage_1_2.java:181) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequest(CorbaMessageMediatorImpl.java:694) at com.sun.corba.se.impl.transport.SocketOrChannelConnectionImpl.dispatch(SocketOrChannelConnectionImpl.java:451) at com.sun.corba.se.impl.transport.SocketOrChannelConnectionImpl.doWork(SocketOrChannelConnectionImpl.java:1213) at com.sun.corba.se.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.performWork(ThreadPoolImpl.java:471) at com.sun.corba.se.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:500) "p: default-threadpool; w: Idle": at com.arjuna.ats.arjuna.StateManager.activate(StateManager.java:232) - waiting to lock <0x00000007d93a06e8> (a java.util.concurrent.locks.ReentrantLock) - locked <0x00000007d934f300> (a com.hp.mwtests.ts.jts.orbspecific.resources.HammerObject) at com.arjuna.ats.arjuna.StateManager.activate(StateManager.java:182) - locked <0x00000007d934f300> (a com.hp.mwtests.ts.jts.orbspecific.resources.HammerObject) at com.arjuna.ats.arjuna.StateManager.activate(StateManager.java:161) at com.arjuna.ats.txoj.LockManager.setlock(LockManager.java:408) - locked <0x00000007d93a0938> (a java.lang.Object) - locked <0x00000007d85dd1f8> (a com.arjuna.ats.internal.jts.orbspecific.coordinator.ArjunaTransactionImple) at com.arjuna.ats.txoj.LockManager.setlock(LockManager.java:323) at com.hp.mwtests.ts.jts.orbspecific.resources.HammerObject.get(HammerObject.java:199) at com.hp.mwtests.ts.jts.TestModule.HammerPOATie.get(HammerPOATie.java:49) at com.hp.mwtests.ts.jts.TestModule.HammerPOA._invoke(HammerPOA.java:64) at com.sun.corba.se.impl.protocol.CorbaServerRequestDispatcherImpl.dispatchToServant(CorbaServerRequestDispatcherImpl.java:637) at com.sun.corba.se.impl.protocol.CorbaServerRequestDispatcherImpl.dispatch(CorbaServerRequestDispatcherImpl.java:189) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequestRequest(CorbaMessageMediatorImpl.java:1682) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequest(CorbaMessageMediatorImpl.java:1540) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleInput(CorbaMessageMediatorImpl.java:922) at com.sun.corba.se.impl.protocol.giopmsgheaders.RequestMessage_1_2.callback(RequestMessage_1_2.java:181) at com.sun.corba.se.impl.protocol.CorbaMessageMediatorImpl.handleRequest(CorbaMessageMediatorImpl.java:694) at com.sun.corba.se.impl.transport.SocketOrChannelConnectionImpl.dispatch(SocketOrChannelConnectionImpl.java:451) at com.sun.corba.se.impl.transport.SocketOrChannelConnectionImpl.doWork(SocketOrChannelConnectionImpl.java:1213) at com.sun.corba.se.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.performWork(ThreadPoolImpl.java:471) at com.sun.corba.se.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:500) Found 1 deadlock.