2017-10-08 08:30:12 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.131-b11 mixed mode): "RMI TCP Connection(76)-172.16.103.156" - Thread t@98 java.lang.Thread.State: RUNNABLE at sun.management.ThreadImpl.dumpThreads0(Native Method) at sun.management.ThreadImpl.dumpAllThreads(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.lang.reflect.Method.invoke(Unknown Source) at sun.reflect.misc.Trampoline.invoke(Unknown Source) at sun.reflect.GeneratedMethodAccessor12.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.lang.reflect.Method.invoke(Unknown Source) at sun.reflect.misc.MethodUtil.invoke(Unknown Source) at com.sun.jmx.mbeanserver.ConvertingMethod.invokeWithOpenReturn(Unknown Source) at com.sun.jmx.mbeanserver.ConvertingMethod.invokeWithOpenReturn(Unknown Source) at com.sun.jmx.mbeanserver.MXBeanIntrospector.invokeM2(Unknown Source) at com.sun.jmx.mbeanserver.MXBeanIntrospector.invokeM2(Unknown Source) at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(Unknown Source) at com.sun.jmx.mbeanserver.PerInterface.invoke(Unknown Source) at com.sun.jmx.mbeanserver.MBeanSupport.invoke(Unknown Source) at javax.management.StandardMBean.invoke(Unknown Source) at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(Unknown Source) at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl.doOperation(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl.access$300(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl.invoke(Unknown Source) at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.lang.reflect.Method.invoke(Unknown Source) at sun.rmi.server.UnicastServerRef.dispatch(Unknown Source) at sun.rmi.transport.Transport$1.run(Unknown Source) at sun.rmi.transport.Transport$1.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.Transport.serviceCall(Unknown Source) at sun.rmi.transport.tcp.TCPTransport.handleMessages(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$6/418950160.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - locked <2c3f67b9> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Connection(71)-172.16.103.156" - Thread t@97 java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.io.BufferedInputStream.fill(Unknown Source) at java.io.BufferedInputStream.read(Unknown Source) - locked <2f8315e8> (a java.io.BufferedInputStream) at java.io.FilterInputStream.read(Unknown Source) at sun.rmi.transport.tcp.TCPTransport.handleMessages(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$6/418950160.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - locked <28f4ec9d> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Connection(73)-172.16.103.156" - Thread t@95 java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.io.BufferedInputStream.fill(Unknown Source) at java.io.BufferedInputStream.read(Unknown Source) - locked <2b5bf239> (a java.io.BufferedInputStream) at java.io.FilterInputStream.read(Unknown Source) at sun.rmi.transport.tcp.TCPTransport.handleMessages(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$6/418950160.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - locked <2b645228> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Connection(75)-172.16.103.156" - Thread t@93 java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.io.BufferedInputStream.fill(Unknown Source) at java.io.BufferedInputStream.read(Unknown Source) - locked <72d2052d> (a java.io.BufferedInputStream) at java.io.FilterInputStream.read(Unknown Source) at sun.rmi.transport.tcp.TCPTransport.handleMessages(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$6/418950160.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - locked <6f87c3d> (a java.util.concurrent.ThreadPoolExecutor$Worker) "JMX server connection timeout 92" - Thread t@92 java.lang.Thread.State: TIMED_WAITING at java.lang.Object.wait(Native Method) - waiting on (a [I) at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "RMI TCP Connection(60)-172.16.103.156" - Thread t@91 java.lang.Thread.State: TIMED_WAITING at java.lang.Object.wait(Native Method) - waiting on <6f193056> (a com.sun.jmx.remote.internal.ArrayNotificationBuffer) at com.sun.jmx.remote.internal.ArrayNotificationBuffer.fetchNotifications(Unknown Source) at com.sun.jmx.remote.internal.ArrayNotificationBuffer$ShareBuffer.fetchNotifications(Unknown Source) at com.sun.jmx.remote.internal.ServerNotifForwarder.fetchNotifs(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl$4.run(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl$4.run(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl.fetchNotifications(Unknown Source) at sun.reflect.GeneratedMethodAccessor93.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.lang.reflect.Method.invoke(Unknown Source) at sun.rmi.server.UnicastServerRef.dispatch(Unknown Source) at sun.rmi.transport.Transport$1.run(Unknown Source) at sun.rmi.transport.Transport$1.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.Transport.serviceCall(Unknown Source) at sun.rmi.transport.tcp.TCPTransport.handleMessages(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$6/418950160.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - locked <4eb71dfd> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Connection(63)-172.16.103.156" - Thread t@90 java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.io.BufferedInputStream.fill(Unknown Source) at java.io.BufferedInputStream.read(Unknown Source) - locked <522f8b8c> (a java.io.BufferedInputStream) at java.io.FilterInputStream.read(Unknown Source) at sun.rmi.transport.tcp.TCPTransport.handleMessages(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$6/418950160.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - locked <4e011174> (a java.util.concurrent.ThreadPoolExecutor$Worker) "JMX server connection timeout 81" - Thread t@81 java.lang.Thread.State: TIMED_WAITING at java.lang.Object.wait(Native Method) - waiting on <2e5a8b83> (a [I) at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "blc-keepalive-eclidb5014.kenshooprd.local:3306" - Thread t@61 java.lang.Thread.State: TIMED_WAITING at java.lang.Thread.sleep(Native Method) at com.github.shyiko.mysql.binlog.BinaryLogClient$3.run(BinaryLogClient.java:594) at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.util.concurrent.FutureTask.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - locked (a java.util.concurrent.ThreadPoolExecutor$Worker) "blc-eclidb5014.kenshooprd.local:3306" - Thread t@60 java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at com.github.shyiko.mysql.binlog.io.BufferedSocketInputStream.read(BufferedSocketInputStream.java:51) at com.github.shyiko.mysql.binlog.io.ByteArrayInputStream.readWithinBlockBoundaries(ByteArrayInputStream.java:202) at com.github.shyiko.mysql.binlog.io.ByteArrayInputStream.peek(ByteArrayInputStream.java:175) at com.github.shyiko.mysql.binlog.BinaryLogClient.listenForEventPackets(BinaryLogClient.java:726) at com.github.shyiko.mysql.binlog.BinaryLogClient.connect(BinaryLogClient.java:472) at com.github.shyiko.mysql.binlog.BinaryLogClient$5.run(BinaryLogClient.java:657) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - locked <5299a2bb> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "kafka-producer-network-thread | KS5014-dbhistory" - Thread t@58 java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(Unknown Source) at sun.nio.ch.EPollSelectorImpl.doSelect(Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(Unknown Source) - locked <28653df8> (a sun.nio.ch.Util$3) - locked <36dda4c9> (a java.util.Collections$UnmodifiableSet) - locked <498420f> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(Unknown Source) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:349) at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:225) at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:126) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "pool-3-thread-3" - Thread t@56 java.lang.Thread.State: WAITING at sun.misc.Unsafe.park(Native Method) - parking to wait for <289191ae> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source) at java.util.concurrent.LinkedBlockingQueue.take(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "pool-3-thread-2" - Thread t@55 java.lang.Thread.State: WAITING at sun.misc.Unsafe.park(Native Method) - parking to wait for <289191ae> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source) at java.util.concurrent.LinkedBlockingQueue.take(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "Scheduler-1313532469" - Thread t@49 java.lang.Thread.State: WAITING at sun.misc.Unsafe.park(Native Method) - parking to wait for <1b8bb45d> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "Abandoned connection cleanup thread" - Thread t@45 java.lang.Thread.State: TIMED_WAITING at java.lang.Object.wait(Native Method) - waiting on <4be6f78b> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(Unknown Source) at com.mysql.jdbc.AbandonedConnectionCleanupThread.run(AbandonedConnectionCleanupThread.java:43) Locked ownable synchronizers: - None "pool-4-thread-1" - Thread t@43 java.lang.Thread.State: TIMED_WAITING at sun.misc.Unsafe.park(Native Method) - parking to wait for <7e12caa1> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "pool-3-thread-1" - Thread t@40 java.lang.Thread.State: WAITING at sun.misc.Unsafe.park(Native Method) - parking to wait for <289191ae> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source) at java.util.concurrent.LinkedBlockingQueue.take(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "JMX server connection timeout 39" - Thread t@39 java.lang.Thread.State: TIMED_WAITING at java.lang.Object.wait(Native Method) - waiting on <4b0e2a25> (a [I) at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "RMI TCP Connection(2)-10.53.220.219" - Thread t@38 java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.net.SocketInputStream.read(Unknown Source) at java.io.BufferedInputStream.fill(Unknown Source) at java.io.BufferedInputStream.read(Unknown Source) - locked <1a80d029> (a java.io.BufferedInputStream) at java.io.FilterInputStream.read(Unknown Source) at sun.rmi.transport.tcp.TCPTransport.handleMessages(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$6/418950160.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - locked <58a3d97> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI Scheduler(0)" - Thread t@37 java.lang.Thread.State: TIMED_WAITING at sun.misc.Unsafe.park(Native Method) - parking to wait for <41a0c45c> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "kafka-coordinator-heartbeat-thread | connect-cluster" - Thread t@35 java.lang.Thread.State: TIMED_WAITING at java.lang.Object.wait(Native Method) - waiting on <7ee6e5d> (a org.apache.kafka.connect.runtime.distributed.WorkerCoordinator) at org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatThread.run(AbstractCoordinator.java:926) Locked ownable synchronizers: - None "KafkaBasedLog Work Thread - connect-cluster-configs" - Thread t@33 java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(Unknown Source) at sun.nio.ch.EPollSelectorImpl.doSelect(Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(Unknown Source) - locked <41011ba1> (a sun.nio.ch.Util$3) - locked <5d1c9a30> (a java.util.Collections$UnmodifiableSet) - locked <1a5927eb> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(Unknown Source) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:349) at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:226) - locked <6eb13e7b> (a org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient) at org.apache.kafka.clients.consumer.KafkaConsumer.pollOnce(KafkaConsumer.java:1047) at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:995) at org.apache.kafka.connect.util.KafkaBasedLog.poll(KafkaBasedLog.java:249) at org.apache.kafka.connect.util.KafkaBasedLog.access$500(KafkaBasedLog.java:70) at org.apache.kafka.connect.util.KafkaBasedLog$WorkThread.run(KafkaBasedLog.java:320) Locked ownable synchronizers: - None "kafka-producer-network-thread | producer-3" - Thread t@32 java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(Unknown Source) at sun.nio.ch.EPollSelectorImpl.doSelect(Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(Unknown Source) - locked <27a4fc1f> (a sun.nio.ch.Util$3) - locked <1e503fc1> (a java.util.Collections$UnmodifiableSet) - locked <4de5359f> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(Unknown Source) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:349) at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:225) at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:126) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "KafkaBasedLog Work Thread - connect-cluster-status" - Thread t@31 java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(Unknown Source) at sun.nio.ch.EPollSelectorImpl.doSelect(Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(Unknown Source) - locked <2a4098be> (a sun.nio.ch.Util$3) - locked <2a84cded> (a java.util.Collections$UnmodifiableSet) - locked <3535514b> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(Unknown Source) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:349) at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:226) - locked <5118d3bf> (a org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient) at org.apache.kafka.clients.consumer.KafkaConsumer.pollOnce(KafkaConsumer.java:1047) at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:995) at org.apache.kafka.connect.util.KafkaBasedLog.poll(KafkaBasedLog.java:249) at org.apache.kafka.connect.util.KafkaBasedLog.access$500(KafkaBasedLog.java:70) at org.apache.kafka.connect.util.KafkaBasedLog$WorkThread.run(KafkaBasedLog.java:320) Locked ownable synchronizers: - None "kafka-producer-network-thread | producer-2" - Thread t@30 java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(Unknown Source) at sun.nio.ch.EPollSelectorImpl.doSelect(Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(Unknown Source) - locked <7f33344a> (a sun.nio.ch.Util$3) - locked <73d465e4> (a java.util.Collections$UnmodifiableSet) - locked <6cf4cddc> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(Unknown Source) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:349) at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:225) at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:126) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "KafkaBasedLog Work Thread - connect-cluster-offsets" - Thread t@29 java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(Unknown Source) at sun.nio.ch.EPollSelectorImpl.doSelect(Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(Unknown Source) - locked <7be66ed6> (a sun.nio.ch.Util$3) - locked <1d118484> (a java.util.Collections$UnmodifiableSet) - locked <4dba2987> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(Unknown Source) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:349) at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:226) - locked <7ea87c1> (a org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient) at org.apache.kafka.clients.consumer.KafkaConsumer.pollOnce(KafkaConsumer.java:1047) at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:995) at org.apache.kafka.connect.util.KafkaBasedLog.poll(KafkaBasedLog.java:249) at org.apache.kafka.connect.util.KafkaBasedLog.access$500(KafkaBasedLog.java:70) at org.apache.kafka.connect.util.KafkaBasedLog$WorkThread.run(KafkaBasedLog.java:320) Locked ownable synchronizers: - None "org.eclipse.jetty.server.session.HashSessionManager@2e385cceTimer" - Thread t@28 java.lang.Thread.State: TIMED_WAITING at sun.misc.Unsafe.park(Native Method) - parking to wait for <6519edd> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "qtp497359413-27" - Thread t@27 java.lang.Thread.State: TIMED_WAITING at sun.misc.Unsafe.park(Native Method) - parking to wait for <72a506b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Unknown Source) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "qtp497359413-26" - Thread t@26 java.lang.Thread.State: TIMED_WAITING at sun.misc.Unsafe.park(Native Method) - parking to wait for <72a506b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Unknown Source) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "qtp497359413-25" - Thread t@25 java.lang.Thread.State: TIMED_WAITING at sun.misc.Unsafe.park(Native Method) - parking to wait for <72a506b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Unknown Source) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "qtp497359413-24-acceptor-0@6e76f4fd-ServerConnector@45efc20d{HTTP/1.1}{0.0.0.0:8083}" - Thread t@24 java.lang.Thread.State: RUNNABLE at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(Unknown Source) at sun.nio.ch.ServerSocketChannelImpl.accept(Unknown Source) - locked <4fb2c934> (a java.lang.Object) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:377) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:500) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "qtp497359413-23-selector-ServerConnectorManager@2b21c14a/3" - Thread t@23 java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(Unknown Source) at sun.nio.ch.EPollSelectorImpl.doSelect(Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(Unknown Source) - locked <5e9641e2> (a sun.nio.ch.Util$3) - locked <427cbda6> (a java.util.Collections$UnmodifiableSet) - locked <11fbfea7> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(Unknown Source) at sun.nio.ch.SelectorImpl.select(Unknown Source) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "qtp497359413-22-selector-ServerConnectorManager@2b21c14a/2" - Thread t@22 java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(Unknown Source) at sun.nio.ch.EPollSelectorImpl.doSelect(Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(Unknown Source) - locked <48597ec5> (a sun.nio.ch.Util$3) - locked <24acaf7d> (a java.util.Collections$UnmodifiableSet) - locked <31aa7c3> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(Unknown Source) at sun.nio.ch.SelectorImpl.select(Unknown Source) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "qtp497359413-21-selector-ServerConnectorManager@2b21c14a/1" - Thread t@21 java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(Unknown Source) at sun.nio.ch.EPollSelectorImpl.doSelect(Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(Unknown Source) - locked <4d5a7fc1> (a sun.nio.ch.Util$3) - locked <4ef69ff9> (a java.util.Collections$UnmodifiableSet) - locked (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(Unknown Source) at sun.nio.ch.SelectorImpl.select(Unknown Source) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "qtp497359413-20-selector-ServerConnectorManager@2b21c14a/0" - Thread t@20 java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(Unknown Source) at sun.nio.ch.EPollSelectorImpl.doSelect(Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(Unknown Source) - locked <59928c9d> (a sun.nio.ch.Util$3) - locked <47285419> (a java.util.Collections$UnmodifiableSet) - locked <3d5b0252> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(Unknown Source) at sun.nio.ch.SelectorImpl.select(Unknown Source) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600) at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549) at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "kafka-producer-network-thread | producer-1" - Thread t@17 java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(Unknown Source) at sun.nio.ch.EPollSelectorImpl.doSelect(Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(Unknown Source) - locked <3204ab1a> (a sun.nio.ch.Util$3) - locked <7f3f3fa3> (a java.util.Collections$UnmodifiableSet) - locked <7ab234c1> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(Unknown Source) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:349) at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:225) at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:126) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "DistributedHerder" - Thread t@16 java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(Unknown Source) at sun.nio.ch.EPollSelectorImpl.doSelect(Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(Unknown Source) - locked <1f1ef057> (a sun.nio.ch.Util$3) - locked <855d4d5> (a java.util.Collections$UnmodifiableSet) - locked (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(Unknown Source) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:349) at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:226) - locked <65ee1974> (a org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient) at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:203) at org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.poll(WorkerCoordinator.java:126) at org.apache.kafka.connect.runtime.distributed.WorkerGroupMember.poll(WorkerGroupMember.java:144) at org.apache.kafka.connect.runtime.distributed.DistributedHerder.tick(DistributedHerder.java:302) at org.apache.kafka.connect.runtime.distributed.DistributedHerder.run(DistributedHerder.java:198) at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.util.concurrent.FutureTask.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - locked (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" - Thread t@14 java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(Unknown Source) at java.net.ServerSocket.implAccept(Unknown Source) at java.net.ServerSocket.accept(Unknown Source) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "RMI TCP Accept-9853" - Thread t@13 java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(Unknown Source) at java.net.ServerSocket.implAccept(Unknown Source) at java.net.ServerSocket.accept(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "RMI TCP Accept-0" - Thread t@12 java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(Unknown Source) at java.net.ServerSocket.implAccept(Unknown Source) at java.net.ServerSocket.accept(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - None "Signal Dispatcher" - Thread t@5 java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" - Thread t@3 java.lang.Thread.State: WAITING at java.lang.Object.wait(Native Method) - waiting on (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(Unknown Source) at java.lang.ref.ReferenceQueue.remove(Unknown Source) at java.lang.ref.Finalizer$FinalizerThread.run(Unknown Source) Locked ownable synchronizers: - None "Reference Handler" - Thread t@2 java.lang.Thread.State: WAITING at java.lang.Object.wait(Native Method) - waiting on <2bcf5fcc> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Unknown Source) at java.lang.ref.Reference.tryHandlePending(Unknown Source) at java.lang.ref.Reference$ReferenceHandler.run(Unknown Source) Locked ownable synchronizers: - None "main" - Thread t@1 java.lang.Thread.State: WAITING at sun.misc.Unsafe.park(Native Method) - parking to wait for <31d85445> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(Unknown Source) at java.util.concurrent.CountDownLatch.await(Unknown Source) at org.apache.kafka.connect.runtime.Connect.awaitStop(Connect.java:80) at org.apache.kafka.connect.cli.ConnectDistributed.main(ConnectDistributed.java:91) Locked ownable synchronizers: - None