Preparing certificates for internal communication Adding /etc/tls-sidecar/cluster-ca-certs/ca.crt to truststore /tmp/topic-operator/replication.truststore.p12 with alias ca Certificate was added to keystore Preparing certificates for internal communication is complete + export MALLOC_ARENA_MAX=2 + MALLOC_ARENA_MAX=2 + JAVA_OPTS=' -Dlog4j2.configurationFile=file:/opt/topic-operator/custom-config/log4j2.properties -Dvertx.cacheDirBase=/tmp -Djava.security.egd=file:/dev/./urandom' ++ get_gc_opts ++ '[' false == true ']' ++ echo '' + JAVA_OPTS=' -Dlog4j2.configurationFile=file:/opt/topic-operator/custom-config/log4j2.properties -Dvertx.cacheDirBase=/tmp -Djava.security.egd=file:/dev/./urandom ' + JAVA_OPTS=' -Dlog4j2.configurationFile=file:/opt/topic-operator/custom-config/log4j2.properties -Dvertx.cacheDirBase=/tmp -Djava.security.egd=file:/dev/./urandom --illegal-access=deny' + exec /usr/bin/tini -w -e 143 -- java -Dlog4j2.configurationFile=file:/opt/topic-operator/custom-config/log4j2.properties -Dvertx.cacheDirBase=/tmp -Djava.security.egd=file:/dev/./urandom --illegal-access=deny -classpath lib/io.strimzi.topic-operator-0.21.0-SNAPSHOT.jar:lib/io.prometheus.simpleclient_common-0.7.0.jar:lib/io.vertx.vertx-micrometer-metrics-3.9.1.jar:lib/com.github.luben.zstd-jni-1.4.5-6.jar:lib/io.fabric8.openshift-model-console-4.13.0.jar:lib/io.fabric8.kubernetes-model-metrics-4.13.0.jar:lib/org.xerial.snappy.snappy-java-1.1.7.7.jar:lib/org.apache.zookeeper.zookeeper-3.5.8.jar:lib/com.101tec.zkclient-0.11.jar:lib/io.fabric8.kubernetes-model-admissionregistration-4.13.0.jar:lib/org.apache.logging.log4j.log4j-api-2.13.3.jar:lib/com.squareup.okio.okio-1.15.0.jar:lib/io.fabric8.kubernetes-model-apps-4.13.0.jar:lib/io.netty.netty-codec-http2-4.1.50.Final.jar:lib/io.fabric8.kubernetes-model-storageclass-4.13.0.jar:lib/io.fabric8.kubernetes-model-node-4.13.0.jar:lib/org.hdrhistogram.HdrHistogram-2.1.11.jar:lib/io.prometheus.simpleclient-0.7.0.jar:lib/io.fabric8.openshift-model-operator-4.13.0.jar:lib/com.squareup.okhttp3.logging-interceptor-3.12.12.jar:lib/javax.xml.bind.jaxb-api-2.3.0.jar:lib/org.apache.yetus.audience-annotations-0.5.0.jar:lib/javax.annotation.javax.annotation-api-1.3.2.jar:lib/io.strimzi.operator-common-0.21.0-SNAPSHOT.jar:lib/com.fasterxml.jackson.core.jackson-core-2.10.5.jar:lib/io.netty.netty-handler-proxy-4.1.50.Final.jar:lib/io.netty.netty-codec-dns-4.1.50.Final.jar:lib/io.micrometer.micrometer-core-1.3.1.jar:lib/org.apache.kafka.kafka-clients-2.7.0.jar:lib/io.fabric8.kubernetes-model-networking-4.13.0.jar:lib/jakarta.activation.jakarta.activation-api-1.2.1.jar:lib/io.fabric8.kubernetes-model-certificates-4.13.0.jar:lib/io.netty.netty-transport-native-epoll-4.1.50.Final.jar:lib/io.netty.netty-transport-4.1.50.Final.jar:lib/com.fasterxml.jackson.dataformat.jackson-dataformat-yaml-2.10.5.jar:lib/io.fabric8.openshift-model-monitoring-4.13.0.jar:lib/com.github.mifmif.generex-1.0.2.jar:lib/io.fabric8.kubernetes-client-4.13.0.jar:lib/io.fabric8.kubernetes-model-settings-4.13.0.jar:lib/io.netty.netty-common-4.1.50.Final.jar:lib/io.strimzi.crd-annotations-0.21.0-SNAPSHOT.jar:lib/io.netty.netty-transport-native-unix-common-4.1.50.Final.jar:lib/io.vertx.vertx-core-3.9.1.jar:lib/io.strimzi.api-0.21.0-SNAPSHOT.jar:lib/io.netty.netty-resolver-dns-4.1.50.Final.jar:lib/org.yaml.snakeyaml-1.26.jar:lib/io.fabric8.kubernetes-model-events-4.13.0.jar:lib/com.fasterxml.jackson.core.jackson-annotations-2.10.5.jar:lib/org.lz4.lz4-java-1.7.1.jar:lib/io.netty.netty-resolver-4.1.50.Final.jar:lib/io.fabric8.kubernetes-model-discovery-4.13.0.jar:lib/org.apache.logging.log4j.log4j-slf4j-impl-2.13.3.jar:lib/dk.brics.automaton.automaton-1.11-8.jar:lib/io.netty.netty-codec-4.1.50.Final.jar:lib/com.fasterxml.jackson.core.jackson-databind-2.10.5.1.jar:lib/io.netty.netty-codec-socks-4.1.50.Final.jar:lib/io.fabric8.kubernetes-model-autoscaling-4.13.0.jar:lib/io.fabric8.kubernetes-model-apiextensions-4.13.0.jar:lib/io.fabric8.kubernetes-model-common-4.13.0.jar:lib/io.fabric8.openshift-client-4.13.0.jar:lib/jakarta.xml.bind.jakarta.xml.bind-api-2.3.2.jar:lib/io.fabric8.zjsonpatch-0.3.0.jar:lib/com.squareup.okhttp3.okhttp-3.12.6.jar:lib/io.fabric8.kubernetes-model-rbac-4.13.0.jar:lib/io.fabric8.kubernetes-model-policy-4.13.0.jar:lib/io.fabric8.kubernetes-model-core-4.13.0.jar:lib/io.netty.netty-handler-4.1.50.Final.jar:lib/io.fabric8.openshift-model-4.13.0.jar:lib/com.fasterxml.jackson.module.jackson-module-jaxb-annotations-2.10.5.jar:lib/org.slf4j.slf4j-api-1.7.25.jar:lib/org.latencyutils.LatencyUtils-2.0.3.jar:lib/io.fabric8.kubernetes-model-coordination-4.13.0.jar:lib/io.fabric8.openshift-model-operatorhub-4.13.0.jar:lib/org.apache.zookeeper.zookeeper-jute-3.5.8.jar:lib/io.strimzi.certificate-manager-0.21.0-SNAPSHOT.jar:lib/io.fabric8.kubernetes-model-scheduling-4.13.0.jar:lib/com.fasterxml.jackson.datatype.jackson-datatype-jsr310-2.11.2.jar:lib/io.netty.netty-buffer-4.1.50.Final.jar:lib/org.apache.logging.log4j.log4j-core-2.13.3.jar:lib/io.fabric8.kubernetes-model-extensions-4.13.0.jar:lib/io.netty.netty-codec-http-4.1.50.Final.jar:lib/io.micrometer.micrometer-registry-prometheus-1.3.1.jar:lib/io.fabric8.kubernetes-model-batch-4.13.0.jar io.strimzi.operator.topic.Main 2021-01-08 08:25:39,01263 INFO [main] Main:30 - TopicOperator 0.21.0-SNAPSHOT is starting 2021-01-08 08:25:39,07696 DEBUG [main] Config:520 - Trying to configure client from Kubernetes config... 2021-01-08 08:25:39,07818 DEBUG [main] Config:526 - Did not find Kubernetes config at: [/.kube/config]. Ignoring. 2021-01-08 08:25:39,07873 DEBUG [main] Config:452 - Trying to configure client from service account... 2021-01-08 08:25:39,07935 DEBUG [main] Config:457 - Found service account host and port: 172.30.0.1:443 2021-01-08 08:25:39,08027 DEBUG [main] Config:463 - Found service account ca cert at: [/var/run/secrets/kubernetes.io/serviceaccount/ca.crt]. 2021-01-08 08:25:39,08110 DEBUG [main] Config:470 - Found service account token at: [/var/run/secrets/kubernetes.io/serviceaccount/token]. 2021-01-08 08:25:39,08166 DEBUG [main] Config:725 - Trying to configure client namespace from Kubernetes service account namespace path... 2021-01-08 08:25:39,08233 DEBUG [main] Config:730 - Found service account namespace at: [/var/run/secrets/kubernetes.io/serviceaccount/namespace]. 2021-01-08 08:25:39,57096 DEBUG [main] InternalLoggerFactory:45 - Using SLF4J as the default logging framework 2021-01-08 08:25:39,57552 DEBUG [main] ResourceLeakDetector:130 - -Dio.netty.leakDetection.level: simple 2021-01-08 08:25:39,57595 DEBUG [main] ResourceLeakDetector:131 - -Dio.netty.leakDetection.targetRecords: 4 2021-01-08 08:25:39,59057 DEBUG [main] InternalThreadLocalMap:56 - -Dio.netty.threadLocalMap.stringBuilder.initialSize: 1024 2021-01-08 08:25:39,59106 DEBUG [main] InternalThreadLocalMap:59 - -Dio.netty.threadLocalMap.stringBuilder.maxSize: 4096 2021-01-08 08:25:39,60303 DEBUG [main] MultithreadEventLoopGroup:44 - -Dio.netty.eventLoopThreads: 2 2021-01-08 08:25:39,64142 DEBUG [main] NioEventLoop:106 - -Dio.netty.noKeySetOptimization: false 2021-01-08 08:25:39,64188 DEBUG [main] NioEventLoop:107 - -Dio.netty.selectorAutoRebuildThreshold: 512 2021-01-08 08:25:39,66511 DEBUG [main] PlatformDependent0:408 - -Dio.netty.noUnsafe: false 2021-01-08 08:25:39,66572 DEBUG [main] PlatformDependent0:876 - Java version: 11 2021-01-08 08:25:39,66773 DEBUG [main] PlatformDependent0:125 - sun.misc.Unsafe.theUnsafe: available 2021-01-08 08:25:39,66881 DEBUG [main] PlatformDependent0:149 - sun.misc.Unsafe.copyMemory: available 2021-01-08 08:25:39,66961 DEBUG [main] PlatformDependent0:187 - java.nio.Buffer.address: available 2021-01-08 08:25:39,67039 DEBUG [main] PlatformDependent0:261 - direct buffer constructor: unavailable java.lang.UnsupportedOperationException: Reflective setAccessible(true) disabled at io.netty.util.internal.ReflectionUtil.trySetAccessible(ReflectionUtil.java:31) ~[io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.util.internal.PlatformDependent0$4.run(PlatformDependent0.java:233) ~[io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at java.security.AccessController.doPrivileged(Native Method) ~[?:?] at io.netty.util.internal.PlatformDependent0.(PlatformDependent0.java:227) [io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.util.internal.PlatformDependent.isAndroid(PlatformDependent.java:289) [io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.util.internal.PlatformDependent.(PlatformDependent.java:92) [io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoop.newTaskQueue0(NioEventLoop.java:279) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoop.newTaskQueue(NioEventLoop.java:150) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoop.(NioEventLoop.java:138) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoopGroup.newChild(NioEventLoopGroup.java:146) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoopGroup.newChild(NioEventLoopGroup.java:37) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.util.concurrent.MultithreadEventExecutorGroup.(MultithreadEventExecutorGroup.java:84) [io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.util.concurrent.MultithreadEventExecutorGroup.(MultithreadEventExecutorGroup.java:58) [io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.util.concurrent.MultithreadEventExecutorGroup.(MultithreadEventExecutorGroup.java:47) [io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.MultithreadEventLoopGroup.(MultithreadEventLoopGroup.java:59) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoopGroup.(NioEventLoopGroup.java:86) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoopGroup.(NioEventLoopGroup.java:81) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoopGroup.(NioEventLoopGroup.java:68) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.vertx.core.net.impl.transport.Transport.eventLoopGroup(Transport.java:153) [io.vertx.vertx-core-3.9.1.jar:3.9.1] at io.vertx.core.impl.VertxImpl.(VertxImpl.java:143) [io.vertx.vertx-core-3.9.1.jar:3.9.1] at io.vertx.core.impl.VertxImpl.vertx(VertxImpl.java:92) [io.vertx.vertx-core-3.9.1.jar:3.9.1] at io.vertx.core.impl.VertxFactoryImpl.vertx(VertxFactoryImpl.java:40) [io.vertx.vertx-core-3.9.1.jar:3.9.1] at io.vertx.core.impl.VertxFactoryImpl.vertx(VertxFactoryImpl.java:32) [io.vertx.vertx-core-3.9.1.jar:3.9.1] at io.vertx.core.Vertx.vertx(Vertx.java:85) [io.vertx.vertx-core-3.9.1.jar:3.9.1] at io.strimzi.operator.topic.Main.deploy(Main.java:50) [io.strimzi.topic-operator-0.21.0-SNAPSHOT.jar:0.21.0-SNAPSHOT] at io.strimzi.operator.topic.Main.run(Main.java:39) [io.strimzi.topic-operator-0.21.0-SNAPSHOT.jar:0.21.0-SNAPSHOT] at io.strimzi.operator.topic.Main.main(Main.java:32) [io.strimzi.topic-operator-0.21.0-SNAPSHOT.jar:0.21.0-SNAPSHOT] 2021-01-08 08:25:39,68065 DEBUG [main] PlatformDependent0:326 - java.nio.Bits.unaligned: available, true 2021-01-08 08:25:39,68267 DEBUG [main] PlatformDependent0:385 - jdk.internal.misc.Unsafe.allocateUninitializedArray(int): unavailable java.lang.IllegalAccessException: class io.netty.util.internal.PlatformDependent0$6 cannot access class jdk.internal.misc.Unsafe (in module java.base) because module java.base does not export jdk.internal.misc to unnamed module @c2db68f at jdk.internal.reflect.Reflection.newIllegalAccessException(Reflection.java:361) ~[?:?] at java.lang.reflect.AccessibleObject.checkAccess(AccessibleObject.java:591) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:558) ~[?:?] at io.netty.util.internal.PlatformDependent0$6.run(PlatformDependent0.java:347) ~[io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at java.security.AccessController.doPrivileged(Native Method) ~[?:?] at io.netty.util.internal.PlatformDependent0.(PlatformDependent0.java:338) [io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.util.internal.PlatformDependent.isAndroid(PlatformDependent.java:289) [io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.util.internal.PlatformDependent.(PlatformDependent.java:92) [io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoop.newTaskQueue0(NioEventLoop.java:279) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoop.newTaskQueue(NioEventLoop.java:150) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoop.(NioEventLoop.java:138) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoopGroup.newChild(NioEventLoopGroup.java:146) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoopGroup.newChild(NioEventLoopGroup.java:37) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.util.concurrent.MultithreadEventExecutorGroup.(MultithreadEventExecutorGroup.java:84) [io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.util.concurrent.MultithreadEventExecutorGroup.(MultithreadEventExecutorGroup.java:58) [io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.util.concurrent.MultithreadEventExecutorGroup.(MultithreadEventExecutorGroup.java:47) [io.netty.netty-common-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.MultithreadEventLoopGroup.(MultithreadEventLoopGroup.java:59) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoopGroup.(NioEventLoopGroup.java:86) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoopGroup.(NioEventLoopGroup.java:81) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.netty.channel.nio.NioEventLoopGroup.(NioEventLoopGroup.java:68) [io.netty.netty-transport-4.1.50.Final.jar:4.1.50.Final] at io.vertx.core.net.impl.transport.Transport.eventLoopGroup(Transport.java:153) [io.vertx.vertx-core-3.9.1.jar:3.9.1] at io.vertx.core.impl.VertxImpl.(VertxImpl.java:143) [io.vertx.vertx-core-3.9.1.jar:3.9.1] at io.vertx.core.impl.VertxImpl.vertx(VertxImpl.java:92) [io.vertx.vertx-core-3.9.1.jar:3.9.1] at io.vertx.core.impl.VertxFactoryImpl.vertx(VertxFactoryImpl.java:40) [io.vertx.vertx-core-3.9.1.jar:3.9.1] at io.vertx.core.impl.VertxFactoryImpl.vertx(VertxFactoryImpl.java:32) [io.vertx.vertx-core-3.9.1.jar:3.9.1] at io.vertx.core.Vertx.vertx(Vertx.java:85) [io.vertx.vertx-core-3.9.1.jar:3.9.1] at io.strimzi.operator.topic.Main.deploy(Main.java:50) [io.strimzi.topic-operator-0.21.0-SNAPSHOT.jar:0.21.0-SNAPSHOT] at io.strimzi.operator.topic.Main.run(Main.java:39) [io.strimzi.topic-operator-0.21.0-SNAPSHOT.jar:0.21.0-SNAPSHOT] at io.strimzi.operator.topic.Main.main(Main.java:32) [io.strimzi.topic-operator-0.21.0-SNAPSHOT.jar:0.21.0-SNAPSHOT] 2021-01-08 08:25:39,68415 DEBUG [main] PlatformDependent0:398 - java.nio.DirectByteBuffer.(long, int): unavailable 2021-01-08 08:25:39,68453 DEBUG [main] PlatformDependent:1058 - sun.misc.Unsafe: available 2021-01-08 08:25:39,68580 DEBUG [main] PlatformDependent:1158 - maxDirectMemory: 4042391552 bytes (maybe) 2021-01-08 08:25:39,68631 DEBUG [main] PlatformDependent:1177 - -Dio.netty.tmpdir: /tmp (java.io.tmpdir) 2021-01-08 08:25:39,68674 DEBUG [main] PlatformDependent:1256 - -Dio.netty.bitMode: 64 (sun.arch.data.model) 2021-01-08 08:25:39,68809 DEBUG [main] PlatformDependent:177 - -Dio.netty.maxDirectMemory: -1 bytes 2021-01-08 08:25:39,68846 DEBUG [main] PlatformDependent:184 - -Dio.netty.uninitializedArrayAllocationThreshold: -1 2021-01-08 08:25:39,69021 DEBUG [main] CleanerJava9:71 - java.nio.ByteBuffer.cleaner(): available 2021-01-08 08:25:39,69059 DEBUG [main] PlatformDependent:204 - -Dio.netty.noPreferDirect: false 2021-01-08 08:25:39,70187 DEBUG [main] PlatformDependent:919 - org.jctools-core.MpscChunkedArrayQueue: available 2021-01-08 08:25:39,77257 DEBUG [main] InternalLoggerFactory:61 - Using SLF4J as the default logging framework 2021-01-08 08:25:39,89291 DEBUG [main] DefaultDnsServerAddressStreamProvider:80 - Default DNS servers: [/172.30.0.2:53] (sun.net.dns.ResolverConfiguration) 2021-01-08 08:25:39,90111 DEBUG [main] NetUtil:139 - -Djava.net.preferIPv4Stack: false 2021-01-08 08:25:39,90155 DEBUG [main] NetUtil:140 - -Djava.net.preferIPv6Addresses: false 2021-01-08 08:25:39,90282 DEBUG [main] NetUtil:224 - Loopback interface: lo (lo, 0:0:0:0:0:0:0:1%lo) 2021-01-08 08:25:39,90413 DEBUG [main] NetUtil:271 - /proc/sys/net/core/somaxconn: 128 2021-01-08 08:25:39,94697 INFO [main] Session:68 - Using config: STRIMZI_TRUSTSTORE_LOCATION: /tmp/topic-operator/replication.truststore.p12 STRIMZI_RESOURCE_LABELS: strimzi.io/cluster=my-cluster STRIMZI_KAFKA_BOOTSTRAP_SERVERS: my-cluster-kafka-bootstrap:9091 STRIMZI_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: HTTPS STRIMZI_NAMESPACE: myproject STRIMZI_ZOOKEEPER_SESSION_TIMEOUT_MS: 20000 STRIMZI_TOPICS_PATH: /strimzi/topics STRIMZI_FULL_RECONCILIATION_INTERVAL_MS: 90000 STRIMZI_ZOOKEEPER_CONNECT: localhost:2181 STRIMZI_TLS_ENABLED: true STRIMZI_KEYSTORE_PASSWORD: ******** STRIMZI_TOPIC_METADATA_MAX_ATTEMPTS: 6 STRIMZI_REASSIGN_VERIFY_INTERVAL_MS: 120000 STRIMZI_KEYSTORE_LOCATION: /tmp/topic-operator/replication.keystore.p12 TC_ZK_CONNECTION_TIMEOUT_MS: 20000 STRIMZI_TRUSTSTORE_PASSWORD: ******** STRIMZI_REASSIGN_THROTTLE: 9223372036854775807 2021-01-08 08:25:39,96559 INFO [vert.x-eventloop-thread-0] Session:138 - Starting 2021-01-08 08:25:39,99843 INFO [vert.x-eventloop-thread-0] AdminClientConfig:361 - AdminClientConfig values: bootstrap.servers = [my-cluster-kafka-bootstrap:9091] client.dns.lookup = use_all_dns_ips client.id = connections.max.idle.ms = 300000 default.api.timeout.ms = 60000 metadata.max.age.ms = 300000 metric.reporters = [] metrics.num.samples = 2 metrics.recording.level = INFO metrics.sample.window.ms = 30000 receive.buffer.bytes = 65536 reconnect.backoff.max.ms = 1000 reconnect.backoff.ms = 50 request.timeout.ms = 30000 retries = 2147483647 retry.backoff.ms = 100 sasl.client.callback.handler.class = null sasl.jaas.config = null sasl.kerberos.kinit.cmd = /usr/bin/kinit sasl.kerberos.min.time.before.relogin = 60000 sasl.kerberos.service.name = null sasl.kerberos.ticket.renew.jitter = 0.05 sasl.kerberos.ticket.renew.window.factor = 0.8 sasl.login.callback.handler.class = null sasl.login.class = null sasl.login.refresh.buffer.seconds = 300 sasl.login.refresh.min.period.seconds = 60 sasl.login.refresh.window.factor = 0.8 sasl.login.refresh.window.jitter = 0.05 sasl.mechanism = GSSAPI security.protocol = SSL security.providers = null send.buffer.bytes = 131072 socket.connection.setup.timeout.max.ms = 127000 socket.connection.setup.timeout.ms = 10000 ssl.cipher.suites = null ssl.enabled.protocols = [TLSv1.2, TLSv1.3] ssl.endpoint.identification.algorithm = HTTPS ssl.engine.factory.class = null ssl.key.password = null ssl.keymanager.algorithm = SunX509 ssl.keystore.certificate.chain = null ssl.keystore.key = null ssl.keystore.location = /tmp/topic-operator/replication.keystore.p12 ssl.keystore.password = [hidden] ssl.keystore.type = JKS ssl.protocol = TLSv1.3 ssl.provider = null ssl.secure.random.implementation = null ssl.trustmanager.algorithm = PKIX ssl.truststore.certificates = null ssl.truststore.location = /tmp/topic-operator/replication.truststore.p12 ssl.truststore.password = [hidden] ssl.truststore.type = JKS 2021-01-08 08:25:40,03806 DEBUG [vert.x-eventloop-thread-0] AdminMetadataManager:245 - [AdminClient clientId=adminclient-1] Setting bootstrap cluster metadata Cluster(id = null, nodes = [my-cluster-kafka-bootstrap:9091 (id: -1 rack: null)], partitions = [], controller = null). 2021-01-08 08:25:40,34827 DEBUG [vert.x-eventloop-thread-0] DefaultSslEngineFactory:264 - Created SSL context with keystore SecurityStore(path=/tmp/topic-operator/replication.keystore.p12, modificationTime=Fri Jan 08 08:25:37 UTC 2021), truststore SecurityStore(path=/tmp/topic-operator/replication.truststore.p12, modificationTime=Fri Jan 08 08:25:37 UTC 2021), provider SunJSSE. 2021-01-08 08:25:40,39465 WARN [vert.x-eventloop-thread-0] AdminClientConfig:369 - The configuration 'ssl.truststore.location' was supplied but isn't a known config. 2021-01-08 08:25:40,39517 WARN [vert.x-eventloop-thread-0] AdminClientConfig:369 - The configuration 'ssl.keystore.password' was supplied but isn't a known config. 2021-01-08 08:25:40,39549 WARN [vert.x-eventloop-thread-0] AdminClientConfig:369 - The configuration 'ssl.keystore.location' was supplied but isn't a known config. 2021-01-08 08:25:40,39579 WARN [vert.x-eventloop-thread-0] AdminClientConfig:369 - The configuration 'ssl.truststore.password' was supplied but isn't a known config. 2021-01-08 08:25:40,39619 WARN [vert.x-eventloop-thread-0] AdminClientConfig:369 - The configuration 'ssl.endpoint.identification.algorithm' was supplied but isn't a known config. 2021-01-08 08:25:40,39983 INFO [vert.x-eventloop-thread-0] AppInfoParser:119 - Kafka version: 2.7.0 2021-01-08 08:25:40,40028 INFO [vert.x-eventloop-thread-0] AppInfoParser:120 - Kafka commitId: 448719dc99a19793 2021-01-08 08:25:40,40061 INFO [vert.x-eventloop-thread-0] AppInfoParser:121 - Kafka startTimeMs: 1610094340396 2021-01-08 08:25:40,40316 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:586 - [AdminClient clientId=adminclient-1] Kafka admin client initialized 2021-01-08 08:25:40,40445 DEBUG [vert.x-eventloop-thread-0] Session:155 - Using AdminClient org.apache.kafka.clients.admin.KafkaAdminClient@665cfb3c 2021-01-08 08:25:40,40831 DEBUG [vert.x-eventloop-thread-0] Session:157 - Using Kafka io.strimzi.operator.topic.KafkaImpl@3be60bc7 2021-01-08 08:25:40,40870 DEBUG [vert.x-eventloop-thread-0] Session:161 - Using namespace myproject 2021-01-08 08:25:40,41802 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:1003 - [AdminClient clientId=adminclient-1] Initiating connection to node my-cluster-kafka-bootstrap:9091 (id: -1 rack: null) using address my-cluster-kafka-bootstrap/172.30.93.106 2021-01-08 08:25:40,46943 DEBUG [vert.x-eventloop-thread-0] Session:163 - Using k8s io.strimzi.operator.topic.K8sImpl@520324be 2021-01-08 08:25:40,47407 DEBUG [kafka-admin-client-thread | adminclient-1] Selector:531 - [AdminClient clientId=adminclient-1] Created socket with SO_RCVBUF = 65536, SO_SNDBUF = 131072, SO_TIMEOUT = 0 to node -1 2021-01-08 08:25:40,61720 DEBUG [vert.x-worker-thread-0] ZkConnection:69 - Creating new ZookKeeper instance to connect to localhost:2181. 2021-01-08 08:25:40,62196 INFO [ZkClient-EventThread-19-localhost:2181] ZkEventThread:65 - Starting ZkClient event thread. 2021-01-08 08:25:40,63002 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:zookeeper.version=3.5.8-f439ca583e70862c3068a1f2a7d4d068eec33315, built on 05/04/2020 15:53 GMT 2021-01-08 08:25:40,63046 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:host.name=my-cluster-entity-operator-76c98dcbcd-xgw8b 2021-01-08 08:25:40,63073 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:java.version=11.0.9.1 2021-01-08 08:25:40,63098 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:java.vendor=Red Hat, Inc. 2021-01-08 08:25:40,63123 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:java.home=/usr/lib/jvm/java-11-openjdk-11.0.9.11-2.el7_9.x86_64 2021-01-08 08:25:40,63151 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:java.class.path=lib/io.strimzi.topic-operator-0.21.0-SNAPSHOT.jar:lib/io.prometheus.simpleclient_common-0.7.0.jar:lib/io.vertx.vertx-micrometer-metrics-3.9.1.jar:lib/com.github.luben.zstd-jni-1.4.5-6.jar:lib/io.fabric8.openshift-model-console-4.13.0.jar:lib/io.fabric8.kubernetes-model-metrics-4.13.0.jar:lib/org.xerial.snappy.snappy-java-1.1.7.7.jar:lib/org.apache.zookeeper.zookeeper-3.5.8.jar:lib/com.101tec.zkclient-0.11.jar:lib/io.fabric8.kubernetes-model-admissionregistration-4.13.0.jar:lib/org.apache.logging.log4j.log4j-api-2.13.3.jar:lib/com.squareup.okio.okio-1.15.0.jar:lib/io.fabric8.kubernetes-model-apps-4.13.0.jar:lib/io.netty.netty-codec-http2-4.1.50.Final.jar:lib/io.fabric8.kubernetes-model-storageclass-4.13.0.jar:lib/io.fabric8.kubernetes-model-node-4.13.0.jar:lib/org.hdrhistogram.HdrHistogram-2.1.11.jar:lib/io.prometheus.simpleclient-0.7.0.jar:lib/io.fabric8.openshift-model-operator-4.13.0.jar:lib/com.squareup.okhttp3.logging-interceptor-3.12.12.jar:lib/javax.xml.bind.jaxb-api-2.3.0.jar:lib/org.apache.yetus.audience-annotations-0.5.0.jar:lib/javax.annotation.javax.annotation-api-1.3.2.jar:lib/io.strimzi.operator-common-0.21.0-SNAPSHOT.jar:lib/com.fasterxml.jackson.core.jackson-core-2.10.5.jar:lib/io.netty.netty-handler-proxy-4.1.50.Final.jar:lib/io.netty.netty-codec-dns-4.1.50.Final.jar:lib/io.micrometer.micrometer-core-1.3.1.jar:lib/org.apache.kafka.kafka-clients-2.7.0.jar:lib/io.fabric8.kubernetes-model-networking-4.13.0.jar:lib/jakarta.activation.jakarta.activation-api-1.2.1.jar:lib/io.fabric8.kubernetes-model-certificates-4.13.0.jar:lib/io.netty.netty-transport-native-epoll-4.1.50.Final.jar:lib/io.netty.netty-transport-4.1.50.Final.jar:lib/com.fasterxml.jackson.dataformat.jackson-dataformat-yaml-2.10.5.jar:lib/io.fabric8.openshift-model-monitoring-4.13.0.jar:lib/com.github.mifmif.generex-1.0.2.jar:lib/io.fabric8.kubernetes-client-4.13.0.jar:lib/io.fabric8.kubernetes-model-settings-4.13.0.jar:lib/io.netty.netty-common-4.1.50.Final.jar:lib/io.strimzi.crd-annotations-0.21.0-SNAPSHOT.jar:lib/io.netty.netty-transport-native-unix-common-4.1.50.Final.jar:lib/io.vertx.vertx-core-3.9.1.jar:lib/io.strimzi.api-0.21.0-SNAPSHOT.jar:lib/io.netty.netty-resolver-dns-4.1.50.Final.jar:lib/org.yaml.snakeyaml-1.26.jar:lib/io.fabric8.kubernetes-model-events-4.13.0.jar:lib/com.fasterxml.jackson.core.jackson-annotations-2.10.5.jar:lib/org.lz4.lz4-java-1.7.1.jar:lib/io.netty.netty-resolver-4.1.50.Final.jar:lib/io.fabric8.kubernetes-model-discovery-4.13.0.jar:lib/org.apache.logging.log4j.log4j-slf4j-impl-2.13.3.jar:lib/dk.brics.automaton.automaton-1.11-8.jar:lib/io.netty.netty-codec-4.1.50.Final.jar:lib/com.fasterxml.jackson.core.jackson-databind-2.10.5.1.jar:lib/io.netty.netty-codec-socks-4.1.50.Final.jar:lib/io.fabric8.kubernetes-model-autoscaling-4.13.0.jar:lib/io.fabric8.kubernetes-model-apiextensions-4.13.0.jar:lib/io.fabric8.kubernetes-model-common-4.13.0.jar:lib/io.fabric8.openshift-client-4.13.0.jar:lib/jakarta.xml.bind.jakarta.xml.bind-api-2.3.2.jar:lib/io.fabric8.zjsonpatch-0.3.0.jar:lib/com.squareup.okhttp3.okhttp-3.12.6.jar:lib/io.fabric8.kubernetes-model-rbac-4.13.0.jar:lib/io.fabric8.kubernetes-model-policy-4.13.0.jar:lib/io.fabric8.kubernetes-model-core-4.13.0.jar:lib/io.netty.netty-handler-4.1.50.Final.jar:lib/io.fabric8.openshift-model-4.13.0.jar:lib/com.fasterxml.jackson.module.jackson-module-jaxb-annotations-2.10.5.jar:lib/org.slf4j.slf4j-api-1.7.25.jar:lib/org.latencyutils.LatencyUtils-2.0.3.jar:lib/io.fabric8.kubernetes-model-coordination-4.13.0.jar:lib/io.fabric8.openshift-model-operatorhub-4.13.0.jar:lib/org.apache.zookeeper.zookeeper-jute-3.5.8.jar:lib/io.strimzi.certificate-manager-0.21.0-SNAPSHOT.jar:lib/io.fabric8.kubernetes-model-scheduling-4.13.0.jar:lib/com.fasterxml.jackson.datatype.jackson-datatype-jsr310-2.11.2.jar:lib/io.netty.netty-buffer-4.1.50.Final.jar:lib/org.apache.logging.log4j.log4j-core-2.13.3.jar:lib/io.fabric8.kubernetes-model-extensions-4.13.0.jar:lib/io.netty.netty-codec-http-4.1.50.Final.jar:lib/io.micrometer.micrometer-registry-prometheus-1.3.1.jar:lib/io.fabric8.kubernetes-model-batch-4.13.0.jar 2021-01-08 08:25:40,63257 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib 2021-01-08 08:25:40,63282 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:java.io.tmpdir=/tmp 2021-01-08 08:25:40,63306 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:java.compiler= 2021-01-08 08:25:40,63330 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:os.name=Linux 2021-01-08 08:25:40,63356 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:os.arch=amd64 2021-01-08 08:25:40,63380 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:os.version=3.10.0-954.el7.x86_64 2021-01-08 08:25:40,63405 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:user.name=? 2021-01-08 08:25:40,63428 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:user.home=? 2021-01-08 08:25:40,63452 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:user.dir=/opt/strimzi 2021-01-08 08:25:40,63479 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:os.memory.free=227MB 2021-01-08 08:25:40,63505 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:os.memory.max=3855MB 2021-01-08 08:25:40,63529 INFO [vert.x-worker-thread-0] ZooKeeper:109 - Client environment:os.memory.total=241MB 2021-01-08 08:25:40,63902 INFO [vert.x-worker-thread-0] ZooKeeper:868 - Initiating client connection, connectString=localhost:2181 sessionTimeout=20000 watcher=org.I0Itec.zkclient.ZkClient@235c415b 2021-01-08 08:25:40,64499 INFO [vert.x-worker-thread-0] X509Util:79 - Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation 2021-01-08 08:25:40,65031 INFO [vert.x-worker-thread-0] ClientCnxnSocket:237 - jute.maxbuffer value is 4194304 Bytes 2021-01-08 08:25:40,65998 INFO [vert.x-worker-thread-0] ClientCnxn:1653 - zookeeper.request.timeout value is 0. feature enabled= 2021-01-08 08:25:40,66172 DEBUG [vert.x-worker-thread-0] ZkClient:1230 - Awaiting connection to Zookeeper server 2021-01-08 08:25:40,66278 INFO [vert.x-worker-thread-0] ZkClient:936 - Waiting for keeper state SyncConnected 2021-01-08 08:25:40,66869 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] SaslServerPrincipal:80 - Canonicalized address to localhost 2021-01-08 08:25:40,67121 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1112 - Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) 2021-01-08 08:25:40,67270 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1244 - Socket error occurred: localhost/127.0.0.1:2181: Connection refused 2021-01-08 08:25:40,67314 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxnSocketNIO:201 - Ignoring exception during shutdown input java.nio.channels.ClosedChannelException: null at sun.nio.ch.SocketChannelImpl.ensureOpen(SocketChannelImpl.java:160) ~[?:?] at sun.nio.ch.SocketChannelImpl.shutdownInput(SocketChannelImpl.java:915) ~[?:?] at sun.nio.ch.SocketAdaptor.shutdownInput(SocketAdaptor.java:398) ~[?:?] at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:198) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1276) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1254) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] 2021-01-08 08:25:40,67388 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxnSocketNIO:208 - Ignoring exception during shutdown output java.nio.channels.ClosedChannelException: null at sun.nio.ch.SocketChannelImpl.ensureOpen(SocketChannelImpl.java:160) ~[?:?] at sun.nio.ch.SocketChannelImpl.shutdownOutput(SocketChannelImpl.java:932) ~[?:?] at sun.nio.ch.SocketAdaptor.shutdownOutput(SocketAdaptor.java:406) ~[?:?] at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:205) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1276) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1254) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] 2021-01-08 08:25:40,98487 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:955 - [AdminClient clientId=adminclient-1] Completed connection to node -1. Fetching API versions. 2021-01-08 08:25:41,20038 DEBUG [kafka-admin-client-thread | adminclient-1] SslTransportLayer:452 - [SslTransportLayer channelId=-1 key=channel=java.nio.channels.SocketChannel[connection-pending remote=my-cluster-kafka-bootstrap/172.30.93.106:9091], selector=sun.nio.ch.EPollSelectorImpl@4947b966, interestOps=8, readyOps=0] SSL handshake completed successfully with peerHost 'my-cluster-kafka-bootstrap' peerPort 9091 peerPrincipal 'CN=my-cluster-kafka, O=io.strimzi' cipherSuite 'TLS_AES_128_GCM_SHA256' 2021-01-08 08:25:41,20297 DEBUG [kafka-admin-client-thread | adminclient-1] Selector:560 - [AdminClient clientId=adminclient-1] Successfully authenticated with my-cluster-kafka-bootstrap/172.30.93.106 2021-01-08 08:25:41,20339 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:969 - [AdminClient clientId=adminclient-1] Initiating API versions fetch from node -1. 2021-01-08 08:25:41,22220 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending API_VERSIONS request with header RequestHeader(apiKey=API_VERSIONS, apiVersion=3, clientId=adminclient-1, correlationId=0) and timeout 3600000 to node -1: {client_software_name=apache-kafka-java,client_software_version=2.7.0,_tagged_fields={}} 2021-01-08 08:25:41,33026 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received API_VERSIONS response from node -1 for request with header RequestHeader(apiKey=API_VERSIONS, apiVersion=3, clientId=adminclient-1, correlationId=0): org.apache.kafka.common.requests.ApiVersionsResponse@1ffd6f 2021-01-08 08:25:41,33403 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:926 - [AdminClient clientId=adminclient-1] Recorded API versions for node -1: (Produce(0): 0 to 8 [usable: 8], Fetch(1): 0 to 12 [usable: 12], ListOffsets(2): 0 to 5 [usable: 5], Metadata(3): 0 to 9 [usable: 9], LeaderAndIsr(4): 0 to 4 [usable: 4], StopReplica(5): 0 to 3 [usable: 3], UpdateMetadata(6): 0 to 6 [usable: 6], ControlledShutdown(7): 0 to 3 [usable: 3], OffsetCommit(8): 0 to 8 [usable: 8], OffsetFetch(9): 0 to 7 [usable: 7], FindCoordinator(10): 0 to 3 [usable: 3], JoinGroup(11): 0 to 7 [usable: 7], Heartbeat(12): 0 to 4 [usable: 4], LeaveGroup(13): 0 to 4 [usable: 4], SyncGroup(14): 0 to 5 [usable: 5], DescribeGroups(15): 0 to 5 [usable: 5], ListGroups(16): 0 to 4 [usable: 4], SaslHandshake(17): 0 to 1 [usable: 1], ApiVersions(18): 0 to 3 [usable: 3], CreateTopics(19): 0 to 6 [usable: 6], DeleteTopics(20): 0 to 5 [usable: 5], DeleteRecords(21): 0 to 2 [usable: 2], InitProducerId(22): 0 to 4 [usable: 4], OffsetForLeaderEpoch(23): 0 to 3 [usable: 3], AddPartitionsToTxn(24): 0 to 2 [usable: 2], AddOffsetsToTxn(25): 0 to 2 [usable: 2], EndTxn(26): 0 to 2 [usable: 2], WriteTxnMarkers(27): 0 [usable: 0], TxnOffsetCommit(28): 0 to 3 [usable: 3], DescribeAcls(29): 0 to 2 [usable: 2], CreateAcls(30): 0 to 2 [usable: 2], DeleteAcls(31): 0 to 2 [usable: 2], DescribeConfigs(32): 0 to 3 [usable: 3], AlterConfigs(33): 0 to 1 [usable: 1], AlterReplicaLogDirs(34): 0 to 1 [usable: 1], DescribeLogDirs(35): 0 to 2 [usable: 2], SaslAuthenticate(36): 0 to 2 [usable: 2], CreatePartitions(37): 0 to 3 [usable: 3], CreateDelegationToken(38): 0 to 2 [usable: 2], RenewDelegationToken(39): 0 to 2 [usable: 2], ExpireDelegationToken(40): 0 to 2 [usable: 2], DescribeDelegationToken(41): 0 to 2 [usable: 2], DeleteGroups(42): 0 to 2 [usable: 2], ElectLeaders(43): 0 to 2 [usable: 2], IncrementalAlterConfigs(44): 0 to 1 [usable: 1], AlterPartitionReassignments(45): 0 [usable: 0], ListPartitionReassignments(46): 0 [usable: 0], OffsetDelete(47): 0 [usable: 0], DescribeClientQuotas(48): 0 [usable: 0], AlterClientQuotas(49): 0 [usable: 0], DescribeUserScramCredentials(50): 0 [usable: 0], AlterUserScramCredentials(51): 0 [usable: 0], AlterIsr(56): 0 [usable: 0], UpdateFeatures(57): 0 [usable: 0]) 2021-01-08 08:25:41,33605 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[], allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-bootstrap:9091 (id: -1 rack: null). correlationId=1 2021-01-08 08:25:41,33672 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=1) and timeout 29069 to node -1: {topics=[],allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:25:41,34315 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node -1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=1): org.apache.kafka.common.requests.MetadataResponse@a269a44 2021-01-08 08:25:41,34734 DEBUG [kafka-admin-client-thread | adminclient-1] AdminMetadataManager:247 - [AdminClient clientId=adminclient-1] Updating cluster metadata to Cluster(id = 24eqSYNVTzaDk8st3RVFrQ, nodes = [my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null), my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null), my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null)], partitions = [], controller = my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null)) 2021-01-08 08:25:41,77841 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] SaslServerPrincipal:80 - Canonicalized address to localhost 2021-01-08 08:25:41,77890 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1112 - Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2181. Will not attempt to authenticate using SASL (unknown error) 2021-01-08 08:25:41,77996 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1244 - Socket error occurred: localhost/0:0:0:0:0:0:0:1:2181: Connection refused 2021-01-08 08:25:41,78025 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxnSocketNIO:201 - Ignoring exception during shutdown input java.nio.channels.ClosedChannelException: null at sun.nio.ch.SocketChannelImpl.ensureOpen(SocketChannelImpl.java:160) ~[?:?] at sun.nio.ch.SocketChannelImpl.shutdownInput(SocketChannelImpl.java:915) ~[?:?] at sun.nio.ch.SocketAdaptor.shutdownInput(SocketAdaptor.java:398) ~[?:?] at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:198) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1276) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1254) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] 2021-01-08 08:25:41,78137 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxnSocketNIO:208 - Ignoring exception during shutdown output java.nio.channels.ClosedChannelException: null at sun.nio.ch.SocketChannelImpl.ensureOpen(SocketChannelImpl.java:160) ~[?:?] at sun.nio.ch.SocketChannelImpl.shutdownOutput(SocketChannelImpl.java:932) ~[?:?] at sun.nio.ch.SocketAdaptor.shutdownOutput(SocketAdaptor.java:406) ~[?:?] at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:205) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1276) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1254) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] 2021-01-08 08:25:42,88272 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] SaslServerPrincipal:80 - Canonicalized address to localhost 2021-01-08 08:25:42,88317 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1112 - Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) 2021-01-08 08:25:42,88422 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1244 - Socket error occurred: localhost/127.0.0.1:2181: Connection refused 2021-01-08 08:25:42,88461 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxnSocketNIO:201 - Ignoring exception during shutdown input java.nio.channels.ClosedChannelException: null at sun.nio.ch.SocketChannelImpl.ensureOpen(SocketChannelImpl.java:160) ~[?:?] at sun.nio.ch.SocketChannelImpl.shutdownInput(SocketChannelImpl.java:915) ~[?:?] at sun.nio.ch.SocketAdaptor.shutdownInput(SocketAdaptor.java:398) ~[?:?] at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:198) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1276) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1254) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] 2021-01-08 08:25:42,88528 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxnSocketNIO:208 - Ignoring exception during shutdown output java.nio.channels.ClosedChannelException: null at sun.nio.ch.SocketChannelImpl.ensureOpen(SocketChannelImpl.java:160) ~[?:?] at sun.nio.ch.SocketChannelImpl.shutdownOutput(SocketChannelImpl.java:932) ~[?:?] at sun.nio.ch.SocketAdaptor.shutdownOutput(SocketAdaptor.java:406) ~[?:?] at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:205) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1276) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1254) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] 2021-01-08 08:25:43,98665 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] SaslServerPrincipal:80 - Canonicalized address to localhost 2021-01-08 08:25:43,98712 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1112 - Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2181. Will not attempt to authenticate using SASL (unknown error) 2021-01-08 08:25:43,98818 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1244 - Socket error occurred: localhost/0:0:0:0:0:0:0:1:2181: Connection refused 2021-01-08 08:25:43,98846 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxnSocketNIO:201 - Ignoring exception during shutdown input java.nio.channels.ClosedChannelException: null at sun.nio.ch.SocketChannelImpl.ensureOpen(SocketChannelImpl.java:160) ~[?:?] at sun.nio.ch.SocketChannelImpl.shutdownInput(SocketChannelImpl.java:915) ~[?:?] at sun.nio.ch.SocketAdaptor.shutdownInput(SocketAdaptor.java:398) ~[?:?] at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:198) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1276) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1254) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] 2021-01-08 08:25:43,98902 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxnSocketNIO:208 - Ignoring exception during shutdown output java.nio.channels.ClosedChannelException: null at sun.nio.ch.SocketChannelImpl.ensureOpen(SocketChannelImpl.java:160) ~[?:?] at sun.nio.ch.SocketChannelImpl.shutdownOutput(SocketChannelImpl.java:932) ~[?:?] at sun.nio.ch.SocketAdaptor.shutdownOutput(SocketAdaptor.java:406) ~[?:?] at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:205) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1276) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1254) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] 2021-01-08 08:25:45,09034 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] SaslServerPrincipal:80 - Canonicalized address to localhost 2021-01-08 08:25:45,09082 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1112 - Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2181. Will not attempt to authenticate using SASL (unknown error) 2021-01-08 08:25:45,09174 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1244 - Socket error occurred: localhost/0:0:0:0:0:0:0:1:2181: Connection refused 2021-01-08 08:25:45,09200 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxnSocketNIO:201 - Ignoring exception during shutdown input java.nio.channels.ClosedChannelException: null at sun.nio.ch.SocketChannelImpl.ensureOpen(SocketChannelImpl.java:160) ~[?:?] at sun.nio.ch.SocketChannelImpl.shutdownInput(SocketChannelImpl.java:915) ~[?:?] at sun.nio.ch.SocketAdaptor.shutdownInput(SocketAdaptor.java:398) ~[?:?] at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:198) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1276) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1254) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] 2021-01-08 08:25:45,09261 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxnSocketNIO:208 - Ignoring exception during shutdown output java.nio.channels.ClosedChannelException: null at sun.nio.ch.SocketChannelImpl.ensureOpen(SocketChannelImpl.java:160) ~[?:?] at sun.nio.ch.SocketChannelImpl.shutdownOutput(SocketChannelImpl.java:932) ~[?:?] at sun.nio.ch.SocketAdaptor.shutdownOutput(SocketAdaptor.java:406) ~[?:?] at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:205) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1276) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1254) [org.apache.zookeeper.zookeeper-3.5.8.jar:3.5.8] 2021-01-08 08:25:46,19396 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] SaslServerPrincipal:80 - Canonicalized address to localhost 2021-01-08 08:25:46,19438 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1112 - Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) 2021-01-08 08:25:46,19550 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:959 - Socket connection established, initiating session, client: /127.0.0.1:35554, server: localhost/127.0.0.1:2181 2021-01-08 08:25:46,19860 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1027 - Session establishment request sent on localhost/127.0.0.1:2181 2021-01-08 08:25:46,26897 INFO [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:1394 - Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x101a07e36350011, negotiated timeout = 20000 2021-01-08 08:25:46,27137 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:None path:null 2021-01-08 08:25:46,27215 INFO [vert.x-worker-thread-0-EventThread] ZkClient:713 - zookeeper state changed (SyncConnected) 2021-01-08 08:25:46,27248 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:25:46,27275 DEBUG [vert.x-worker-thread-0] ZkClient:950 - State is SyncConnected 2021-01-08 08:25:46,27797 DEBUG [vert.x-eventloop-thread-0] Session:174 - Using ZooKeeper io.strimzi.operator.topic.zk.ZkImpl@42d7efec 2021-01-08 08:25:46,28640 DEBUG [vert.x-eventloop-thread-0] Session:179 - Using TopicStore io.strimzi.operator.topic.ZkTopicStore@6a9c5772 2021-01-08 08:25:46,30042 DEBUG [vert.x-eventloop-thread-0] Session:182 - Using Operator io.strimzi.operator.topic.TopicOperator@28f01e73 2021-01-08 08:25:46,30282 DEBUG [vert.x-eventloop-thread-0] Session:185 - Using TopicConfigsWatcher io.strimzi.operator.topic.TopicConfigsWatcher@60b60078 2021-01-08 08:25:46,30371 DEBUG [vert.x-eventloop-thread-0] Session:187 - Using TopicWatcher io.strimzi.operator.topic.ZkTopicWatcher@3a8b5500 2021-01-08 08:25:46,30456 DEBUG [vert.x-eventloop-thread-0] Session:189 - Using TopicsWatcher io.strimzi.operator.topic.ZkTopicsWatcher@5c8ac3f8 2021-01-08 08:25:46,30542 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 1,1 replyHeader:: 1,4294967568,0 request:: '/strimzi,,v{s{15,s{'world,'anyone}}},0 response:: '/strimzi 2021-01-08 08:25:46,30963 DEBUG [vert.x-eventloop-thread-0] Session:195 - Starting watcher 2021-01-08 08:25:46,31010 DEBUG [vert.x-eventloop-thread-0] Session:230 - Watching KafkaTopics matching {strimzi.io/cluster=my-cluster} 2021-01-08 08:25:46,31507 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 2,3 replyHeader:: 2,4294967568,0 request:: '/brokers/topics,T response:: s{4294967304,4294967304,1610094013731,1610094013731,0,3,0,0,0,3,4294967441} 2021-01-08 08:25:46,31991 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 3,8 replyHeader:: 3,4294967568,0 request:: '/brokers/topics,T response:: v{'my-topic,'__consumer_offsets,'my-topic-reversed} 2021-01-08 08:25:46,58335 DEBUG [vert.x-eventloop-thread-0] WatchConnectionManager:111 - Connecting websocket ... io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager@73688495 2021-01-08 08:25:46,71433 DEBUG [OkHttp https://172.30.0.1/...] WatchConnectionManager:162 - WebSocket successfully opened 2021-01-08 08:25:46,71562 DEBUG [vert.x-eventloop-thread-0] Session:234 - Watching setup 2021-01-08 08:25:46,71678 DEBUG [vert.x-eventloop-thread-0] Session:198 - Starting health server 2021-01-08 08:25:46,88360 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:156 - -Dio.netty.allocator.numHeapArenas: 2 2021-01-08 08:25:46,88399 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:157 - -Dio.netty.allocator.numDirectArenas: 2 2021-01-08 08:25:46,88424 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:159 - -Dio.netty.allocator.pageSize: 8192 2021-01-08 08:25:46,88448 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:164 - -Dio.netty.allocator.maxOrder: 11 2021-01-08 08:25:46,88474 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:168 - -Dio.netty.allocator.chunkSize: 16777216 2021-01-08 08:25:46,88498 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:169 - -Dio.netty.allocator.tinyCacheSize: 512 2021-01-08 08:25:46,88522 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:170 - -Dio.netty.allocator.smallCacheSize: 256 2021-01-08 08:25:46,88552 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:171 - -Dio.netty.allocator.normalCacheSize: 64 2021-01-08 08:25:46,88579 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:172 - -Dio.netty.allocator.maxCachedBufferCapacity: 32768 2021-01-08 08:25:46,88604 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:173 - -Dio.netty.allocator.cacheTrimInterval: 8192 2021-01-08 08:25:46,88632 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:174 - -Dio.netty.allocator.cacheTrimIntervalMillis: 0 2021-01-08 08:25:46,88706 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:175 - -Dio.netty.allocator.useCacheForAllThreads: true 2021-01-08 08:25:46,88737 DEBUG [vert.x-eventloop-thread-0] PooledByteBufAllocator:176 - -Dio.netty.allocator.maxCachedByteBuffersPerChunk: 1023 2021-01-08 08:25:46,99288 DEBUG [vert.x-eventloop-thread-0] DefaultChannelId:79 - -Dio.netty.processId: 27 (auto-detected) 2021-01-08 08:25:46,99606 DEBUG [vert.x-eventloop-thread-0] DefaultChannelId:101 - -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:10 (auto-detected) 2021-01-08 08:25:47,01708 DEBUG [OkHttp https://172.30.0.1/...] K8sTopicWatcher:42 - Ignoring initial event for KafkaTopic my-topic during initial reconcile 2021-01-08 08:25:47,01943 DEBUG [OkHttp https://172.30.0.1/...] K8sTopicWatcher:42 - Ignoring initial event for KafkaTopic my-topic-reversed during initial reconcile 2021-01-08 08:25:47,02135 DEBUG [OkHttp https://172.30.0.1/...] K8sTopicWatcher:42 - Ignoring initial event for KafkaTopic consumer-offsets during initial reconcile 2021-01-08 08:25:47,03993 DEBUG [vert.x-eventloop-thread-0] ByteBufUtil:86 - -Dio.netty.allocator.type: pooled 2021-01-08 08:25:47,04050 DEBUG [vert.x-eventloop-thread-0] ByteBufUtil:95 - -Dio.netty.threadLocalDirectBufferSize: 0 2021-01-08 08:25:47,04082 DEBUG [vert.x-eventloop-thread-0] ByteBufUtil:98 - -Dio.netty.maxThreadLocalCharBufferSize: 16384 2021-01-08 08:25:47,06501 INFO [vert.x-eventloop-thread-0] TopicOperator:1267 - Starting initial reconciliation 2021-01-08 08:25:47,06460 INFO [vert.x-eventloop-thread-1] Main:54 - Session deployed 2021-01-08 08:25:47,06548 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:127 - Listing topics 2021-01-08 08:25:47,06767 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610094407066, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:25:47,07496 INFO [vert.x-eventloop-thread-0] Session:223 - Started 2021-01-08 08:25:47,07602 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:1003 - [AdminClient clientId=adminclient-1] Initiating connection to node my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null) using address my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc/172.17.0.15 2021-01-08 08:25:47,08293 DEBUG [kafka-admin-client-thread | adminclient-1] Selector:531 - [AdminClient clientId=adminclient-1] Created socket with SO_RCVBUF = 65536, SO_SNDBUF = 131072, SO_TIMEOUT = 0 to node 1 2021-01-08 08:25:47,09681 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 4,1 replyHeader:: 4,4294967571,0 request:: '/strimzi/topics,,v{s{15,s{'world,'anyone}}},0 response:: '/strimzi/topics 2021-01-08 08:25:47,10386 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 5,8 replyHeader:: 5,4294967571,0 request:: '/brokers/topics,T response:: v{'my-topic,'__consumer_offsets,'my-topic-reversed} 2021-01-08 08:25:47,10771 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:955 - [AdminClient clientId=adminclient-1] Completed connection to node 1. Fetching API versions. 2021-01-08 08:25:47,11022 DEBUG [vert.x-eventloop-thread-0] ZkTopicsWatcher:73 - Setting initial children [my-topic, __consumer_offsets, my-topic-reversed] 2021-01-08 08:25:47,11199 DEBUG [vert.x-eventloop-thread-0] TopicConfigsWatcher:73 - Watching znode /config/topics/my-topic for changes 2021-01-08 08:25:47,11526 DEBUG [vert.x-eventloop-thread-0] ZkTopicWatcher:73 - Watching znode /brokers/topics/my-topic for changes 2021-01-08 08:25:47,11594 DEBUG [vert.x-eventloop-thread-0] TopicConfigsWatcher:73 - Watching znode /config/topics/__consumer_offsets for changes 2021-01-08 08:25:47,11635 DEBUG [vert.x-eventloop-thread-0] ZkTopicWatcher:73 - Watching znode /brokers/topics/__consumer_offsets for changes 2021-01-08 08:25:47,11678 DEBUG [vert.x-eventloop-thread-0] TopicConfigsWatcher:73 - Watching znode /config/topics/my-topic-reversed for changes 2021-01-08 08:25:47,11714 DEBUG [vert.x-eventloop-thread-0] ZkTopicWatcher:73 - Watching znode /brokers/topics/my-topic-reversed for changes 2021-01-08 08:25:47,12291 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 6,3 replyHeader:: 6,4294967571,0 request:: '/config/topics/my-topic,T response:: s{4294967380,4294967380,1610094075626,1610094075626,0,0,0,0,25,0,4294967380} 2021-01-08 08:25:47,12347 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-2] ZkClient:196 - Subscribed data changes for /config/topics/my-topic 2021-01-08 08:25:47,13027 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 7,3 replyHeader:: 7,4294967571,0 request:: '/brokers/topics/my-topic,T response:: s{4294967381,4294967381,1610094075681,1610094075681,0,1,0,0,218,1,4294967382} 2021-01-08 08:25:47,13087 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-2] ZkClient:196 - Subscribed data changes for /brokers/topics/my-topic 2021-01-08 08:25:47,13773 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 8,3 replyHeader:: 8,4294967571,0 request:: '/config/topics/__consumer_offsets,T response:: s{4294967440,4294967440,1610094086370,1610094086370,0,0,0,0,109,0,4294967440} 2021-01-08 08:25:47,13837 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-2] ZkClient:196 - Subscribed data changes for /config/topics/__consumer_offsets 2021-01-08 08:25:47,14269 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 9,3 replyHeader:: 9,4294967571,0 request:: '/brokers/topics/__consumer_offsets,T response:: s{4294967441,4294967441,1610094086394,1610094086394,0,1,0,0,712,1,4294967442} 2021-01-08 08:25:47,14318 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-2] ZkClient:196 - Subscribed data changes for /brokers/topics/__consumer_offsets 2021-01-08 08:25:47,14768 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 10,3 replyHeader:: 10,4294967571,0 request:: '/config/topics/my-topic-reversed,T response:: s{4294967409,4294967409,1610094077466,1610094077466,0,0,0,0,25,0,4294967409} 2021-01-08 08:25:47,14810 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-2] ZkClient:196 - Subscribed data changes for /config/topics/my-topic-reversed 2021-01-08 08:25:47,15205 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 11,3 replyHeader:: 11,4294967571,0 request:: '/brokers/topics/my-topic-reversed,T response:: s{4294967410,4294967410,1610094077480,1610094077480,0,1,0,0,218,1,4294967411} 2021-01-08 08:25:47,15247 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-2] ZkClient:196 - Subscribed data changes for /brokers/topics/my-topic-reversed 2021-01-08 08:25:47,15910 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 12,4 replyHeader:: 12,4294967571,0 request:: '/config/topics/my-topic,T response:: #7b2276657273696f6e223a312c22636f6e666967223a7b7d7d,s{4294967380,4294967380,1610094075626,1610094075626,0,0,0,0,25,0,4294967380} 2021-01-08 08:25:47,16458 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 13,4 replyHeader:: 13,4294967571,0 request:: '/brokers/topics/my-topic,T response:: #7b2276657273696f6e223a322c22706172746974696f6e73223a7b2232223a5b322c302c315d2c2235223a5b322c312c305d2c2238223a5b322c302c315d2c2237223a5b302c312c325d2c2231223a5b302c312c325d2c2234223a5b302c322c315d2c223131223a5b322c312c305d2c2236223a5b312c322c305d2c2230223a5b312c322c305d2c2239223a5b312c302c325d2c2233223a5b312c302c325d2c223130223a5b302c322c315d7d2c22616464696e675f7265706c69636173223a7b7d2c2272656d6f76696e675f7265706c69636173223a7b7d7d,s{4294967381,4294967381,1610094075681,1610094075681,0,1,0,0,218,1,4294967382} 2021-01-08 08:25:47,17179 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 14,4 replyHeader:: 14,4294967571,0 request:: '/config/topics/__consumer_offsets,T response:: #7b2276657273696f6e223a312c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294967440,4294967440,1610094086370,1610094086370,0,0,0,0,109,0,4294967440} 2021-01-08 08:25:47,17761 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 15,4 replyHeader:: 15,4294967571,0 request:: '/brokers/topics/__consumer_offsets,T response:: #7b2276657273696f6e223a322c22706172746974696f6e73223a7b223330223a5b322c312c305d2c223339223a5b322c302c315d2c223435223a5b322c302c315d2c2232223a5b312c302c325d2c2235223a5b312c322c305d2c223438223a5b322c312c305d2c223333223a5b322c302c315d2c223237223a5b322c302c315d2c223132223a5b322c312c305d2c2238223a5b312c302c325d2c223135223a5b322c302c315d2c223432223a5b322c312c305d2c223336223a5b322c312c305d2c223231223a5b322c302c315d2c223138223a5b322c312c305d2c223234223a5b322c312c305d2c223335223a5b312c322c305d2c223431223a5b312c322c305d2c2237223a5b302c322c315d2c223137223a5b312c322c305d2c2231223a5b302c322c315d2c223434223a5b312c302c325d2c223233223a5b312c322c305d2c223338223a5b312c302c325d2c223437223a5b312c322c305d2c2234223a5b302c312c325d2c223236223a5b312c302c325d2c223131223a5b312c322c305d2c223332223a5b312c302c325d2c223134223a5b312c302c325d2c223230223a5b312c302c325d2c223239223a5b312c322c305d2c223436223a5b302c312c325d2c223334223a5b302c312c325d2c223238223a5b302c312c325d2c2236223a5b322c312c305d2c223430223a5b302c312c325d2c223439223a5b302c322c315d2c2239223a5b322c302c315d2c223433223a5b302c322c315d2c2230223a5b322c312c305d2c223232223a5b302c312c325d2c223136223a5b302c312c325d2c223337223a5b302c322c315d2c223139223a5b302c322c315d2c2233223a5b322c302c315d2c223130223a5b302c312c325d2c223331223a5b302c322c315d2c223235223a5b302c322c315d2c223133223a5b302c322c315d7d2c22616464696e675f7265706c69636173223a7b7d2c2272656d6f76696e675f7265706c69636173223a7b7d7d,s{4294967441,4294967441,1610094086394,1610094086394,0,1,0,0,712,1,4294967442} 2021-01-08 08:25:47,18251 DEBUG [kafka-admin-client-thread | adminclient-1] SslTransportLayer:452 - [SslTransportLayer channelId=1 key=channel=java.nio.channels.SocketChannel[connection-pending remote=my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc/172.17.0.15:9091], selector=sun.nio.ch.EPollSelectorImpl@4947b966, interestOps=8, readyOps=0] SSL handshake completed successfully with peerHost 'my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc' peerPort 9091 peerPrincipal 'CN=my-cluster-kafka, O=io.strimzi' cipherSuite 'TLS_AES_128_GCM_SHA256' 2021-01-08 08:25:47,18302 DEBUG [kafka-admin-client-thread | adminclient-1] Selector:560 - [AdminClient clientId=adminclient-1] Successfully authenticated with my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc/172.17.0.15 2021-01-08 08:25:47,18331 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:969 - [AdminClient clientId=adminclient-1] Initiating API versions fetch from node 1. 2021-01-08 08:25:47,18365 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending API_VERSIONS request with header RequestHeader(apiKey=API_VERSIONS, apiVersion=3, clientId=adminclient-1, correlationId=2) and timeout 3600000 to node 1: {client_software_name=apache-kafka-java,client_software_version=2.7.0,_tagged_fields={}} 2021-01-08 08:25:47,18480 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 16,4 replyHeader:: 16,4294967571,0 request:: '/config/topics/my-topic-reversed,T response:: #7b2276657273696f6e223a312c22636f6e666967223a7b7d7d,s{4294967409,4294967409,1610094077466,1610094077466,0,0,0,0,25,0,4294967409} 2021-01-08 08:25:47,18788 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received API_VERSIONS response from node 1 for request with header RequestHeader(apiKey=API_VERSIONS, apiVersion=3, clientId=adminclient-1, correlationId=2): org.apache.kafka.common.requests.ApiVersionsResponse@f13088a 2021-01-08 08:25:47,18910 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:926 - [AdminClient clientId=adminclient-1] Recorded API versions for node 1: (Produce(0): 0 to 8 [usable: 8], Fetch(1): 0 to 12 [usable: 12], ListOffsets(2): 0 to 5 [usable: 5], Metadata(3): 0 to 9 [usable: 9], LeaderAndIsr(4): 0 to 4 [usable: 4], StopReplica(5): 0 to 3 [usable: 3], UpdateMetadata(6): 0 to 6 [usable: 6], ControlledShutdown(7): 0 to 3 [usable: 3], OffsetCommit(8): 0 to 8 [usable: 8], OffsetFetch(9): 0 to 7 [usable: 7], FindCoordinator(10): 0 to 3 [usable: 3], JoinGroup(11): 0 to 7 [usable: 7], Heartbeat(12): 0 to 4 [usable: 4], LeaveGroup(13): 0 to 4 [usable: 4], SyncGroup(14): 0 to 5 [usable: 5], DescribeGroups(15): 0 to 5 [usable: 5], ListGroups(16): 0 to 4 [usable: 4], SaslHandshake(17): 0 to 1 [usable: 1], ApiVersions(18): 0 to 3 [usable: 3], CreateTopics(19): 0 to 6 [usable: 6], DeleteTopics(20): 0 to 5 [usable: 5], DeleteRecords(21): 0 to 2 [usable: 2], InitProducerId(22): 0 to 4 [usable: 4], OffsetForLeaderEpoch(23): 0 to 3 [usable: 3], AddPartitionsToTxn(24): 0 to 2 [usable: 2], AddOffsetsToTxn(25): 0 to 2 [usable: 2], EndTxn(26): 0 to 2 [usable: 2], WriteTxnMarkers(27): 0 [usable: 0], TxnOffsetCommit(28): 0 to 3 [usable: 3], DescribeAcls(29): 0 to 2 [usable: 2], CreateAcls(30): 0 to 2 [usable: 2], DeleteAcls(31): 0 to 2 [usable: 2], DescribeConfigs(32): 0 to 3 [usable: 3], AlterConfigs(33): 0 to 1 [usable: 1], AlterReplicaLogDirs(34): 0 to 1 [usable: 1], DescribeLogDirs(35): 0 to 2 [usable: 2], SaslAuthenticate(36): 0 to 2 [usable: 2], CreatePartitions(37): 0 to 3 [usable: 3], CreateDelegationToken(38): 0 to 2 [usable: 2], RenewDelegationToken(39): 0 to 2 [usable: 2], ExpireDelegationToken(40): 0 to 2 [usable: 2], DescribeDelegationToken(41): 0 to 2 [usable: 2], DeleteGroups(42): 0 to 2 [usable: 2], ElectLeaders(43): 0 to 2 [usable: 2], IncrementalAlterConfigs(44): 0 to 1 [usable: 1], AlterPartitionReassignments(45): 0 [usable: 0], ListPartitionReassignments(46): 0 [usable: 0], OffsetDelete(47): 0 [usable: 0], DescribeClientQuotas(48): 0 [usable: 0], AlterClientQuotas(49): 0 [usable: 0], DescribeUserScramCredentials(50): 0 [usable: 0], AlterUserScramCredentials(51): 0 [usable: 0], AlterIsr(56): 0 [usable: 0], UpdateFeatures(57): 0 [usable: 0]) 2021-01-08 08:25:47,18966 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=3 2021-01-08 08:25:47,19008 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=3) and timeout 30000 to node 1: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:25:47,19398 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 17,4 replyHeader:: 17,4294967571,0 request:: '/brokers/topics/my-topic-reversed,T response:: #7b2276657273696f6e223a322c22706172746974696f6e73223a7b2232223a5b312c322c305d2c2235223a5b312c302c325d2c2238223a5b312c322c305d2c2237223a5b322c302c315d2c2231223a5b322c302c315d2c223131223a5b312c302c325d2c2234223a5b322c312c305d2c2236223a5b302c312c325d2c2230223a5b302c312c325d2c2239223a5b302c322c315d2c223130223a5b322c312c305d2c2233223a5b302c322c315d7d2c22616464696e675f7265706c69636173223a7b7d2c2272656d6f76696e675f7265706c69636173223a7b7d7d,s{4294967410,4294967410,1610094077480,1610094077480,0,1,0,0,218,1,4294967411} 2021-01-08 08:25:47,19938 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=3): org.apache.kafka.common.requests.MetadataResponse@5f00d148 2021-01-08 08:25:47,20609 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:25:47,20907 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 3|initial kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:25:47,20987 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 3|initial kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:25:47,21565 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 4|initial kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:25:47,21595 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 4|initial kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:25:47,21630 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 5|initial kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:25:47,21652 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 5|initial kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:25:47,22330 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 3|initial kafka my-topic-reversed: Lock acquired 2021-01-08 08:25:47,22370 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 3|initial kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:25:47,22907 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 18,4 replyHeader:: 18,4294967571,-101 request:: '/strimzi/topics/my-topic-reversed,F response:: 2021-01-08 08:25:47,22993 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 4|initial kafka my-topic: Lock acquired 2021-01-08 08:25:47,23023 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 4|initial kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:25:47,23077 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 5|initial kafka __consumer_offsets: Lock acquired 2021-01-08 08:25:47,23098 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 5|initial kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:25:47,23529 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1364 - 3|initial kafka my-topic-reversed: No private topic for topic my-topic-reversed in Kafka -> undetermined 2021-01-08 08:25:47,23566 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 3|initial kafka my-topic-reversed: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:25:47,23593 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 3|initial kafka my-topic-reversed: No KafkaTopic to set status 2021-01-08 08:25:47,23730 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 19,4 replyHeader:: 19,4294967571,-101 request:: '/strimzi/topics/my-topic,F response:: 2021-01-08 08:25:47,23764 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 3|initial kafka my-topic-reversed: Lock released 2021-01-08 08:25:47,23796 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 3|initial kafka my-topic-reversed: Removing last waiter reconcile-from-kafka 2021-01-08 08:25:47,23904 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1364 - 4|initial kafka my-topic: No private topic for topic my-topic in Kafka -> undetermined 2021-01-08 08:25:47,23931 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 4|initial kafka my-topic: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:25:47,23952 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 4|initial kafka my-topic: No KafkaTopic to set status 2021-01-08 08:25:47,23984 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 4|initial kafka my-topic: Lock released 2021-01-08 08:25:47,24007 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 4|initial kafka my-topic: Removing last waiter reconcile-from-kafka 2021-01-08 08:25:47,24095 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 20,4 replyHeader:: 20,4294967571,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:25:47,24247 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1364 - 5|initial kafka __consumer_offsets: No private topic for topic __consumer_offsets in Kafka -> undetermined 2021-01-08 08:25:47,24279 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 5|initial kafka __consumer_offsets: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:25:47,24301 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 5|initial kafka __consumer_offsets: No KafkaTopic to set status 2021-01-08 08:25:47,24789 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 5|initial kafka __consumer_offsets: Lock released 2021-01-08 08:25:47,24825 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 5|initial kafka __consumer_offsets: Removing last waiter reconcile-from-kafka 2021-01-08 08:25:47,24952 WARN [kubernetes-ops-tool-0] VersionUsageUtils:60 - The client is using resource type 'kafkatopics' with unstable version 'v1beta1' 2021-01-08 08:25:47,38881 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 6|initial kube consumer-offsets: Concurrent modification in kube: new version 1209467 2021-01-08 08:25:47,39330 DEBUG [vert.x-eventloop-thread-1] TopicOperator:957 - Metric reconcile-with-kube triggered 2021-01-08 08:25:47,39470 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 6|initial kube consumer-offsets|1209467: Queuing action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:25:47,39503 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 6|initial kube consumer-offsets|1209467: Adding first waiter reconcile-with-kube 2021-01-08 08:25:47,39621 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 7|initial kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:25:47,39661 DEBUG [vert.x-eventloop-thread-1] TopicOperator:957 - Metric reconcile-with-kube triggered 2021-01-08 08:25:47,39697 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 7|initial kube my-topic|1208692: Queuing action reconcile-with-kube on topic my-topic 2021-01-08 08:25:47,39722 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 7|initial kube my-topic|1208692: Adding first waiter reconcile-with-kube 2021-01-08 08:25:47,39763 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 8|initial kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:25:47,39792 DEBUG [vert.x-eventloop-thread-1] TopicOperator:957 - Metric reconcile-with-kube triggered 2021-01-08 08:25:47,39820 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 8|initial kube my-topic-reversed|1208693: Queuing action reconcile-with-kube on topic my-topic-reversed 2021-01-08 08:25:47,39847 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 8|initial kube my-topic-reversed|1208693: Adding first waiter reconcile-with-kube 2021-01-08 08:25:47,39963 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 6|initial kube consumer-offsets|1209467: Lock acquired 2021-01-08 08:25:47,39988 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 6|initial kube consumer-offsets|1209467: Executing action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:25:47,40189 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:25:47,40798 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094407406, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:25:47,41076 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:1003 - [AdminClient clientId=adminclient-1] Initiating connection to node my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null) using address my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc/172.17.0.11 2021-01-08 08:25:47,41544 DEBUG [kafka-admin-client-thread | adminclient-1] Selector:531 - [AdminClient clientId=adminclient-1] Created socket with SO_RCVBUF = 65536, SO_SNDBUF = 131072, SO_TIMEOUT = 0 to node 2 2021-01-08 08:25:47,41582 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 7|initial kube my-topic|1208692: Lock acquired 2021-01-08 08:25:47,41611 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 7|initial kube my-topic|1208692: Executing action reconcile-with-kube on topic my-topic 2021-01-08 08:25:47,41652 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 21,4 replyHeader:: 21,4294967571,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:25:47,41676 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:25:47,41712 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094407416, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:25:47,41760 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 8|initial kube my-topic-reversed|1208693: Lock acquired 2021-01-08 08:25:47,41781 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 8|initial kube my-topic-reversed|1208693: Executing action reconcile-with-kube on topic my-topic-reversed 2021-01-08 08:25:47,41831 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:25:47,41863 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094407418, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:25:47,42659 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:955 - [AdminClient clientId=adminclient-1] Completed connection to node 2. Fetching API versions. 2021-01-08 08:25:47,43215 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 22,4 replyHeader:: 22,4294967571,-101 request:: '/strimzi/topics/my-topic,F response:: 2021-01-08 08:25:47,45144 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 23,4 replyHeader:: 23,4294967571,-101 request:: '/strimzi/topics/my-topic-reversed,F response:: 2021-01-08 08:25:47,50939 DEBUG [kafka-admin-client-thread | adminclient-1] SslTransportLayer:452 - [SslTransportLayer channelId=2 key=channel=java.nio.channels.SocketChannel[connection-pending remote=my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc/172.17.0.11:9091], selector=sun.nio.ch.EPollSelectorImpl@4947b966, interestOps=8, readyOps=0] SSL handshake completed successfully with peerHost 'my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc' peerPort 9091 peerPrincipal 'CN=my-cluster-kafka, O=io.strimzi' cipherSuite 'TLS_AES_128_GCM_SHA256' 2021-01-08 08:25:47,50984 DEBUG [kafka-admin-client-thread | adminclient-1] Selector:560 - [AdminClient clientId=adminclient-1] Successfully authenticated with my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc/172.17.0.11 2021-01-08 08:25:47,51010 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:969 - [AdminClient clientId=adminclient-1] Initiating API versions fetch from node 2. 2021-01-08 08:25:47,51039 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending API_VERSIONS request with header RequestHeader(apiKey=API_VERSIONS, apiVersion=3, clientId=adminclient-1, correlationId=4) and timeout 3600000 to node 2: {client_software_name=apache-kafka-java,client_software_version=2.7.0,_tagged_fields={}} 2021-01-08 08:25:47,51630 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received API_VERSIONS response from node 2 for request with header RequestHeader(apiKey=API_VERSIONS, apiVersion=3, clientId=adminclient-1, correlationId=4): org.apache.kafka.common.requests.ApiVersionsResponse@38df21a6 2021-01-08 08:25:47,51766 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:926 - [AdminClient clientId=adminclient-1] Recorded API versions for node 2: (Produce(0): 0 to 8 [usable: 8], Fetch(1): 0 to 12 [usable: 12], ListOffsets(2): 0 to 5 [usable: 5], Metadata(3): 0 to 9 [usable: 9], LeaderAndIsr(4): 0 to 4 [usable: 4], StopReplica(5): 0 to 3 [usable: 3], UpdateMetadata(6): 0 to 6 [usable: 6], ControlledShutdown(7): 0 to 3 [usable: 3], OffsetCommit(8): 0 to 8 [usable: 8], OffsetFetch(9): 0 to 7 [usable: 7], FindCoordinator(10): 0 to 3 [usable: 3], JoinGroup(11): 0 to 7 [usable: 7], Heartbeat(12): 0 to 4 [usable: 4], LeaveGroup(13): 0 to 4 [usable: 4], SyncGroup(14): 0 to 5 [usable: 5], DescribeGroups(15): 0 to 5 [usable: 5], ListGroups(16): 0 to 4 [usable: 4], SaslHandshake(17): 0 to 1 [usable: 1], ApiVersions(18): 0 to 3 [usable: 3], CreateTopics(19): 0 to 6 [usable: 6], DeleteTopics(20): 0 to 5 [usable: 5], DeleteRecords(21): 0 to 2 [usable: 2], InitProducerId(22): 0 to 4 [usable: 4], OffsetForLeaderEpoch(23): 0 to 3 [usable: 3], AddPartitionsToTxn(24): 0 to 2 [usable: 2], AddOffsetsToTxn(25): 0 to 2 [usable: 2], EndTxn(26): 0 to 2 [usable: 2], WriteTxnMarkers(27): 0 [usable: 0], TxnOffsetCommit(28): 0 to 3 [usable: 3], DescribeAcls(29): 0 to 2 [usable: 2], CreateAcls(30): 0 to 2 [usable: 2], DeleteAcls(31): 0 to 2 [usable: 2], DescribeConfigs(32): 0 to 3 [usable: 3], AlterConfigs(33): 0 to 1 [usable: 1], AlterReplicaLogDirs(34): 0 to 1 [usable: 1], DescribeLogDirs(35): 0 to 2 [usable: 2], SaslAuthenticate(36): 0 to 2 [usable: 2], CreatePartitions(37): 0 to 3 [usable: 3], CreateDelegationToken(38): 0 to 2 [usable: 2], RenewDelegationToken(39): 0 to 2 [usable: 2], ExpireDelegationToken(40): 0 to 2 [usable: 2], DescribeDelegationToken(41): 0 to 2 [usable: 2], DeleteGroups(42): 0 to 2 [usable: 2], ElectLeaders(43): 0 to 2 [usable: 2], IncrementalAlterConfigs(44): 0 to 1 [usable: 1], AlterPartitionReassignments(45): 0 [usable: 0], ListPartitionReassignments(46): 0 [usable: 0], OffsetDelete(47): 0 [usable: 0], DescribeClientQuotas(48): 0 [usable: 0], AlterClientQuotas(49): 0 [usable: 0], DescribeUserScramCredentials(50): 0 [usable: 0], AlterUserScramCredentials(51): 0 [usable: 0], AlterIsr(56): 0 [usable: 0], UpdateFeatures(57): 0 [usable: 0]) 2021-01-08 08:25:47,51852 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=5 2021-01-08 08:25:47,52093 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=5) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:25:47,52529 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=5): org.apache.kafka.common.requests.CreateTopicsResponse@77c2259b 2021-01-08 08:25:47,52903 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094407528, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:25:47,52919 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=6 2021-01-08 08:25:47,52963 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=6) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:25:47,53140 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094407530, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:25:47,53183 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=7 2021-01-08 08:25:47,53218 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=7) and timeout 30000 to node 1: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:25:47,53552 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:1003 - [AdminClient clientId=adminclient-1] Initiating connection to node my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null) using address my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc/172.17.0.14 2021-01-08 08:25:47,54067 DEBUG [kafka-admin-client-thread | adminclient-1] Selector:531 - [AdminClient clientId=adminclient-1] Created socket with SO_RCVBUF = 65536, SO_SNDBUF = 131072, SO_TIMEOUT = 0 to node 0 2021-01-08 08:25:47,55074 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=6): org.apache.kafka.common.requests.CreateTopicsResponse@7b5c97af 2021-01-08 08:25:47,55192 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=7): org.apache.kafka.common.requests.MetadataResponse@71a8050a 2021-01-08 08:25:47,55217 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:955 - [AdminClient clientId=adminclient-1] Completed connection to node 0. Fetching API versions. 2021-01-08 08:25:47,55358 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094407553, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:25:47,55393 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094407553, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:25:47,56544 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=8 2021-01-08 08:25:47,56602 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=8) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:25:47,59881 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=8): org.apache.kafka.common.requests.CreateTopicsResponse@6926b91c 2021-01-08 08:25:47,59996 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=9 2021-01-08 08:25:47,60030 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=9) and timeout 30000 to node 2: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:25:47,60260 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094407602, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:25:47,60259 DEBUG [kafka-admin-client-thread | adminclient-1] SslTransportLayer:452 - [SslTransportLayer channelId=0 key=channel=java.nio.channels.SocketChannel[connection-pending remote=my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc/172.17.0.14:9091], selector=sun.nio.ch.EPollSelectorImpl@4947b966, interestOps=8, readyOps=0] SSL handshake completed successfully with peerHost 'my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc' peerPort 9091 peerPrincipal 'CN=my-cluster-kafka, O=io.strimzi' cipherSuite 'TLS_AES_128_GCM_SHA256' 2021-01-08 08:25:47,60310 DEBUG [kafka-admin-client-thread | adminclient-1] Selector:560 - [AdminClient clientId=adminclient-1] Successfully authenticated with my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc/172.17.0.14 2021-01-08 08:25:47,60343 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094407603, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:25:47,60371 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:969 - [AdminClient clientId=adminclient-1] Initiating API versions fetch from node 0. 2021-01-08 08:25:47,60406 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending API_VERSIONS request with header RequestHeader(apiKey=API_VERSIONS, apiVersion=3, clientId=adminclient-1, correlationId=10) and timeout 3600000 to node 0: {client_software_name=apache-kafka-java,client_software_version=2.7.0,_tagged_fields={}} 2021-01-08 08:25:47,60479 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=11 2021-01-08 08:25:47,60508 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=11) and timeout 30000 to node 1: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:25:47,61026 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=11): org.apache.kafka.common.requests.MetadataResponse@4aa16c57 2021-01-08 08:25:47,61434 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@3aea6efd to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=12 2021-01-08 08:25:47,61502 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=12) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:25:47,61798 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received API_VERSIONS response from node 0 for request with header RequestHeader(apiKey=API_VERSIONS, apiVersion=3, clientId=adminclient-1, correlationId=10): org.apache.kafka.common.requests.ApiVersionsResponse@798ad644 2021-01-08 08:25:47,61886 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:926 - [AdminClient clientId=adminclient-1] Recorded API versions for node 0: (Produce(0): 0 to 8 [usable: 8], Fetch(1): 0 to 12 [usable: 12], ListOffsets(2): 0 to 5 [usable: 5], Metadata(3): 0 to 9 [usable: 9], LeaderAndIsr(4): 0 to 4 [usable: 4], StopReplica(5): 0 to 3 [usable: 3], UpdateMetadata(6): 0 to 6 [usable: 6], ControlledShutdown(7): 0 to 3 [usable: 3], OffsetCommit(8): 0 to 8 [usable: 8], OffsetFetch(9): 0 to 7 [usable: 7], FindCoordinator(10): 0 to 3 [usable: 3], JoinGroup(11): 0 to 7 [usable: 7], Heartbeat(12): 0 to 4 [usable: 4], LeaveGroup(13): 0 to 4 [usable: 4], SyncGroup(14): 0 to 5 [usable: 5], DescribeGroups(15): 0 to 5 [usable: 5], ListGroups(16): 0 to 4 [usable: 4], SaslHandshake(17): 0 to 1 [usable: 1], ApiVersions(18): 0 to 3 [usable: 3], CreateTopics(19): 0 to 6 [usable: 6], DeleteTopics(20): 0 to 5 [usable: 5], DeleteRecords(21): 0 to 2 [usable: 2], InitProducerId(22): 0 to 4 [usable: 4], OffsetForLeaderEpoch(23): 0 to 3 [usable: 3], AddPartitionsToTxn(24): 0 to 2 [usable: 2], AddOffsetsToTxn(25): 0 to 2 [usable: 2], EndTxn(26): 0 to 2 [usable: 2], WriteTxnMarkers(27): 0 [usable: 0], TxnOffsetCommit(28): 0 to 3 [usable: 3], DescribeAcls(29): 0 to 2 [usable: 2], CreateAcls(30): 0 to 2 [usable: 2], DeleteAcls(31): 0 to 2 [usable: 2], DescribeConfigs(32): 0 to 3 [usable: 3], AlterConfigs(33): 0 to 1 [usable: 1], AlterReplicaLogDirs(34): 0 to 1 [usable: 1], DescribeLogDirs(35): 0 to 2 [usable: 2], SaslAuthenticate(36): 0 to 2 [usable: 2], CreatePartitions(37): 0 to 3 [usable: 3], CreateDelegationToken(38): 0 to 2 [usable: 2], RenewDelegationToken(39): 0 to 2 [usable: 2], ExpireDelegationToken(40): 0 to 2 [usable: 2], DescribeDelegationToken(41): 0 to 2 [usable: 2], DeleteGroups(42): 0 to 2 [usable: 2], ElectLeaders(43): 0 to 2 [usable: 2], IncrementalAlterConfigs(44): 0 to 1 [usable: 1], AlterPartitionReassignments(45): 0 [usable: 0], ListPartitionReassignments(46): 0 [usable: 0], OffsetDelete(47): 0 [usable: 0], DescribeClientQuotas(48): 0 [usable: 0], AlterClientQuotas(49): 0 [usable: 0], DescribeUserScramCredentials(50): 0 [usable: 0], AlterUserScramCredentials(51): 0 [usable: 0], AlterIsr(56): 0 [usable: 0], UpdateFeatures(57): 0 [usable: 0]) 2021-01-08 08:25:47,61940 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=9): org.apache.kafka.common.requests.MetadataResponse@5959e7cb 2021-01-08 08:25:47,62139 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@30ade79b to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=13 2021-01-08 08:25:47,62175 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=13) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:25:47,62218 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@80c8ecb to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=14 2021-01-08 08:25:47,62243 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=14) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:25:47,63197 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=12): org.apache.kafka.common.requests.DescribeConfigsResponse@31ea8dff 2021-01-08 08:25:47,64050 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 8|initial kube my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:null 2021-01-08 08:25:47,64166 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=14): org.apache.kafka.common.requests.DescribeConfigsResponse@30d8876a 2021-01-08 08:25:47,64188 DEBUG [vert.x-eventloop-thread-0] TopicOperator:638 - 8|initial kube my-topic-reversed|1208693: KafkaTopic created in k8s and topic created in kafka, but they're identical => just creating in topicStore 2021-01-08 08:25:47,64217 DEBUG [vert.x-eventloop-thread-0] TopicOperator:639 - 8|initial kube my-topic-reversed|1208693: k8s and kafka versions of topic 'my-topic-reversed' are the same 2021-01-08 08:25:47,64236 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=13): org.apache.kafka.common.requests.DescribeConfigsResponse@59a7ea9d 2021-01-08 08:25:47,64488 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 6|initial kube consumer-offsets|1209467: Reconciling topic __consumer_offsets, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:null 2021-01-08 08:25:47,64519 DEBUG [vert.x-eventloop-thread-1] TopicOperator:638 - 6|initial kube consumer-offsets|1209467: KafkaTopic created in k8s and topic created in kafka, but they're identical => just creating in topicStore 2021-01-08 08:25:47,64543 DEBUG [vert.x-eventloop-thread-1] TopicOperator:639 - 6|initial kube consumer-offsets|1209467: k8s and kafka versions of topic '__consumer_offsets' are the same 2021-01-08 08:25:47,64632 DEBUG [vert.x-eventloop-thread-1] TopicOperator:769 - Enqueuing event CreateInTopicStore(topicName=__consumer_offsets,ctx=6|initial kube consumer-offsets|1209467) 2021-01-08 08:25:47,64727 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1171 - Executing CreateInTopicStore(topicName=__consumer_offsets,ctx=6|initial kube consumer-offsets|1209467) 2021-01-08 08:25:47,65532 DEBUG [vert.x-eventloop-thread-0] TopicOperator:769 - Enqueuing event CreateInTopicStore(topicName=my-topic-reversed,ctx=8|initial kube my-topic-reversed|1208693) 2021-01-08 08:25:47,66099 DEBUG [vert.x-eventloop-thread-1] ZkTopicStore:90 - create znode /strimzi/topics/__consumer_offsets 2021-01-08 08:25:47,66120 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 7|initial kube my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:null 2021-01-08 08:25:47,66157 DEBUG [vert.x-eventloop-thread-0] TopicOperator:638 - 7|initial kube my-topic|1208692: KafkaTopic created in k8s and topic created in kafka, but they're identical => just creating in topicStore 2021-01-08 08:25:47,66184 DEBUG [vert.x-eventloop-thread-0] TopicOperator:639 - 7|initial kube my-topic|1208692: k8s and kafka versions of topic 'my-topic' are the same 2021-01-08 08:25:47,66208 DEBUG [vert.x-eventloop-thread-0] TopicOperator:769 - Enqueuing event CreateInTopicStore(topicName=my-topic,ctx=7|initial kube my-topic|1208692) 2021-01-08 08:25:47,66274 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1171 - Executing CreateInTopicStore(topicName=my-topic-reversed,ctx=8|initial kube my-topic-reversed|1208693) 2021-01-08 08:25:47,66348 DEBUG [vert.x-eventloop-thread-0] ZkTopicStore:90 - create znode /strimzi/topics/my-topic-reversed 2021-01-08 08:25:47,66416 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1171 - Executing CreateInTopicStore(topicName=my-topic,ctx=7|initial kube my-topic|1208692) 2021-01-08 08:25:47,66475 DEBUG [vert.x-eventloop-thread-0] ZkTopicStore:90 - create znode /strimzi/topics/my-topic 2021-01-08 08:25:47,67260 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 24,1 replyHeader:: 24,4294967572,0 request:: '/strimzi/topics/__consumer_offsets,#7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,v{s{15,s{'world,'anyone}}},0 response:: '/strimzi/topics/__consumer_offsets 2021-01-08 08:25:47,67374 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1173 - Completing CreateInTopicStore(topicName=__consumer_offsets,ctx=6|initial kube consumer-offsets|1209467) 2021-01-08 08:25:47,67405 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1178 - CreateInTopicStore(topicName=__consumer_offsets,ctx=6|initial kube consumer-offsets|1209467) succeeded 2021-01-08 08:25:47,67430 DEBUG [vert.x-eventloop-thread-1] TopicOperator:973 - succeeded reconciliation reconcile-with-kube 2021-01-08 08:25:47,67463 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 6|initial kube consumer-offsets|1209467: Executing handler for action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:25:47,67490 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 6|initial kube consumer-offsets|1209467: There is a KafkaTopic to set status on, rv=1209467, generation=1 2021-01-08 08:25:47,67831 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 25,1 replyHeader:: 25,4294967573,0 request:: '/strimzi/topics/my-topic-reversed,#7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,v{s{15,s{'world,'anyone}}},0 response:: '/strimzi/topics/my-topic-reversed 2021-01-08 08:25:47,67906 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 26,1 replyHeader:: 26,4294967574,0 request:: '/strimzi/topics/my-topic,#7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,v{s{15,s{'world,'anyone}}},0 response:: '/strimzi/topics/my-topic 2021-01-08 08:25:47,68006 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1173 - Completing CreateInTopicStore(topicName=my-topic-reversed,ctx=8|initial kube my-topic-reversed|1208693) 2021-01-08 08:25:47,68029 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1178 - CreateInTopicStore(topicName=my-topic-reversed,ctx=8|initial kube my-topic-reversed|1208693) succeeded 2021-01-08 08:25:47,68048 DEBUG [vert.x-eventloop-thread-0] TopicOperator:973 - succeeded reconciliation reconcile-with-kube 2021-01-08 08:25:47,68075 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 8|initial kube my-topic-reversed|1208693: Executing handler for action reconcile-with-kube on topic my-topic-reversed 2021-01-08 08:25:47,68097 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 8|initial kube my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:25:47,72706 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:25:47.681078Z"} 2021-01-08 08:25:47,72713 DEBUG [vert.x-eventloop-thread-1] StatusDiff:48 - Status differs: {"op":"replace","path":"/","value":{"conditions":[{"type":"Ready","status":"True","lastTransitionTime":"2021-01-08T08:25:47.677492Z"}],"observedGeneration":1}} 2021-01-08 08:25:47,72779 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 8|initial kube my-topic-reversed|1208693: Lock released 2021-01-08 08:25:47,72782 DEBUG [vert.x-eventloop-thread-1] StatusDiff:49 - Current Status path / has value 2021-01-08 08:25:47,72806 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 8|initial kube my-topic-reversed|1208693: Removing last waiter reconcile-with-kube 2021-01-08 08:25:47,72809 DEBUG [vert.x-eventloop-thread-1] StatusDiff:50 - Desired Status path / has value 2021-01-08 08:25:47,72834 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1173 - Completing CreateInTopicStore(topicName=my-topic,ctx=7|initial kube my-topic|1208692) 2021-01-08 08:25:47,72856 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1178 - CreateInTopicStore(topicName=my-topic,ctx=7|initial kube my-topic|1208692) succeeded 2021-01-08 08:25:47,72876 DEBUG [vert.x-eventloop-thread-0] TopicOperator:973 - succeeded reconciliation reconcile-with-kube 2021-01-08 08:25:47,72901 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 7|initial kube my-topic|1208692: Executing handler for action reconcile-with-kube on topic my-topic 2021-01-08 08:25:47,72922 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 7|initial kube my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:25:47,73124 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:25:47.729355Z"} 2021-01-08 08:25:47,73159 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 7|initial kube my-topic|1208692: Lock released 2021-01-08 08:25:47,73181 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 7|initial kube my-topic|1208692: Removing last waiter reconcile-with-kube 2021-01-08 08:25:47,77277 DEBUG [OkHttp https://172.30.0.1/...] K8sTopicWatcher:42 - Ignoring initial event for KafkaTopic consumer-offsets during initial reconcile 2021-01-08 08:25:47,77278 DEBUG [kubernetes-ops-pool-0] CrdOperator:117 - Status of KafkaTopic consumer-offsets in namespace myproject has been updated 2021-01-08 08:25:47,77438 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1014 - 6|initial kube consumer-offsets|1209467: status was set rv=1210026, generation=1, observedGeneration=1 2021-01-08 08:25:47,77527 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 6|initial kube consumer-offsets|1209467: Lock released 2021-01-08 08:25:47,77553 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 6|initial kube consumer-offsets|1209467: Removing last waiter reconcile-with-kube 2021-01-08 08:25:52,45949 DEBUG [vert.x-eventloop-thread-0] Recycler:100 - -Dio.netty.recycler.maxCapacityPerThread: 4096 2021-01-08 08:25:52,45980 DEBUG [vert.x-eventloop-thread-0] Recycler:101 - -Dio.netty.recycler.maxSharedCapacityFactor: 2 2021-01-08 08:25:52,46000 DEBUG [vert.x-eventloop-thread-0] Recycler:102 - -Dio.netty.recycler.linkCapacity: 16 2021-01-08 08:25:52,46022 DEBUG [vert.x-eventloop-thread-0] Recycler:103 - -Dio.netty.recycler.ratio: 8 2021-01-08 08:25:52,46041 DEBUG [vert.x-eventloop-thread-0] Recycler:104 - -Dio.netty.recycler.delayedQueue.ratio: 8 2021-01-08 08:25:52,47189 DEBUG [vert.x-eventloop-thread-0] AbstractByteBuf:63 - -Dio.netty.buffer.checkAccessible: true 2021-01-08 08:25:52,47218 DEBUG [vert.x-eventloop-thread-0] AbstractByteBuf:64 - -Dio.netty.buffer.checkBounds: true 2021-01-08 08:25:52,47368 DEBUG [vert.x-eventloop-thread-0] ResourceLeakDetectorFactory:195 - Loaded default ResourceLeakDetector: io.netty.util.ResourceLeakDetector@f0448e3 2021-01-08 08:25:52,58752 DEBUG [vert.x-eventloop-thread-0] ZlibCodecFactory:39 - -Dio.netty.noJdkZlibDecoder: false 2021-01-08 08:25:52,58794 DEBUG [vert.x-eventloop-thread-0] ZlibCodecFactory:42 - -Dio.netty.noJdkZlibEncoder: false 2021-01-08 08:25:54,34139 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 2ms 2021-01-08 08:26:01,01384 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:26:07,69000 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 3ms 2021-01-08 08:26:14,36198 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:26:21,03237 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:26:27,70463 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:26:34,37790 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:26:41,04876 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:26:47,72034 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:26:54,39405 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:27:01,06428 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:27:07,73750 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:27:14,40882 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:27:17,86338 INFO [vert.x-eventloop-thread-1] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:27:17,86377 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:127 - Listing topics 2021-01-08 08:27:17,86408 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610094497863, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:17,86513 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=15 2021-01-08 08:27:17,86563 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=15) and timeout 30000 to node 0: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:27:17,87286 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=15): org.apache.kafka.common.requests.MetadataResponse@71abd631 2021-01-08 08:27:17,87459 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:27:17,87489 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 10|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:27:17,87516 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 10|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:27:17,87552 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 11|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:27:17,87574 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 11|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:27:17,87603 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 12|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:27:17,87620 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 12|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:27:17,87649 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 10|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:27:17,87667 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 10|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:27:17,87735 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 11|periodic kafka my-topic: Lock acquired 2021-01-08 08:27:17,87757 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 11|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:27:17,87790 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 12|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:27:17,87806 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 12|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:27:17,88415 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 27,4 replyHeader:: 27,4294967577,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:27:17,88704 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 10|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:27:17,88735 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 28,4 replyHeader:: 28,4294967577,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:27:17,89134 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 11|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:27:17,89241 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 29,4 replyHeader:: 29,4294967577,0 request:: '/strimzi/topics/__consumer_offsets,F response:: #7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294967572,4294967572,1610094347663,1610094347663,0,0,0,0,233,0,4294967572} 2021-01-08 08:27:17,89498 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 12|periodic kafka __consumer_offsets: Have private topic for topic __consumer_offsets in Kafka 2021-01-08 08:27:17,91364 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 10|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:27:17,91427 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:27:17,91480 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094497914, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:17,91607 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=16 2021-01-08 08:27:17,91659 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=16) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:27:17,92118 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=16): org.apache.kafka.common.requests.CreateTopicsResponse@6350d8a 2021-01-08 08:27:17,92173 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 11|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:27:17,92249 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:27:17,92461 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094497924, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:17,92506 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094497924, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:17,92572 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094497922, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:17,92645 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=17 2021-01-08 08:27:17,92780 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=17) and timeout 30000 to node 0: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:27:17,92931 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@722229f1 to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=18 2021-01-08 08:27:17,92960 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=18) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:27:17,93536 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=17): org.apache.kafka.common.requests.MetadataResponse@1437d62a 2021-01-08 08:27:17,93875 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=18): org.apache.kafka.common.requests.DescribeConfigsResponse@3b8280c3 2021-01-08 08:27:17,94190 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 10|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:27:17,94230 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 10|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:27:17,94388 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 10|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:27:17,94417 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 10|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:27:17,94462 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 10|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:27:17,94484 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 10|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:27:17,94556 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:27:17,94587 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:27:17,94611 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 10|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:27:17,94633 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 10|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:27:17,94897 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:27:17.946504Z"} 2021-01-08 08:27:17,94928 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=19 2021-01-08 08:27:17,95016 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 10|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:27:17,95024 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=19) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:27:17,95174 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 10|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:27:17,95324 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=19): org.apache.kafka.common.requests.CreateTopicsResponse@15f58364 2021-01-08 08:27:17,95399 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094497953, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:17,95430 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094497954, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:17,95590 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=20 2021-01-08 08:27:17,95621 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=20) and timeout 30000 to node 0: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:27:17,95663 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@76fb4d8a to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=21 2021-01-08 08:27:17,95804 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=21) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:27:17,96271 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:27:17,96319 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094497962, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:17,96388 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=22 2021-01-08 08:27:17,96426 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=22) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:27:17,96612 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=20): org.apache.kafka.common.requests.MetadataResponse@5c5743f9 2021-01-08 08:27:17,96914 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=22): org.apache.kafka.common.requests.CreateTopicsResponse@73fc5428 2021-01-08 08:27:17,96978 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=21): org.apache.kafka.common.requests.DescribeConfigsResponse@4aab1790 2021-01-08 08:27:17,97113 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094497970, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:17,97144 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094497971, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:17,97211 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@66908452 to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=23 2021-01-08 08:27:17,97239 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=23) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:27:17,97280 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=24 2021-01-08 08:27:17,97304 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=24) and timeout 30000 to node 1: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:27:17,97404 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 11|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:27:17,97430 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 11|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:27:17,97458 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 11|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:27:17,97482 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 11|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:27:17,97505 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 11|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:27:17,97524 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 11|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:27:17,97546 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:27:17,97580 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:27:17,97612 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 11|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:27:17,97633 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 11|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:27:17,97776 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:27:17.976430Z"} 2021-01-08 08:27:17,97828 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 11|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:27:17,97852 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 11|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:27:17,97903 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=24): org.apache.kafka.common.requests.MetadataResponse@3e4bf2 2021-01-08 08:27:18,00093 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=23): org.apache.kafka.common.requests.DescribeConfigsResponse@11f212c9 2021-01-08 08:27:18,01153 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 12|periodic kafka __consumer_offsets: Reconciling topic __consumer_offsets, k8sTopic:null, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:27:18,01185 DEBUG [vert.x-eventloop-thread-0] TopicOperator:584 - 12|periodic kafka __consumer_offsets: KafkaTopic deleted in k8s => delete topic from kafka and from topicStore 2021-01-08 08:27:18,01375 DEBUG [vert.x-eventloop-thread-0] TopicOperator:769 - Enqueuing event DeleteKafkaTopic(topicName=__consumer_offsets,ctx=12|periodic kafka __consumer_offsets) 2021-01-08 08:27:18,01516 INFO [vert.x-eventloop-thread-0] TopicOperator:364 - 12|periodic kafka __consumer_offsets: Deleting topic '__consumer_offsets' 2021-01-08 08:27:18,01538 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:57 - Deleting topic __consumer_offsets 2021-01-08 08:27:18,01682 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=deleteTopics, deadlineMs=1610094498016, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:18,01773 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending DeleteTopicsRequestData(topicNames=[__consumer_offsets], timeoutMs=30000) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=25 2021-01-08 08:27:18,01832 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DELETE_TOPICS request with header RequestHeader(apiKey=DELETE_TOPICS, apiVersion=5, clientId=adminclient-1, correlationId=25) and timeout 30000 to node 2: {topic_names=[__consumer_offsets],timeout_ms=30000,_tagged_fields={}} 2021-01-08 08:27:18,10362 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DELETE_TOPICS response from node 2 for request with header RequestHeader(apiKey=DELETE_TOPICS, apiVersion=5, clientId=adminclient-1, correlationId=25): org.apache.kafka.common.requests.DeleteTopicsResponse@42522840 2021-01-08 08:27:18,10859 DEBUG [vert.x-eventloop-thread-1] TopicOperator:769 - Enqueuing event DeleteFromTopicStore(topicName=__consumer_offsets,ctx=12|periodic kafka __consumer_offsets) 2021-01-08 08:27:18,10902 DEBUG [vert.x-eventloop-thread-1] ZkTopicStore:117 - delete znode /strimzi/topics/__consumer_offsets 2021-01-08 08:27:18,12913 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 30,2 replyHeader:: 30,4294967579,0 request:: '/strimzi/topics/__consumer_offsets,-1 response:: null 2021-01-08 08:27:18,13073 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic null 2021-01-08 08:27:18,13103 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - __consumer_offsets reconcile success -> succeeded 2021-01-08 08:27:18,13129 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 12|periodic kafka __consumer_offsets: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:27:18,13147 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 12|periodic kafka __consumer_offsets: No KafkaTopic to set status 2021-01-08 08:27:18,14150 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 12|periodic kafka __consumer_offsets: Lock released 2021-01-08 08:27:18,14191 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 12|periodic kafka __consumer_offsets: Removing last waiter reconcile-from-kafka 2021-01-08 08:27:18,16194 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 13|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:27:18,16245 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 14|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:27:18,16271 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 15|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:27:19,95173 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:27:19,95322 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/brokers/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:27:19,95396 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:27:19,95526 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics for sessionid 0x101a07e36350011 2021-01-08 08:27:19,95584 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeDeleted path:/brokers/topics/__consumer_offsets 2021-01-08 08:27:19,95637 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Data of /brokers/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@1bc0a214] 2021-01-08 08:27:19,95665 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:27:19,95681 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics 2021-01-08 08:27:19,95700 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #1 ZkEvent[Data of /brokers/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@1bc0a214] 2021-01-08 08:27:19,95709 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:27:19,95759 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:27:19,95887 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:27:19,95914 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/config/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:27:19,95944 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeDeleted path:/config/topics/__consumer_offsets 2021-01-08 08:27:19,95970 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Data of /config/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@6fd6d4ef] 2021-01-08 08:27:19,95984 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:27:19,96025 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 31,3 replyHeader:: 31,4294967833,-101 request:: '/brokers/topics/__consumer_offsets,T response:: 2021-01-08 08:27:19,96379 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 32,4 replyHeader:: 32,4294967834,-101 request:: '/brokers/topics/__consumer_offsets,T response:: 2021-01-08 08:27:19,96421 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #1 done 2021-01-08 08:27:19,96441 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #2 ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:27:19,96630 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 33,3 replyHeader:: 33,4294967834,0 request:: '/brokers/topics,T response:: s{4294967304,4294967304,1610094013731,1610094013731,0,4,0,0,0,2,4294967832} 2021-01-08 08:27:19,96975 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 34,8 replyHeader:: 34,4294967834,0 request:: '/brokers/topics,T response:: v{'my-topic,'my-topic-reversed} 2021-01-08 08:27:19,97046 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:115 - 1: znode /brokers/topics now has children [my-topic, my-topic-reversed], previous children [my-topic, __consumer_offsets, my-topic-reversed] 2021-01-08 08:27:19,97081 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:127 - Topics deleted from ZK for watch 1: [__consumer_offsets] 2021-01-08 08:27:19,97105 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicConfigsWatcher:98 - Unwatching znode __consumer_offsets for changes 2021-01-08 08:27:19,97227 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicWatcher:98 - Unwatching znode __consumer_offsets for changes 2021-01-08 08:27:19,97536 DEBUG [ZkClient-EventThread-19-localhost:2181] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094499975, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:19,97607 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=26 2021-01-08 08:27:19,97646 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=26) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:27:19,97933 DEBUG [ZkClient-EventThread-19-localhost:2181] Util:94 - Waiting for 16|/brokers/topics 1:-__consumer_offsets to get confirmed non-existence 2021-01-08 08:27:19,98783 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=26): org.apache.kafka.common.requests.CreateTopicsResponse@197451d9 2021-01-08 08:27:19,99183 DEBUG [vert.x-eventloop-thread-1] Util:116 - 16|/brokers/topics 1:-__consumer_offsets is confirmed non-existence 2021-01-08 08:27:19,99224 DEBUG [vert.x-eventloop-thread-1] TopicOperator:957 - Metric onTopicDeleted triggered 2021-01-08 08:27:19,99227 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:143 - Topics created in ZK for watch 1: [] 2021-01-08 08:27:19,99247 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 16|/brokers/topics 1:-__consumer_offsets: Queuing action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:27:19,99259 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #2 done 2021-01-08 08:27:19,99273 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 16|/brokers/topics 1:-__consumer_offsets: Adding first waiter onTopicDeleted 2021-01-08 08:27:19,99278 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #3 ZkEvent[Data of /config/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@6fd6d4ef] 2021-01-08 08:27:19,99324 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 16|/brokers/topics 1:-__consumer_offsets: Lock acquired 2021-01-08 08:27:19,99343 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 16|/brokers/topics 1:-__consumer_offsets: Executing action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:27:19,99516 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 35,3 replyHeader:: 35,4294967834,-101 request:: '/config/topics/__consumer_offsets,T response:: 2021-01-08 08:27:19,99586 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 36,4 replyHeader:: 36,4294967834,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:27:19,99714 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 37,4 replyHeader:: 37,4294967834,-101 request:: '/config/topics/__consumer_offsets,T response:: 2021-01-08 08:27:19,99740 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #3 done 2021-01-08 08:27:20,00778 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 16|/brokers/topics 1:-__consumer_offsets: Reconciling topic null, k8sTopic:null, kafkaTopic:null, privateTopic:null 2021-01-08 08:27:20,00809 DEBUG [vert.x-eventloop-thread-1] TopicOperator:544 - 16|/brokers/topics 1:-__consumer_offsets: All three topics null during reconciliation. 2021-01-08 08:27:20,00825 DEBUG [vert.x-eventloop-thread-1] TopicOperator:973 - succeeded reconciliation onTopicDeleted 2021-01-08 08:27:20,00844 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 16|/brokers/topics 1:-__consumer_offsets: Executing handler for action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:27:20,00860 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 16|/brokers/topics 1:-__consumer_offsets: No KafkaTopic to set status 2021-01-08 08:27:20,00885 DEBUG [vert.x-eventloop-thread-1] ZkTopicsWatcher:135 - 16|/brokers/topics 1:-__consumer_offsets: Success responding to deletion of topic __consumer_offsets 2021-01-08 08:27:20,00903 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 16|/brokers/topics 1:-__consumer_offsets: Lock released 2021-01-08 08:27:20,00919 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 16|/brokers/topics 1:-__consumer_offsets: Removing last waiter onTopicDeleted 2021-01-08 08:27:20,06707 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:27:20,06739 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeCreated path:/config/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:27:20,06774 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeCreated path:/config/topics/__consumer_offsets 2021-01-08 08:27:20,06793 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:27:20,09604 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:27:20,09636 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeCreated path:/brokers/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:27:20,09673 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:27:20,09683 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeCreated path:/brokers/topics/__consumer_offsets 2021-01-08 08:27:20,09691 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics for sessionid 0x101a07e36350011 2021-01-08 08:27:20,09711 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:27:20,09728 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics 2021-01-08 08:27:20,09744 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:27:20,09763 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:27:20,09777 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #4 ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:27:20,10030 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 38,3 replyHeader:: 38,4294967838,0 request:: '/brokers/topics,T response:: s{4294967304,4294967304,1610094013731,1610094013731,0,5,0,0,0,3,4294967837} 2021-01-08 08:27:20,10297 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 39,8 replyHeader:: 39,4294967838,0 request:: '/brokers/topics,T response:: v{'my-topic,'__consumer_offsets,'my-topic-reversed} 2021-01-08 08:27:20,10351 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:115 - 2: znode /brokers/topics now has children [my-topic, __consumer_offsets, my-topic-reversed], previous children [my-topic, my-topic-reversed] 2021-01-08 08:27:20,10383 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:127 - Topics deleted from ZK for watch 2: [] 2021-01-08 08:27:20,10399 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:143 - Topics created in ZK for watch 2: [__consumer_offsets] 2021-01-08 08:27:20,10416 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicConfigsWatcher:73 - Watching znode /config/topics/__consumer_offsets for changes 2021-01-08 08:27:20,10466 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicWatcher:73 - Watching znode /brokers/topics/__consumer_offsets for changes 2021-01-08 08:27:20,10745 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:957 - Metric onTopicCreated triggered 2021-01-08 08:27:20,10777 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:460 - 17|/brokers/topics 2:+__consumer_offsets: Queuing action onTopicCreated on topic __consumer_offsets 2021-01-08 08:27:20,10792 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:463 - 17|/brokers/topics 2:+__consumer_offsets: Adding first waiter onTopicCreated 2021-01-08 08:27:20,10885 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 17|/brokers/topics 2:+__consumer_offsets: Lock acquired 2021-01-08 08:27:20,10915 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 17|/brokers/topics 2:+__consumer_offsets: Executing action onTopicCreated on topic __consumer_offsets 2021-01-08 08:27:20,11071 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094500110, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:20,11105 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 40,3 replyHeader:: 40,4294967839,0 request:: '/config/topics/__consumer_offsets,T response:: s{4294967836,4294967838,1610094440060,1610094440086,1,0,0,0,109,0,4294967836} 2021-01-08 08:27:20,11127 DEBUG [vert.x-eventloop-thread-1] Util:94 - Waiting for 17|/brokers/topics 2:+__consumer_offsets to get confirmed existence 2021-01-08 08:27:20,11179 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #4 done 2021-01-08 08:27:20,11215 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=27 2021-01-08 08:27:20,11234 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 41,3 replyHeader:: 41,4294967839,0 request:: '/brokers/topics/__consumer_offsets,T response:: s{4294967837,4294967837,1610094440085,1610094440085,0,0,0,0,712,0,4294967837} 2021-01-08 08:27:20,11258 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=27) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:27:20,11682 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-3] ZkClient:196 - Subscribed data changes for /config/topics/__consumer_offsets 2021-01-08 08:27:20,11295 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-0] ZkClient:196 - Subscribed data changes for /brokers/topics/__consumer_offsets 2021-01-08 08:27:20,12040 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=27): org.apache.kafka.common.requests.CreateTopicsResponse@45d09d1d 2021-01-08 08:27:20,12532 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 42,4 replyHeader:: 42,4294967839,0 request:: '/brokers/topics/__consumer_offsets,T response:: #7b2276657273696f6e223a322c22706172746974696f6e73223a7b223330223a5b302c312c325d2c223339223a5b302c322c315d2c223435223a5b302c322c315d2c2232223a5b322c302c315d2c2235223a5b322c312c305d2c223438223a5b302c312c325d2c223333223a5b302c322c315d2c223237223a5b302c322c315d2c223132223a5b302c312c325d2c2238223a5b322c302c315d2c223135223a5b302c322c315d2c223432223a5b302c312c325d2c223336223a5b302c312c325d2c223231223a5b302c322c315d2c223138223a5b302c312c325d2c223234223a5b302c312c325d2c223335223a5b322c312c305d2c223431223a5b322c312c305d2c2237223a5b312c322c305d2c223137223a5b322c312c305d2c2231223a5b312c322c305d2c223434223a5b322c302c315d2c223233223a5b322c312c305d2c223338223a5b322c302c315d2c223437223a5b322c312c305d2c2234223a5b312c302c325d2c223236223a5b322c302c315d2c223131223a5b322c312c305d2c223332223a5b322c302c315d2c223134223a5b322c302c315d2c223230223a5b322c302c315d2c223239223a5b322c312c305d2c223436223a5b312c302c325d2c223334223a5b312c302c325d2c223238223a5b312c302c325d2c2236223a5b302c312c325d2c223430223a5b312c302c325d2c223439223a5b312c322c305d2c2239223a5b302c322c315d2c223433223a5b312c322c305d2c2230223a5b302c312c325d2c223232223a5b312c302c325d2c223136223a5b312c302c325d2c223337223a5b312c322c305d2c223139223a5b312c322c305d2c2233223a5b302c322c315d2c223130223a5b312c302c325d2c223331223a5b312c322c305d2c223235223a5b312c322c305d2c223133223a5b312c322c305d7d2c22616464696e675f7265706c69636173223a7b7d2c2272656d6f76696e675f7265706c69636173223a7b7d7d,s{4294967837,4294967837,1610094440085,1610094440085,0,0,0,0,712,0,4294967837} 2021-01-08 08:27:20,12612 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 43,4 replyHeader:: 43,4294967839,0 request:: '/config/topics/__consumer_offsets,T response:: #7b2276657273696f6e223a312c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294967836,4294967838,1610094440060,1610094440086,1,0,0,0,109,0,4294967836} 2021-01-08 08:27:21,12560 DEBUG [vert.x-eventloop-thread-1] Util:116 - 17|/brokers/topics 2:+__consumer_offsets is confirmed existence 2021-01-08 08:27:21,12596 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:27:21,12629 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094501126, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:21,12760 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=28 2021-01-08 08:27:21,12809 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=28) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:27:21,13308 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=28): org.apache.kafka.common.requests.CreateTopicsResponse@2d892149 2021-01-08 08:27:21,13592 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094501135, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:21,13628 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094501136, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:27:21,13658 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=29 2021-01-08 08:27:21,13689 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=29) and timeout 30000 to node 0: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:27:21,13816 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@16532a35 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=30 2021-01-08 08:27:21,13840 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=30) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:27:21,14188 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=30): org.apache.kafka.common.requests.DescribeConfigsResponse@19e75722 2021-01-08 08:27:21,14319 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 17|/brokers/topics 2:+__consumer_offsets: Executing handler for action onTopicCreated on topic __consumer_offsets 2021-01-08 08:27:21,14346 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 17|/brokers/topics 2:+__consumer_offsets: No KafkaTopic to set status 2021-01-08 08:27:21,14374 WARN [vert.x-eventloop-thread-1] ZkTopicsWatcher:153 - 17|/brokers/topics 2:+__consumer_offsets: Error responding to creation of topic __consumer_offsets org.apache.kafka.common.errors.UnknownTopicOrPartitionException: 2021-01-08 08:27:21,14420 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 17|/brokers/topics 2:+__consumer_offsets: Lock released 2021-01-08 08:27:21,14439 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 17|/brokers/topics 2:+__consumer_offsets: Removing last waiter onTopicCreated 2021-01-08 08:27:21,15652 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=29): org.apache.kafka.common.requests.MetadataResponse@581e1976 2021-01-08 08:27:26,79530 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:27:33,46865 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:27:40,14242 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:27:46,81583 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:27:53,48995 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:28:00,16275 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:28:06,83560 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:28:13,50908 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:28:20,17533 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:28:26,84860 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:28:33,51776 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:28:40,18530 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:28:46,85923 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:28:48,25017 INFO [vert.x-eventloop-thread-1] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:28:48,25060 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:127 - Listing topics 2021-01-08 08:28:48,25086 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610094588250, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:28:48,25172 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=31 2021-01-08 08:28:48,25209 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=31) and timeout 30000 to node 2: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:28:48,25649 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=31): org.apache.kafka.common.requests.MetadataResponse@3cf9e855 2021-01-08 08:28:48,25790 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:28:48,25819 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 18|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:28:48,25838 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 18|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:28:48,26178 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 19|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:28:48,26225 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 19|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:28:48,26271 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 20|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:28:48,26288 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 20|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:28:48,26319 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 18|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:28:48,26334 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 18|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:28:48,26474 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 19|periodic kafka my-topic: Lock acquired 2021-01-08 08:28:48,26496 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 19|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:28:48,26525 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 20|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:28:48,26540 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 20|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:28:48,27049 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 44,4 replyHeader:: 44,4294967940,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:28:48,27331 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 18|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:28:48,27335 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 45,4 replyHeader:: 45,4294967940,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:28:48,27499 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 19|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:28:48,27698 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 46,4 replyHeader:: 46,4294967940,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:28:48,27774 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1364 - 20|periodic kafka __consumer_offsets: No private topic for topic __consumer_offsets in Kafka -> undetermined 2021-01-08 08:28:48,27797 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 20|periodic kafka __consumer_offsets: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:28:48,27813 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 20|periodic kafka __consumer_offsets: No KafkaTopic to set status 2021-01-08 08:28:48,27831 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 20|periodic kafka __consumer_offsets: Lock released 2021-01-08 08:28:48,27847 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 20|periodic kafka __consumer_offsets: Removing last waiter reconcile-from-kafka 2021-01-08 08:28:48,29288 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 18|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:28:48,29326 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:28:48,29365 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094588293, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:28:48,29445 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=32 2021-01-08 08:28:48,29490 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=32) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:28:48,29777 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 19|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:28:48,29811 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:28:48,29841 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094588298, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:28:48,30148 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=32): org.apache.kafka.common.requests.CreateTopicsResponse@33f47f7d 2021-01-08 08:28:48,30258 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=33 2021-01-08 08:28:48,30267 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094588302, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:28:48,30293 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=33) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:28:48,30299 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094588302, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:28:48,30405 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=34 2021-01-08 08:28:48,30429 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=34) and timeout 30000 to node 1: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:28:48,30612 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=33): org.apache.kafka.common.requests.CreateTopicsResponse@10897066 2021-01-08 08:28:48,30711 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094588306, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:28:48,30744 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094588307, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:28:48,30764 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=35 2021-01-08 08:28:48,30791 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=35) and timeout 30000 to node 0: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:28:48,30940 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=34): org.apache.kafka.common.requests.MetadataResponse@38741ea8 2021-01-08 08:28:48,31107 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@4bbcf2b3 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=36 2021-01-08 08:28:48,31138 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=36) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:28:48,31300 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=35): org.apache.kafka.common.requests.MetadataResponse@5ba3732e 2021-01-08 08:28:48,32135 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=36): org.apache.kafka.common.requests.DescribeConfigsResponse@72227c 2021-01-08 08:28:48,32244 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@18ebfd84 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=37 2021-01-08 08:28:48,32266 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 18|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:28:48,32275 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=37) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:28:48,32292 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 18|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:28:48,32317 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 18|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:28:48,32336 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 18|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:28:48,32357 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 18|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:28:48,32373 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 18|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:28:48,32392 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:28:48,32410 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:28:48,32432 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 18|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:28:48,32456 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 18|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:28:48,32582 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:28:48.324667Z"} 2021-01-08 08:28:48,32647 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 18|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:28:48,32677 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 18|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:28:48,33231 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=37): org.apache.kafka.common.requests.DescribeConfigsResponse@34eebd40 2021-01-08 08:28:48,33358 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 19|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:28:48,33382 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 19|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:28:48,33404 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 19|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:28:48,33425 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 19|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:28:48,33444 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 19|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:28:48,33462 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 19|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:28:48,33483 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:28:48,33504 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:28:48,33524 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 19|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:28:48,33543 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 19|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:28:48,33658 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:28:48.335521Z"} 2021-01-08 08:28:48,33802 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 19|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:28:48,33830 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 19|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:28:48,35029 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 21|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:28:48,35075 DEBUG [vert.x-eventloop-thread-1] TopicOperator:957 - Metric reconcile-with-kube triggered 2021-01-08 08:28:48,35096 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 21|periodic kube consumer-offsets|1210026: Queuing action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:28:48,35114 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 21|periodic kube consumer-offsets|1210026: Adding first waiter reconcile-with-kube 2021-01-08 08:28:48,35151 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 22|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:28:48,35177 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 23|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:28:48,35203 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 21|periodic kube consumer-offsets|1210026: Lock acquired 2021-01-08 08:28:48,35219 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 21|periodic kube consumer-offsets|1210026: Executing action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:28:48,35303 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:28:48,35344 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094588353, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:28:48,35397 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=38 2021-01-08 08:28:48,35436 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=38) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:28:48,35712 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=38): org.apache.kafka.common.requests.CreateTopicsResponse@741482f0 2021-01-08 08:28:48,35866 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094588358, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:28:48,35896 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094588358, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:28:48,35913 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=39 2021-01-08 08:28:48,35942 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=39) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:28:48,36050 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@65e43fcc to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=40 2021-01-08 08:28:48,36080 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=40) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:28:48,36196 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 47,4 replyHeader:: 47,4294967940,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:28:48,36806 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=40): org.apache.kafka.common.requests.DescribeConfigsResponse@22a4e929 2021-01-08 08:28:48,37035 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=39): org.apache.kafka.common.requests.MetadataResponse@63e15480 2021-01-08 08:28:48,37342 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 21|periodic kube consumer-offsets|1210026: Reconciling topic __consumer_offsets, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:null 2021-01-08 08:28:48,37374 DEBUG [vert.x-eventloop-thread-0] TopicOperator:638 - 21|periodic kube consumer-offsets|1210026: KafkaTopic created in k8s and topic created in kafka, but they're identical => just creating in topicStore 2021-01-08 08:28:48,37394 DEBUG [vert.x-eventloop-thread-0] TopicOperator:639 - 21|periodic kube consumer-offsets|1210026: k8s and kafka versions of topic '__consumer_offsets' are the same 2021-01-08 08:28:48,37413 DEBUG [vert.x-eventloop-thread-0] TopicOperator:769 - Enqueuing event CreateInTopicStore(topicName=__consumer_offsets,ctx=21|periodic kube consumer-offsets|1210026) 2021-01-08 08:28:48,37433 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1171 - Executing CreateInTopicStore(topicName=__consumer_offsets,ctx=21|periodic kube consumer-offsets|1210026) 2021-01-08 08:28:48,37562 DEBUG [vert.x-eventloop-thread-0] ZkTopicStore:90 - create znode /strimzi/topics/__consumer_offsets 2021-01-08 08:28:48,38600 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 48,1 replyHeader:: 48,4294967941,0 request:: '/strimzi/topics/__consumer_offsets,#7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,v{s{15,s{'world,'anyone}}},0 response:: '/strimzi/topics/__consumer_offsets 2021-01-08 08:28:48,38690 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1173 - Completing CreateInTopicStore(topicName=__consumer_offsets,ctx=21|periodic kube consumer-offsets|1210026) 2021-01-08 08:28:48,38711 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1178 - CreateInTopicStore(topicName=__consumer_offsets,ctx=21|periodic kube consumer-offsets|1210026) succeeded 2021-01-08 08:28:48,38732 DEBUG [vert.x-eventloop-thread-0] TopicOperator:973 - succeeded reconciliation reconcile-with-kube 2021-01-08 08:28:48,38790 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 21|periodic kube consumer-offsets|1210026: Executing handler for action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:28:48,38814 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 21|periodic kube consumer-offsets|1210026: There is a KafkaTopic to set status on, rv=1210026, generation=1 2021-01-08 08:28:48,39195 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:28:48.388254Z"} 2021-01-08 08:28:48,39233 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 21|periodic kube consumer-offsets|1210026: Lock released 2021-01-08 08:28:48,39252 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 21|periodic kube consumer-offsets|1210026: Removing last waiter reconcile-with-kube 2021-01-08 08:28:55,05071 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:29:01,72438 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:29:08,39823 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:29:15,07100 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:29:21,74363 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:29:28,41694 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:29:35,08707 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:29:41,76071 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:29:48,43074 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:29:55,10481 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:30:01,77826 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:30:08,45174 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:30:15,12517 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:30:18,39459 INFO [vert.x-eventloop-thread-0] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:30:18,39493 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:127 - Listing topics 2021-01-08 08:30:18,39519 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610094678395, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:18,39588 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=41 2021-01-08 08:30:18,39639 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=41) and timeout 30000 to node 2: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:30:18,40122 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=41): org.apache.kafka.common.requests.MetadataResponse@33b2590e 2021-01-08 08:30:18,40286 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:30:18,40318 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 24|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:30:18,40338 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 24|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:30:18,40371 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 25|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:30:18,40392 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 25|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:30:18,40417 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 26|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:30:18,40432 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 26|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:30:18,40458 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 24|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:30:18,40472 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 24|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:30:18,40538 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 25|periodic kafka my-topic: Lock acquired 2021-01-08 08:30:18,40561 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 25|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:30:18,40592 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 26|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:30:18,40605 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 26|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:30:18,40873 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 49,4 replyHeader:: 49,4294967943,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:30:18,41077 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 24|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:30:18,41231 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 50,4 replyHeader:: 50,4294967943,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:30:18,41401 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 25|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:30:18,41429 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 51,4 replyHeader:: 51,4294967943,0 request:: '/strimzi/topics/__consumer_offsets,F response:: #7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294967941,4294967941,1610094528377,1610094528377,0,0,0,0,233,0,4294967941} 2021-01-08 08:30:18,41579 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 26|periodic kafka __consumer_offsets: Have private topic for topic __consumer_offsets in Kafka 2021-01-08 08:30:18,42706 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 24|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:30:18,42729 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:30:18,42764 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094678427, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:18,42825 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=42 2021-01-08 08:30:18,42862 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=42) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:30:18,43216 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=42): org.apache.kafka.common.requests.CreateTopicsResponse@77ec1aeb 2021-01-08 08:30:18,43336 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094678433, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:18,43376 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094678433, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:18,43446 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=43 2021-01-08 08:30:18,43481 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=43) and timeout 30000 to node 0: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:30:18,43677 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 25|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:30:18,43707 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:30:18,43734 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094678437, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:18,43919 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=44 2021-01-08 08:30:18,43968 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=44) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:30:18,44131 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=43): org.apache.kafka.common.requests.MetadataResponse@7204589e 2021-01-08 08:30:18,44302 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@1b28470a to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=45 2021-01-08 08:30:18,44327 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=45) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:30:18,44466 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=44): org.apache.kafka.common.requests.CreateTopicsResponse@3af31748 2021-01-08 08:30:18,44696 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:30:18,44726 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094678446, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:18,44733 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094678447, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:18,44770 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094678447, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:18,44829 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=46 2021-01-08 08:30:18,44859 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=46) and timeout 30000 to node 2: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:30:18,45178 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=46): org.apache.kafka.common.requests.MetadataResponse@2ddf79bf 2021-01-08 08:30:18,45301 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=47 2021-01-08 08:30:18,45331 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=47) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:30:18,45485 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=45): org.apache.kafka.common.requests.DescribeConfigsResponse@7d4f0432 2021-01-08 08:30:18,45586 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 24|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:30:18,45600 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=47): org.apache.kafka.common.requests.CreateTopicsResponse@24061c86 2021-01-08 08:30:18,45606 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 24|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:30:18,45629 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 24|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:30:18,45645 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 24|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:30:18,45666 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 24|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:30:18,45679 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 24|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:30:18,45695 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:30:18,45715 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:30:18,45737 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 24|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:30:18,45756 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 24|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:30:18,45885 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:30:18.457652Z"} 2021-01-08 08:30:18,45910 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 24|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:30:18,45930 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 24|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:30:18,46002 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@66d1619 to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=48 2021-01-08 08:30:18,46026 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=48) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:30:18,46164 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094678461, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:18,46190 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094678461, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:18,46239 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=49 2021-01-08 08:30:18,46264 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=49) and timeout 30000 to node 0: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:30:18,46646 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=49): org.apache.kafka.common.requests.MetadataResponse@502995ea 2021-01-08 08:30:18,46871 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@55c27549 to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=50 2021-01-08 08:30:18,46898 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=50) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:30:18,47384 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=48): org.apache.kafka.common.requests.DescribeConfigsResponse@41b4b15b 2021-01-08 08:30:18,47509 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 25|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:30:18,47531 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 25|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:30:18,47553 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 25|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:30:18,47574 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 25|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:30:18,47592 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 25|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:30:18,47607 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 25|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:30:18,47624 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:30:18,47639 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:30:18,47660 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 25|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:30:18,47680 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 25|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:30:18,47806 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:30:18.476886Z"} 2021-01-08 08:30:18,47859 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 25|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:30:18,47882 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 25|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:30:18,47886 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=50): org.apache.kafka.common.requests.DescribeConfigsResponse@2f87aaaa 2021-01-08 08:30:18,47979 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 26|periodic kafka __consumer_offsets: Reconciling topic __consumer_offsets, k8sTopic:null, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:30:18,47998 DEBUG [vert.x-eventloop-thread-0] TopicOperator:584 - 26|periodic kafka __consumer_offsets: KafkaTopic deleted in k8s => delete topic from kafka and from topicStore 2021-01-08 08:30:18,48014 DEBUG [vert.x-eventloop-thread-0] TopicOperator:769 - Enqueuing event DeleteKafkaTopic(topicName=__consumer_offsets,ctx=26|periodic kafka __consumer_offsets) 2021-01-08 08:30:18,48035 INFO [vert.x-eventloop-thread-0] TopicOperator:364 - 26|periodic kafka __consumer_offsets: Deleting topic '__consumer_offsets' 2021-01-08 08:30:18,48048 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:57 - Deleting topic __consumer_offsets 2021-01-08 08:30:18,48073 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=deleteTopics, deadlineMs=1610094678480, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:18,48114 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending DeleteTopicsRequestData(topicNames=[__consumer_offsets], timeoutMs=30000) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=51 2021-01-08 08:30:18,48145 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DELETE_TOPICS request with header RequestHeader(apiKey=DELETE_TOPICS, apiVersion=5, clientId=adminclient-1, correlationId=51) and timeout 30000 to node 2: {topic_names=[__consumer_offsets],timeout_ms=30000,_tagged_fields={}} 2021-01-08 08:30:18,52383 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DELETE_TOPICS response from node 2 for request with header RequestHeader(apiKey=DELETE_TOPICS, apiVersion=5, clientId=adminclient-1, correlationId=51): org.apache.kafka.common.requests.DeleteTopicsResponse@339dec75 2021-01-08 08:30:18,52515 DEBUG [vert.x-eventloop-thread-1] TopicOperator:769 - Enqueuing event DeleteFromTopicStore(topicName=__consumer_offsets,ctx=26|periodic kafka __consumer_offsets) 2021-01-08 08:30:18,52544 DEBUG [vert.x-eventloop-thread-1] ZkTopicStore:117 - delete znode /strimzi/topics/__consumer_offsets 2021-01-08 08:30:18,53094 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 52,2 replyHeader:: 52,4294967945,0 request:: '/strimzi/topics/__consumer_offsets,-1 response:: null 2021-01-08 08:30:18,53167 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic null 2021-01-08 08:30:18,53187 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - __consumer_offsets reconcile success -> succeeded 2021-01-08 08:30:18,53208 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 26|periodic kafka __consumer_offsets: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:30:18,53224 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 26|periodic kafka __consumer_offsets: No KafkaTopic to set status 2021-01-08 08:30:18,53344 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 26|periodic kafka __consumer_offsets: Lock released 2021-01-08 08:30:18,53377 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 26|periodic kafka __consumer_offsets: Removing last waiter reconcile-from-kafka 2021-01-08 08:30:18,54177 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 27|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:30:18,54215 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 28|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:30:18,54235 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 29|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:30:19,75536 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:30:19,75571 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/brokers/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:30:19,75599 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:30:19,75614 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics for sessionid 0x101a07e36350011 2021-01-08 08:30:19,75609 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeDeleted path:/brokers/topics/__consumer_offsets 2021-01-08 08:30:19,75636 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Data of /brokers/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@71181b03] 2021-01-08 08:30:19,75651 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:30:19,75665 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics 2021-01-08 08:30:19,75677 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:30:19,75674 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #5 ZkEvent[Data of /brokers/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@71181b03] 2021-01-08 08:30:19,75687 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:30:19,75906 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:30:19,75926 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/config/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:30:19,75949 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeDeleted path:/config/topics/__consumer_offsets 2021-01-08 08:30:19,75969 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Data of /config/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@3eedd047] 2021-01-08 08:30:19,75980 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:30:19,75988 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 53,3 replyHeader:: 53,4294968198,-101 request:: '/brokers/topics/__consumer_offsets,T response:: 2021-01-08 08:30:19,76342 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 54,4 replyHeader:: 54,4294968199,-101 request:: '/brokers/topics/__consumer_offsets,T response:: 2021-01-08 08:30:19,76370 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #5 done 2021-01-08 08:30:19,76389 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #6 ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:30:19,76562 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 55,3 replyHeader:: 55,4294968199,0 request:: '/brokers/topics,T response:: s{4294967304,4294967304,1610094013731,1610094013731,0,6,0,0,0,2,4294968197} 2021-01-08 08:30:19,76724 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 56,8 replyHeader:: 56,4294968199,0 request:: '/brokers/topics,T response:: v{'my-topic,'my-topic-reversed} 2021-01-08 08:30:19,76767 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:115 - 3: znode /brokers/topics now has children [my-topic, my-topic-reversed], previous children [my-topic, __consumer_offsets, my-topic-reversed] 2021-01-08 08:30:19,76793 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:127 - Topics deleted from ZK for watch 3: [__consumer_offsets] 2021-01-08 08:30:19,76805 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicConfigsWatcher:98 - Unwatching znode __consumer_offsets for changes 2021-01-08 08:30:19,76844 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicWatcher:98 - Unwatching znode __consumer_offsets for changes 2021-01-08 08:30:19,77039 DEBUG [ZkClient-EventThread-19-localhost:2181] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094679770, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:19,77071 DEBUG [ZkClient-EventThread-19-localhost:2181] Util:94 - Waiting for 30|/brokers/topics 3:-__consumer_offsets to get confirmed non-existence 2021-01-08 08:30:19,77114 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=52 2021-01-08 08:30:19,77141 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:143 - Topics created in ZK for watch 3: [] 2021-01-08 08:30:19,77178 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=52) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:30:19,77187 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #6 done 2021-01-08 08:30:19,77209 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #7 ZkEvent[Data of /config/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@3eedd047] 2021-01-08 08:30:19,77376 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 57,3 replyHeader:: 57,4294968199,-101 request:: '/config/topics/__consumer_offsets,T response:: 2021-01-08 08:30:19,77503 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 58,4 replyHeader:: 58,4294968199,-101 request:: '/config/topics/__consumer_offsets,T response:: 2021-01-08 08:30:19,77530 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #7 done 2021-01-08 08:30:19,77953 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=52): org.apache.kafka.common.requests.CreateTopicsResponse@75b3af72 2021-01-08 08:30:19,86300 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:30:19,86326 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeCreated path:/config/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:30:19,86357 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeCreated path:/config/topics/__consumer_offsets 2021-01-08 08:30:19,86369 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:30:19,90658 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:30:19,90673 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeCreated path:/brokers/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:30:19,90695 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:30:19,90710 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics for sessionid 0x101a07e36350011 2021-01-08 08:30:19,90729 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeCreated path:/brokers/topics/__consumer_offsets 2021-01-08 08:30:19,90753 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:30:19,90764 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics 2021-01-08 08:30:19,90778 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:30:19,90792 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:30:19,90810 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #8 ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:30:19,90975 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 59,3 replyHeader:: 59,4294968202,0 request:: '/brokers/topics,T response:: s{4294967304,4294967304,1610094013731,1610094013731,0,7,0,0,0,3,4294968202} 2021-01-08 08:30:19,91118 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 60,8 replyHeader:: 60,4294968202,0 request:: '/brokers/topics,T response:: v{'my-topic,'__consumer_offsets,'my-topic-reversed} 2021-01-08 08:30:19,91147 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:115 - 4: znode /brokers/topics now has children [my-topic, __consumer_offsets, my-topic-reversed], previous children [my-topic, my-topic-reversed] 2021-01-08 08:30:19,91166 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:127 - Topics deleted from ZK for watch 4: [] 2021-01-08 08:30:19,91177 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:143 - Topics created in ZK for watch 4: [__consumer_offsets] 2021-01-08 08:30:19,91190 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicConfigsWatcher:73 - Watching znode /config/topics/__consumer_offsets for changes 2021-01-08 08:30:19,91232 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicWatcher:73 - Watching znode /brokers/topics/__consumer_offsets for changes 2021-01-08 08:30:19,91574 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:957 - Metric onTopicCreated triggered 2021-01-08 08:30:19,91591 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:460 - 31|/brokers/topics 4:+__consumer_offsets: Queuing action onTopicCreated on topic __consumer_offsets 2021-01-08 08:30:19,91603 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:463 - 31|/brokers/topics 4:+__consumer_offsets: Adding first waiter onTopicCreated 2021-01-08 08:30:19,91633 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #8 done 2021-01-08 08:30:19,91660 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 31|/brokers/topics 4:+__consumer_offsets: Lock acquired 2021-01-08 08:30:19,91671 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 31|/brokers/topics 4:+__consumer_offsets: Executing action onTopicCreated on topic __consumer_offsets 2021-01-08 08:30:19,91706 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094679916, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:19,91739 DEBUG [vert.x-eventloop-thread-1] Util:94 - Waiting for 31|/brokers/topics 4:+__consumer_offsets to get confirmed existence 2021-01-08 08:30:19,91772 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=53 2021-01-08 08:30:19,91835 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=53) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:30:19,92027 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 61,3 replyHeader:: 61,4294968202,0 request:: '/config/topics/__consumer_offsets,T response:: s{4294968201,4294968201,1610094619860,1610094619860,0,0,0,0,109,0,4294968201} 2021-01-08 08:30:19,92089 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 62,3 replyHeader:: 62,4294968202,0 request:: '/brokers/topics/__consumer_offsets,T response:: s{4294968202,4294968202,1610094619902,1610094619902,0,0,0,0,712,0,4294968202} 2021-01-08 08:30:19,92113 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-0] ZkClient:196 - Subscribed data changes for /brokers/topics/__consumer_offsets 2021-01-08 08:30:19,92074 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-1] ZkClient:196 - Subscribed data changes for /config/topics/__consumer_offsets 2021-01-08 08:30:19,92406 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=53): org.apache.kafka.common.requests.CreateTopicsResponse@11e9e6f6 2021-01-08 08:30:19,93201 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 63,4 replyHeader:: 63,4294968203,0 request:: '/brokers/topics/__consumer_offsets,T response:: #7b2276657273696f6e223a322c22706172746974696f6e73223a7b223330223a5b302c322c315d2c223339223a5b302c312c325d2c223435223a5b302c312c325d2c2232223a5b322c312c305d2c2235223a5b322c302c315d2c223438223a5b302c322c315d2c223333223a5b302c312c325d2c223237223a5b302c312c325d2c223132223a5b302c322c315d2c2238223a5b322c312c305d2c223135223a5b302c312c325d2c223432223a5b302c322c315d2c223336223a5b302c322c315d2c223231223a5b302c312c325d2c223138223a5b302c322c315d2c223234223a5b302c322c315d2c223335223a5b322c302c315d2c223431223a5b322c302c315d2c2237223a5b312c302c325d2c223137223a5b322c302c315d2c2231223a5b312c302c325d2c223434223a5b322c312c305d2c223233223a5b322c302c315d2c223338223a5b322c312c305d2c223437223a5b322c302c315d2c2234223a5b312c322c305d2c223236223a5b322c312c305d2c223131223a5b322c302c315d2c223332223a5b322c312c305d2c223134223a5b322c312c305d2c223230223a5b322c312c305d2c223239223a5b322c302c315d2c223436223a5b312c322c305d2c223334223a5b312c322c305d2c223238223a5b312c322c305d2c2236223a5b302c322c315d2c223430223a5b312c322c305d2c223439223a5b312c302c325d2c2239223a5b302c312c325d2c223433223a5b312c302c325d2c2230223a5b302c322c315d2c223232223a5b312c322c305d2c223136223a5b312c322c305d2c223337223a5b312c302c325d2c223139223a5b312c302c325d2c2233223a5b302c312c325d2c223130223a5b312c322c305d2c223331223a5b312c302c325d2c223235223a5b312c302c325d2c223133223a5b312c302c325d7d2c22616464696e675f7265706c69636173223a7b7d2c2272656d6f76696e675f7265706c69636173223a7b7d7d,s{4294968202,4294968202,1610094619902,1610094619902,0,1,0,0,712,1,4294968203} 2021-01-08 08:30:19,93268 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 64,4 replyHeader:: 64,4294968203,0 request:: '/config/topics/__consumer_offsets,T response:: #7b2276657273696f6e223a312c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294968201,4294968201,1610094619860,1610094619860,0,0,0,0,109,0,4294968201} 2021-01-08 08:30:20,77531 DEBUG [vert.x-eventloop-thread-1] Util:116 - 30|/brokers/topics 3:-__consumer_offsets is confirmed non-existence 2021-01-08 08:30:20,77560 DEBUG [vert.x-eventloop-thread-1] TopicOperator:957 - Metric onTopicDeleted triggered 2021-01-08 08:30:20,77574 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 30|/brokers/topics 3:-__consumer_offsets: Queuing action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:30:20,77590 DEBUG [vert.x-eventloop-thread-1] TopicOperator:466 - 30|/brokers/topics 3:-__consumer_offsets: Adding waiter onTopicDeleted: 2 2021-01-08 08:30:20,94076 DEBUG [vert.x-eventloop-thread-1] Util:116 - 31|/brokers/topics 4:+__consumer_offsets is confirmed existence 2021-01-08 08:30:20,94103 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:30:20,94140 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094680941, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:20,94201 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=54 2021-01-08 08:30:20,94233 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=54) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:30:20,96150 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=54): org.apache.kafka.common.requests.CreateTopicsResponse@329dc9b 2021-01-08 08:30:20,96432 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094680964, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:20,96471 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094680964, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:30:20,96502 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=55 2021-01-08 08:30:20,96530 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=55) and timeout 30000 to node 1: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:30:20,96644 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@5f8b76cf to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=56 2021-01-08 08:30:20,96668 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=56) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:30:20,98266 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=56): org.apache.kafka.common.requests.DescribeConfigsResponse@372b9a73 2021-01-08 08:30:20,98800 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=55): org.apache.kafka.common.requests.MetadataResponse@692b0436 2021-01-08 08:30:20,98953 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 31|/brokers/topics 4:+__consumer_offsets: Executing handler for action onTopicCreated on topic __consumer_offsets 2021-01-08 08:30:20,98976 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 31|/brokers/topics 4:+__consumer_offsets: No KafkaTopic to set status 2021-01-08 08:30:20,98993 WARN [vert.x-eventloop-thread-1] ZkTopicsWatcher:153 - 31|/brokers/topics 4:+__consumer_offsets: Error responding to creation of topic __consumer_offsets org.apache.kafka.common.errors.UnknownTopicOrPartitionException: 2021-01-08 08:30:20,99038 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 31|/brokers/topics 4:+__consumer_offsets: Lock released 2021-01-08 08:30:20,99057 DEBUG [vert.x-eventloop-thread-1] TopicOperator:452 - 31|/brokers/topics 4:+__consumer_offsets: Removing waiter onTopicCreated, 1 waiters left 2021-01-08 08:30:20,99071 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 30|/brokers/topics 3:-__consumer_offsets: Lock acquired 2021-01-08 08:30:20,99080 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 30|/brokers/topics 3:-__consumer_offsets: Executing action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:30:20,99341 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 65,4 replyHeader:: 65,4294968303,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:30:21,00665 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 30|/brokers/topics 3:-__consumer_offsets: Reconciling topic null, k8sTopic:null, kafkaTopic:null, privateTopic:null 2021-01-08 08:30:21,00691 DEBUG [vert.x-eventloop-thread-1] TopicOperator:544 - 30|/brokers/topics 3:-__consumer_offsets: All three topics null during reconciliation. 2021-01-08 08:30:21,00705 DEBUG [vert.x-eventloop-thread-1] TopicOperator:973 - succeeded reconciliation onTopicDeleted 2021-01-08 08:30:21,00723 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 30|/brokers/topics 3:-__consumer_offsets: Executing handler for action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:30:21,00735 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 30|/brokers/topics 3:-__consumer_offsets: No KafkaTopic to set status 2021-01-08 08:30:21,00750 DEBUG [vert.x-eventloop-thread-1] ZkTopicsWatcher:135 - 30|/brokers/topics 3:-__consumer_offsets: Success responding to deletion of topic __consumer_offsets 2021-01-08 08:30:21,00765 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 30|/brokers/topics 3:-__consumer_offsets: Lock released 2021-01-08 08:30:21,00778 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 30|/brokers/topics 3:-__consumer_offsets: Removing last waiter onTopicDeleted 2021-01-08 08:30:27,66676 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:30:34,33721 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:30:41,01012 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:30:41,36674 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:938 - [AdminClient clientId=adminclient-1] Node -1 disconnected. 2021-01-08 08:30:41,36890 DEBUG [kafka-admin-client-thread | adminclient-1] AdminMetadataManager:174 - [AdminClient clientId=adminclient-1] Requesting metadata update. 2021-01-08 08:30:41,36944 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[], allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=57 2021-01-08 08:30:41,36979 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=57) and timeout 30000 to node 2: {topics=[],allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:30:41,37291 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=57): org.apache.kafka.common.requests.MetadataResponse@77a31ce5 2021-01-08 08:30:41,37343 DEBUG [kafka-admin-client-thread | adminclient-1] AdminMetadataManager:247 - [AdminClient clientId=adminclient-1] Updating cluster metadata to Cluster(id = 24eqSYNVTzaDk8st3RVFrQ, nodes = [my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null), my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null), my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null)], partitions = [], controller = my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null)) 2021-01-08 08:30:47,68313 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:30:54,35614 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:31:01,02362 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:31:07,69653 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:31:14,36872 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:31:21,04156 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:31:27,70877 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:31:34,38248 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:31:41,05557 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:31:47,72904 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:31:48,62893 INFO [vert.x-eventloop-thread-1] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:31:48,62918 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:127 - Listing topics 2021-01-08 08:31:48,62944 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610094768629, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:31:48,63018 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=58 2021-01-08 08:31:48,63045 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=58) and timeout 30000 to node 2: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:31:48,63489 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=58): org.apache.kafka.common.requests.MetadataResponse@54cc7d5f 2021-01-08 08:31:48,63639 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:31:48,63682 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 32|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:31:48,63702 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 32|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:31:48,63748 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 33|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:31:48,63770 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 33|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:31:48,63799 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 34|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:31:48,63813 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 34|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:31:48,63841 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 32|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:31:48,63852 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 32|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:31:48,63921 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 33|periodic kafka my-topic: Lock acquired 2021-01-08 08:31:48,63934 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 33|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:31:48,63964 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 34|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:31:48,63976 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 34|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:31:48,64180 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 66,4 replyHeader:: 66,4294968305,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:31:48,64516 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 32|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:31:48,64688 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 67,4 replyHeader:: 67,4294968305,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:31:48,65229 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 68,4 replyHeader:: 68,4294968305,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:31:48,65311 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 33|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:31:48,65374 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1364 - 34|periodic kafka __consumer_offsets: No private topic for topic __consumer_offsets in Kafka -> undetermined 2021-01-08 08:31:48,65393 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 34|periodic kafka __consumer_offsets: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:31:48,65406 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 34|periodic kafka __consumer_offsets: No KafkaTopic to set status 2021-01-08 08:31:48,65421 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 34|periodic kafka __consumer_offsets: Lock released 2021-01-08 08:31:48,65434 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 34|periodic kafka __consumer_offsets: Removing last waiter reconcile-from-kafka 2021-01-08 08:31:48,66139 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 32|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:31:48,66205 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:31:48,66240 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094768662, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:31:48,66751 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 33|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:31:48,66794 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:31:48,66833 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094768668, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:31:48,66835 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=59 2021-01-08 08:31:48,66873 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=59) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:31:48,67179 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=59): org.apache.kafka.common.requests.CreateTopicsResponse@4e796d66 2021-01-08 08:31:48,67280 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=60 2021-01-08 08:31:48,67297 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094768672, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:31:48,67311 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=60) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:31:48,67327 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094768673, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:31:48,67419 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=61 2021-01-08 08:31:48,67441 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=61) and timeout 30000 to node 0: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:31:48,67549 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=60): org.apache.kafka.common.requests.CreateTopicsResponse@4e0b7d79 2021-01-08 08:31:48,67647 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094768676, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:31:48,67672 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094768676, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:31:48,67800 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=61): org.apache.kafka.common.requests.MetadataResponse@21201469 2021-01-08 08:31:48,67940 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@1631a397 to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=62 2021-01-08 08:31:48,67968 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=62) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:31:48,68691 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=62): org.apache.kafka.common.requests.DescribeConfigsResponse@13a6fe6a 2021-01-08 08:31:48,68801 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=63 2021-01-08 08:31:48,68827 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=63) and timeout 30000 to node 0: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:31:48,68959 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 32|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:31:48,68983 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 32|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:31:48,69008 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 32|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:31:48,69023 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 32|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:31:48,69037 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 32|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:31:48,69048 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 32|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:31:48,69067 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:31:48,69083 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:31:48,69103 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 32|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:31:48,69118 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 32|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:31:48,69184 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=63): org.apache.kafka.common.requests.MetadataResponse@6d3a9d2 2021-01-08 08:31:48,69364 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:31:48.691253Z"} 2021-01-08 08:31:48,69382 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@27f1663d to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=64 2021-01-08 08:31:48,69407 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 32|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:31:48,69411 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=64) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:31:48,69427 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 32|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:31:48,69999 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=64): org.apache.kafka.common.requests.DescribeConfigsResponse@3cc7544f 2021-01-08 08:31:48,70131 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 33|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:31:48,70150 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 33|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:31:48,70195 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 33|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:31:48,70213 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 33|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:31:48,70229 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 33|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:31:48,70243 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 33|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:31:48,70260 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:31:48,70276 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:31:48,70292 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 33|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:31:48,70306 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 33|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:31:48,70421 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:31:48.703130Z"} 2021-01-08 08:31:48,70528 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 33|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:31:48,70605 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 33|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:31:48,71431 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 35|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:31:48,71457 DEBUG [vert.x-eventloop-thread-1] TopicOperator:957 - Metric reconcile-with-kube triggered 2021-01-08 08:31:48,71473 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 35|periodic kube consumer-offsets|1210026: Queuing action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:31:48,71487 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 35|periodic kube consumer-offsets|1210026: Adding first waiter reconcile-with-kube 2021-01-08 08:31:48,71517 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 36|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:31:48,71536 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 37|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:31:48,71559 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 35|periodic kube consumer-offsets|1210026: Lock acquired 2021-01-08 08:31:48,71570 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 35|periodic kube consumer-offsets|1210026: Executing action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:31:48,71634 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:31:48,71665 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094768716, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:31:48,71738 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=65 2021-01-08 08:31:48,71774 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=65) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:31:48,72075 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=65): org.apache.kafka.common.requests.CreateTopicsResponse@18b6a813 2021-01-08 08:31:48,72207 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094768721, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:31:48,72232 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094768722, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:31:48,72258 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=66 2021-01-08 08:31:48,72288 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=66) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:31:48,72395 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@3300fee to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=67 2021-01-08 08:31:48,72415 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=67) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:31:48,72584 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 69,4 replyHeader:: 69,4294968305,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:31:48,72612 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=66): org.apache.kafka.common.requests.MetadataResponse@2f5c413b 2021-01-08 08:31:48,73185 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=67): org.apache.kafka.common.requests.DescribeConfigsResponse@293f1db3 2021-01-08 08:31:48,73314 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 35|periodic kube consumer-offsets|1210026: Reconciling topic __consumer_offsets, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:null 2021-01-08 08:31:48,73343 DEBUG [vert.x-eventloop-thread-0] TopicOperator:638 - 35|periodic kube consumer-offsets|1210026: KafkaTopic created in k8s and topic created in kafka, but they're identical => just creating in topicStore 2021-01-08 08:31:48,73359 DEBUG [vert.x-eventloop-thread-0] TopicOperator:639 - 35|periodic kube consumer-offsets|1210026: k8s and kafka versions of topic '__consumer_offsets' are the same 2021-01-08 08:31:48,73378 DEBUG [vert.x-eventloop-thread-0] TopicOperator:769 - Enqueuing event CreateInTopicStore(topicName=__consumer_offsets,ctx=35|periodic kube consumer-offsets|1210026) 2021-01-08 08:31:48,73395 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1171 - Executing CreateInTopicStore(topicName=__consumer_offsets,ctx=35|periodic kube consumer-offsets|1210026) 2021-01-08 08:31:48,73485 DEBUG [vert.x-eventloop-thread-0] ZkTopicStore:90 - create znode /strimzi/topics/__consumer_offsets 2021-01-08 08:31:48,76465 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 70,1 replyHeader:: 70,4294968306,0 request:: '/strimzi/topics/__consumer_offsets,#7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,v{s{15,s{'world,'anyone}}},0 response:: '/strimzi/topics/__consumer_offsets 2021-01-08 08:31:48,76541 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1173 - Completing CreateInTopicStore(topicName=__consumer_offsets,ctx=35|periodic kube consumer-offsets|1210026) 2021-01-08 08:31:48,76556 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1178 - CreateInTopicStore(topicName=__consumer_offsets,ctx=35|periodic kube consumer-offsets|1210026) succeeded 2021-01-08 08:31:48,76568 DEBUG [vert.x-eventloop-thread-0] TopicOperator:973 - succeeded reconciliation reconcile-with-kube 2021-01-08 08:31:48,76584 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 35|periodic kube consumer-offsets|1210026: Executing handler for action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:31:48,76596 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 35|periodic kube consumer-offsets|1210026: There is a KafkaTopic to set status on, rv=1210026, generation=1 2021-01-08 08:31:48,76724 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:31:48.766023Z"} 2021-01-08 08:31:48,76761 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 35|periodic kube consumer-offsets|1210026: Lock released 2021-01-08 08:31:48,76776 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 35|periodic kube consumer-offsets|1210026: Removing last waiter reconcile-with-kube 2021-01-08 08:31:55,40885 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:32:02,08093 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:32:08,75357 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:32:15,42335 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:32:22,09201 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:32:28,76231 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:32:35,43573 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:32:42,10876 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:32:48,77987 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:32:55,45101 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:33:02,12416 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:33:08,79645 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:33:15,46544 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:33:18,77021 INFO [vert.x-eventloop-thread-0] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:33:18,77042 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:127 - Listing topics 2021-01-08 08:33:18,77068 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610094858770, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:18,77149 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=68 2021-01-08 08:33:18,77183 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=68) and timeout 30000 to node 0: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:33:18,77626 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=68): org.apache.kafka.common.requests.MetadataResponse@464ceb7 2021-01-08 08:33:18,77743 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:33:18,77770 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 38|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:33:18,77785 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 38|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:33:18,77814 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 39|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:33:18,77825 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 39|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:33:18,77846 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 40|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:33:18,77858 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 40|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:33:18,77880 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 38|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:33:18,77893 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 38|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:33:18,77962 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 39|periodic kafka my-topic: Lock acquired 2021-01-08 08:33:18,77985 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 39|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:33:18,78029 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 40|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:33:18,78040 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 40|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:33:18,78190 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 71,4 replyHeader:: 71,4294968307,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:33:18,78486 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 38|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:33:18,78786 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 72,4 replyHeader:: 72,4294968307,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:33:18,78977 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 39|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:33:18,79015 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 73,4 replyHeader:: 73,4294968307,0 request:: '/strimzi/topics/__consumer_offsets,F response:: #7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294968306,4294968306,1610094708736,1610094708736,0,0,0,0,233,0,4294968306} 2021-01-08 08:33:18,79164 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 40|periodic kafka __consumer_offsets: Have private topic for topic __consumer_offsets in Kafka 2021-01-08 08:33:18,80388 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 38|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:33:18,80414 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:33:18,80444 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094858804, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:18,80518 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=69 2021-01-08 08:33:18,80556 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=69) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:33:18,80920 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=69): org.apache.kafka.common.requests.CreateTopicsResponse@7a8feaa 2021-01-08 08:33:18,81321 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094858812, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:18,81360 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 39|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:33:18,81364 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094858813, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:18,81607 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=70 2021-01-08 08:33:18,81637 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:33:18,81667 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=70) and timeout 30000 to node 1: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:33:18,81679 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094858816, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:18,81874 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=71 2021-01-08 08:33:18,81904 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=71) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:33:18,82087 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=70): org.apache.kafka.common.requests.MetadataResponse@4ab022fd 2021-01-08 08:33:18,82113 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:33:18,82158 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094858821, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:18,82230 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@1b336c9d to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=72 2021-01-08 08:33:18,82252 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=72) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:33:18,82363 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=71): org.apache.kafka.common.requests.CreateTopicsResponse@3900183a 2021-01-08 08:33:18,82662 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=73 2021-01-08 08:33:18,82667 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094858826, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:18,82697 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094858826, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:18,82699 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=73) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:33:18,82834 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=74 2021-01-08 08:33:18,82879 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=74) and timeout 30000 to node 0: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:33:18,83022 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=73): org.apache.kafka.common.requests.CreateTopicsResponse@799dfc1e 2021-01-08 08:33:18,83100 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094858830, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:18,83136 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094858831, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:18,83170 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=75 2021-01-08 08:33:18,83197 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=75) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:33:18,83380 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=74): org.apache.kafka.common.requests.MetadataResponse@32719e1a 2021-01-08 08:33:18,83417 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=72): org.apache.kafka.common.requests.DescribeConfigsResponse@32264b4d 2021-01-08 08:33:18,83576 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@b5e44fb to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=76 2021-01-08 08:33:18,83596 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=76) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:33:18,83628 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@1db52482 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=77 2021-01-08 08:33:18,83647 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=77) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:33:18,83822 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=75): org.apache.kafka.common.requests.MetadataResponse@238ae860 2021-01-08 08:33:18,84025 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 38|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:33:18,84043 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 38|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:33:18,84065 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 38|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:33:18,84078 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 38|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:33:18,84092 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 38|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:33:18,84102 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 38|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:33:18,84116 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:33:18,84127 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:33:18,84142 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 38|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:33:18,84156 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 38|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:33:18,84286 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:33:18.841629Z"} 2021-01-08 08:33:18,84312 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 38|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:33:18,84326 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 38|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:33:18,84424 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=77): org.apache.kafka.common.requests.DescribeConfigsResponse@23f6a37f 2021-01-08 08:33:18,84618 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 40|periodic kafka __consumer_offsets: Reconciling topic __consumer_offsets, k8sTopic:null, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:33:18,84632 DEBUG [vert.x-eventloop-thread-1] TopicOperator:584 - 40|periodic kafka __consumer_offsets: KafkaTopic deleted in k8s => delete topic from kafka and from topicStore 2021-01-08 08:33:18,84647 DEBUG [vert.x-eventloop-thread-1] TopicOperator:769 - Enqueuing event DeleteKafkaTopic(topicName=__consumer_offsets,ctx=40|periodic kafka __consumer_offsets) 2021-01-08 08:33:18,84667 INFO [vert.x-eventloop-thread-1] TopicOperator:364 - 40|periodic kafka __consumer_offsets: Deleting topic '__consumer_offsets' 2021-01-08 08:33:18,84678 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:57 - Deleting topic __consumer_offsets 2021-01-08 08:33:18,84677 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=76): org.apache.kafka.common.requests.DescribeConfigsResponse@5af65266 2021-01-08 08:33:18,84696 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=deleteTopics, deadlineMs=1610094858846, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:18,84749 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending DeleteTopicsRequestData(topicNames=[__consumer_offsets], timeoutMs=30000) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=78 2021-01-08 08:33:18,84750 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 39|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:33:18,84780 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DELETE_TOPICS request with header RequestHeader(apiKey=DELETE_TOPICS, apiVersion=5, clientId=adminclient-1, correlationId=78) and timeout 30000 to node 2: {topic_names=[__consumer_offsets],timeout_ms=30000,_tagged_fields={}} 2021-01-08 08:33:18,84785 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 39|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:33:18,84805 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 39|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:33:18,84818 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 39|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:33:18,84832 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 39|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:33:18,84847 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 39|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:33:18,84864 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:33:18,84875 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:33:18,84892 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 39|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:33:18,84905 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 39|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:33:18,85023 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:33:18.849111Z"} 2021-01-08 08:33:18,85043 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 39|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:33:18,85062 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 39|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:33:18,90291 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DELETE_TOPICS response from node 2 for request with header RequestHeader(apiKey=DELETE_TOPICS, apiVersion=5, clientId=adminclient-1, correlationId=78): org.apache.kafka.common.requests.DeleteTopicsResponse@6ed3ec8e 2021-01-08 08:33:18,90403 DEBUG [vert.x-eventloop-thread-1] TopicOperator:769 - Enqueuing event DeleteFromTopicStore(topicName=__consumer_offsets,ctx=40|periodic kafka __consumer_offsets) 2021-01-08 08:33:18,90421 DEBUG [vert.x-eventloop-thread-1] ZkTopicStore:117 - delete znode /strimzi/topics/__consumer_offsets 2021-01-08 08:33:18,92108 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 74,2 replyHeader:: 74,4294968310,0 request:: '/strimzi/topics/__consumer_offsets,-1 response:: null 2021-01-08 08:33:18,92184 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic null 2021-01-08 08:33:18,92199 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - __consumer_offsets reconcile success -> succeeded 2021-01-08 08:33:18,92216 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 40|periodic kafka __consumer_offsets: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:33:18,92226 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 40|periodic kafka __consumer_offsets: No KafkaTopic to set status 2021-01-08 08:33:18,92373 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 40|periodic kafka __consumer_offsets: Lock released 2021-01-08 08:33:18,92390 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 40|periodic kafka __consumer_offsets: Removing last waiter reconcile-from-kafka 2021-01-08 08:33:18,93661 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 41|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:33:18,93702 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 42|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:33:18,93724 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 43|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:33:20,28169 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:33:20,28194 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/brokers/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:33:20,28225 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:33:20,28237 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics for sessionid 0x101a07e36350011 2021-01-08 08:33:20,28236 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeDeleted path:/brokers/topics/__consumer_offsets 2021-01-08 08:33:20,28258 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Data of /brokers/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@2c58591] 2021-01-08 08:33:20,28271 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:33:20,28279 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics 2021-01-08 08:33:20,28288 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:33:20,28294 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:33:20,28370 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #9 ZkEvent[Data of /brokers/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@2c58591] 2021-01-08 08:33:20,28512 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:33:20,28527 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/config/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:33:20,28579 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 75,3 replyHeader:: 75,4294968563,-101 request:: '/brokers/topics/__consumer_offsets,T response:: 2021-01-08 08:33:20,28619 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeDeleted path:/config/topics/__consumer_offsets 2021-01-08 08:33:20,28633 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Data of /config/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@5bed488a] 2021-01-08 08:33:20,28641 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:33:20,28744 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 76,4 replyHeader:: 76,4294968563,-101 request:: '/brokers/topics/__consumer_offsets,T response:: 2021-01-08 08:33:20,28770 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #9 done 2021-01-08 08:33:20,28779 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #10 ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:33:20,28918 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 77,3 replyHeader:: 77,4294968563,0 request:: '/brokers/topics,T response:: s{4294967304,4294967304,1610094013731,1610094013731,0,8,0,0,0,2,4294968562} 2021-01-08 08:33:20,29392 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 78,8 replyHeader:: 78,4294968564,0 request:: '/brokers/topics,T response:: v{'my-topic,'my-topic-reversed} 2021-01-08 08:33:20,29433 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:115 - 5: znode /brokers/topics now has children [my-topic, my-topic-reversed], previous children [my-topic, __consumer_offsets, my-topic-reversed] 2021-01-08 08:33:20,29452 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:127 - Topics deleted from ZK for watch 5: [__consumer_offsets] 2021-01-08 08:33:20,29465 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicConfigsWatcher:98 - Unwatching znode __consumer_offsets for changes 2021-01-08 08:33:20,29505 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicWatcher:98 - Unwatching znode __consumer_offsets for changes 2021-01-08 08:33:20,29594 DEBUG [ZkClient-EventThread-19-localhost:2181] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094860295, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:20,29615 DEBUG [ZkClient-EventThread-19-localhost:2181] Util:94 - Waiting for 44|/brokers/topics 5:-__consumer_offsets to get confirmed non-existence 2021-01-08 08:33:20,29652 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=79 2021-01-08 08:33:20,29720 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=79) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:33:20,29730 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:143 - Topics created in ZK for watch 5: [] 2021-01-08 08:33:20,29749 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #10 done 2021-01-08 08:33:20,29758 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #11 ZkEvent[Data of /config/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@5bed488a] 2021-01-08 08:33:20,30228 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 79,3 replyHeader:: 79,4294968564,-101 request:: '/config/topics/__consumer_offsets,T response:: 2021-01-08 08:33:20,30418 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 80,4 replyHeader:: 80,4294968564,-101 request:: '/config/topics/__consumer_offsets,T response:: 2021-01-08 08:33:20,30449 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #11 done 2021-01-08 08:33:20,30687 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=79): org.apache.kafka.common.requests.CreateTopicsResponse@1aeec0dc 2021-01-08 08:33:20,38125 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:33:20,38150 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeCreated path:/config/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:33:20,38196 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeCreated path:/config/topics/__consumer_offsets 2021-01-08 08:33:20,38215 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:33:20,38941 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:33:20,38961 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeCreated path:/brokers/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:33:20,38987 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:33:20,38999 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics for sessionid 0x101a07e36350011 2021-01-08 08:33:20,39002 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeCreated path:/brokers/topics/__consumer_offsets 2021-01-08 08:33:20,39022 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:33:20,39030 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics 2021-01-08 08:33:20,39043 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:33:20,39062 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:33:20,39076 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #12 ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:33:20,39353 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 81,3 replyHeader:: 81,4294968568,0 request:: '/brokers/topics,T response:: s{4294967304,4294967304,1610094013731,1610094013731,0,9,0,0,0,3,4294968567} 2021-01-08 08:33:20,39540 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 82,8 replyHeader:: 82,4294968568,0 request:: '/brokers/topics,T response:: v{'my-topic,'__consumer_offsets,'my-topic-reversed} 2021-01-08 08:33:20,39574 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:115 - 6: znode /brokers/topics now has children [my-topic, __consumer_offsets, my-topic-reversed], previous children [my-topic, my-topic-reversed] 2021-01-08 08:33:20,39591 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:127 - Topics deleted from ZK for watch 6: [] 2021-01-08 08:33:20,39599 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:143 - Topics created in ZK for watch 6: [__consumer_offsets] 2021-01-08 08:33:20,39610 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicConfigsWatcher:73 - Watching znode /config/topics/__consumer_offsets for changes 2021-01-08 08:33:20,39652 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicWatcher:73 - Watching znode /brokers/topics/__consumer_offsets for changes 2021-01-08 08:33:20,39724 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:957 - Metric onTopicCreated triggered 2021-01-08 08:33:20,39737 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:460 - 45|/brokers/topics 6:+__consumer_offsets: Queuing action onTopicCreated on topic __consumer_offsets 2021-01-08 08:33:20,39746 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:463 - 45|/brokers/topics 6:+__consumer_offsets: Adding first waiter onTopicCreated 2021-01-08 08:33:20,39813 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #12 done 2021-01-08 08:33:20,39838 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 45|/brokers/topics 6:+__consumer_offsets: Lock acquired 2021-01-08 08:33:20,39847 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 45|/brokers/topics 6:+__consumer_offsets: Executing action onTopicCreated on topic __consumer_offsets 2021-01-08 08:33:20,39878 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094860398, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:20,39898 DEBUG [vert.x-eventloop-thread-1] Util:94 - Waiting for 45|/brokers/topics 6:+__consumer_offsets to get confirmed existence 2021-01-08 08:33:20,39939 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=80 2021-01-08 08:33:20,39987 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=80) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:33:20,40383 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 83,3 replyHeader:: 83,4294968569,0 request:: '/config/topics/__consumer_offsets,T response:: s{4294968566,4294968568,1610094800375,1610094800388,1,0,0,0,109,0,4294968566} 2021-01-08 08:33:20,40430 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-0] ZkClient:196 - Subscribed data changes for /config/topics/__consumer_offsets 2021-01-08 08:33:20,40490 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 84,3 replyHeader:: 84,4294968569,0 request:: '/brokers/topics/__consumer_offsets,T response:: s{4294968567,4294968567,1610094800384,1610094800384,0,0,0,0,712,0,4294968567} 2021-01-08 08:33:20,40527 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-1] ZkClient:196 - Subscribed data changes for /brokers/topics/__consumer_offsets 2021-01-08 08:33:20,40742 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 85,4 replyHeader:: 85,4294968569,0 request:: '/config/topics/__consumer_offsets,T response:: #7b2276657273696f6e223a312c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294968566,4294968568,1610094800375,1610094800388,1,0,0,0,109,0,4294968566} 2021-01-08 08:33:20,40820 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 86,4 replyHeader:: 86,4294968569,0 request:: '/brokers/topics/__consumer_offsets,T response:: #7b2276657273696f6e223a322c22706172746974696f6e73223a7b223330223a5b312c302c325d2c223339223a5b312c322c305d2c223435223a5b312c322c305d2c2232223a5b302c322c315d2c2235223a5b302c312c325d2c223438223a5b312c302c325d2c223333223a5b312c322c305d2c223237223a5b312c322c305d2c223132223a5b312c302c325d2c2238223a5b302c322c315d2c223135223a5b312c322c305d2c223432223a5b312c302c325d2c223336223a5b312c302c325d2c223231223a5b312c322c305d2c223138223a5b312c302c325d2c223234223a5b312c302c325d2c223335223a5b302c312c325d2c223431223a5b302c312c325d2c2237223a5b322c312c305d2c223137223a5b302c312c325d2c2231223a5b322c312c305d2c223434223a5b302c322c315d2c223233223a5b302c312c325d2c223338223a5b302c322c315d2c223437223a5b302c312c325d2c2234223a5b322c302c315d2c223236223a5b302c322c315d2c223131223a5b302c312c325d2c223332223a5b302c322c315d2c223134223a5b302c322c315d2c223230223a5b302c322c315d2c223239223a5b302c312c325d2c223436223a5b322c302c315d2c223334223a5b322c302c315d2c223238223a5b322c302c315d2c2236223a5b312c302c325d2c223430223a5b322c302c315d2c223439223a5b322c312c305d2c2239223a5b312c322c305d2c223433223a5b322c312c305d2c2230223a5b312c302c325d2c223232223a5b322c302c315d2c223136223a5b322c302c315d2c223337223a5b322c312c305d2c223139223a5b322c312c305d2c2233223a5b312c322c305d2c223130223a5b322c302c315d2c223331223a5b322c312c305d2c223235223a5b322c312c305d2c223133223a5b322c312c305d7d2c22616464696e675f7265706c69636173223a7b7d2c2272656d6f76696e675f7265706c69636173223a7b7d7d,s{4294968567,4294968567,1610094800384,1610094800384,0,0,0,0,712,0,4294968567} 2021-01-08 08:33:20,44322 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=80): org.apache.kafka.common.requests.CreateTopicsResponse@c1565a5 2021-01-08 08:33:21,30068 DEBUG [vert.x-eventloop-thread-1] Util:116 - 44|/brokers/topics 5:-__consumer_offsets is confirmed non-existence 2021-01-08 08:33:21,30090 DEBUG [vert.x-eventloop-thread-1] TopicOperator:957 - Metric onTopicDeleted triggered 2021-01-08 08:33:21,30103 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 44|/brokers/topics 5:-__consumer_offsets: Queuing action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:33:21,30113 DEBUG [vert.x-eventloop-thread-1] TopicOperator:466 - 44|/brokers/topics 5:-__consumer_offsets: Adding waiter onTopicDeleted: 2 2021-01-08 08:33:21,40408 DEBUG [vert.x-eventloop-thread-1] Util:116 - 45|/brokers/topics 6:+__consumer_offsets is confirmed existence 2021-01-08 08:33:21,40430 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:33:21,40460 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094861404, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:21,40517 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=81 2021-01-08 08:33:21,40545 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=81) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:33:21,41309 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=81): org.apache.kafka.common.requests.CreateTopicsResponse@64187323 2021-01-08 08:33:21,41431 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094861414, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:21,41453 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094861414, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:33:21,41507 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=82 2021-01-08 08:33:21,41529 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=82) and timeout 30000 to node 1: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:33:21,47411 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=82): org.apache.kafka.common.requests.MetadataResponse@155e44b2 2021-01-08 08:33:21,47524 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@795a8ee5 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=83 2021-01-08 08:33:21,47544 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=83) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:33:21,47671 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 45|/brokers/topics 6:+__consumer_offsets: Executing handler for action onTopicCreated on topic __consumer_offsets 2021-01-08 08:33:21,47686 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 45|/brokers/topics 6:+__consumer_offsets: No KafkaTopic to set status 2021-01-08 08:33:21,47700 WARN [vert.x-eventloop-thread-1] ZkTopicsWatcher:153 - 45|/brokers/topics 6:+__consumer_offsets: Error responding to creation of topic __consumer_offsets org.apache.kafka.common.errors.LeaderNotAvailableException: There is no leader for this topic-partition as we are in the middle of a leadership election. 2021-01-08 08:33:21,47745 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 45|/brokers/topics 6:+__consumer_offsets: Lock released 2021-01-08 08:33:21,47759 DEBUG [vert.x-eventloop-thread-1] TopicOperator:452 - 45|/brokers/topics 6:+__consumer_offsets: Removing waiter onTopicCreated, 1 waiters left 2021-01-08 08:33:21,47767 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=83): org.apache.kafka.common.requests.DescribeConfigsResponse@5e45e426 2021-01-08 08:33:21,47769 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 44|/brokers/topics 5:-__consumer_offsets: Lock acquired 2021-01-08 08:33:21,47780 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 44|/brokers/topics 5:-__consumer_offsets: Executing action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:33:21,48393 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 87,4 replyHeader:: 87,4294968670,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:33:21,54892 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 44|/brokers/topics 5:-__consumer_offsets: Reconciling topic null, k8sTopic:null, kafkaTopic:null, privateTopic:null 2021-01-08 08:33:21,54913 DEBUG [vert.x-eventloop-thread-1] TopicOperator:544 - 44|/brokers/topics 5:-__consumer_offsets: All three topics null during reconciliation. 2021-01-08 08:33:21,54923 DEBUG [vert.x-eventloop-thread-1] TopicOperator:973 - succeeded reconciliation onTopicDeleted 2021-01-08 08:33:21,54936 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 44|/brokers/topics 5:-__consumer_offsets: Executing handler for action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:33:21,54943 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 44|/brokers/topics 5:-__consumer_offsets: No KafkaTopic to set status 2021-01-08 08:33:21,54959 DEBUG [vert.x-eventloop-thread-1] ZkTopicsWatcher:135 - 44|/brokers/topics 5:-__consumer_offsets: Success responding to deletion of topic __consumer_offsets 2021-01-08 08:33:21,54969 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 44|/brokers/topics 5:-__consumer_offsets: Lock released 2021-01-08 08:33:21,54979 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 44|/brokers/topics 5:-__consumer_offsets: Removing last waiter onTopicDeleted 2021-01-08 08:33:28,15107 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:33:34,82420 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:33:41,49265 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:33:48,16464 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:33:54,83877 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 2ms 2021-01-08 08:34:01,50984 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:34:08,18057 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:34:14,85309 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:34:21,52518 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:34:28,20237 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:34:34,87500 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:34:41,54352 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:34:48,21473 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:34:49,02491 INFO [vert.x-eventloop-thread-1] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:34:49,02514 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:127 - Listing topics 2021-01-08 08:34:49,02536 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610094949025, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:34:49,02600 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=84 2021-01-08 08:34:49,02628 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=84) and timeout 30000 to node 1: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:34:49,03056 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=84): org.apache.kafka.common.requests.MetadataResponse@79e315a6 2021-01-08 08:34:49,03188 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:34:49,03211 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 46|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:34:49,03223 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 46|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:34:49,03252 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 47|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:34:49,03264 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 47|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:34:49,03285 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 48|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:34:49,03295 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 48|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:34:49,03322 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 46|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:34:49,03332 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 46|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:34:49,03394 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 47|periodic kafka my-topic: Lock acquired 2021-01-08 08:34:49,03405 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 47|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:34:49,03426 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 48|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:34:49,03435 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 48|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:34:49,03601 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 88,4 replyHeader:: 88,4294968670,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:34:49,03778 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 89,4 replyHeader:: 89,4294968670,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:34:49,03785 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 46|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:34:49,04135 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 47|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:34:49,04198 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 90,4 replyHeader:: 90,4294968670,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:34:49,04297 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1364 - 48|periodic kafka __consumer_offsets: No private topic for topic __consumer_offsets in Kafka -> undetermined 2021-01-08 08:34:49,04315 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 48|periodic kafka __consumer_offsets: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:34:49,04324 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 48|periodic kafka __consumer_offsets: No KafkaTopic to set status 2021-01-08 08:34:49,04337 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 48|periodic kafka __consumer_offsets: Lock released 2021-01-08 08:34:49,04348 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 48|periodic kafka __consumer_offsets: Removing last waiter reconcile-from-kafka 2021-01-08 08:34:49,05629 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 46|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:34:49,05661 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:34:49,05688 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094949056, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:34:49,05767 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=85 2021-01-08 08:34:49,05803 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=85) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:34:49,06144 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=85): org.apache.kafka.common.requests.CreateTopicsResponse@1e7fa5fc 2021-01-08 08:34:49,06239 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 47|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:34:49,06264 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:34:49,06272 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094949062, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:34:49,06290 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094949062, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:34:49,06302 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094949062, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:34:49,06316 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=86 2021-01-08 08:34:49,06338 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=86) and timeout 30000 to node 1: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:34:49,06426 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=87 2021-01-08 08:34:49,06450 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=87) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:34:49,06700 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=87): org.apache.kafka.common.requests.CreateTopicsResponse@337f2d70 2021-01-08 08:34:49,06735 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=86): org.apache.kafka.common.requests.MetadataResponse@42ea9bed 2021-01-08 08:34:49,06896 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@2345e833 to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=88 2021-01-08 08:34:49,06915 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=88) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:34:49,06974 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094949069, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:34:49,06998 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094949069, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:34:49,07013 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=89 2021-01-08 08:34:49,07031 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=89) and timeout 30000 to node 0: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:34:49,07114 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@40f6f9e2 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=90 2021-01-08 08:34:49,07132 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=90) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:34:49,07390 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=89): org.apache.kafka.common.requests.MetadataResponse@a285821 2021-01-08 08:34:49,07711 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=88): org.apache.kafka.common.requests.DescribeConfigsResponse@1c8b9f7a 2021-01-08 08:34:49,07817 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 46|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:34:49,07835 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 46|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:34:49,07855 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 46|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:34:49,07869 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 46|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:34:49,07882 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 46|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:34:49,07893 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 46|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:34:49,07893 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=90): org.apache.kafka.common.requests.DescribeConfigsResponse@1083f8a8 2021-01-08 08:34:49,07907 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:34:49,07922 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:34:49,07978 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 46|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:34:49,07993 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 46|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:34:49,08111 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:34:49.080Z"} 2021-01-08 08:34:49,08114 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 47|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:34:49,08168 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 46|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:34:49,08185 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 46|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:34:49,08214 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 47|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:34:49,08238 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 47|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:34:49,08255 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 47|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:34:49,08274 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 47|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:34:49,08285 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 47|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:34:49,08299 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:34:49,08315 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:34:49,08332 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 47|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:34:49,08346 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 47|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:34:49,08460 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:34:49.083515Z"} 2021-01-08 08:34:49,08528 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 47|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:34:49,08594 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 47|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:34:49,09401 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 49|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:34:49,09429 DEBUG [vert.x-eventloop-thread-1] TopicOperator:957 - Metric reconcile-with-kube triggered 2021-01-08 08:34:49,09442 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 49|periodic kube consumer-offsets|1210026: Queuing action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:34:49,09456 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 49|periodic kube consumer-offsets|1210026: Adding first waiter reconcile-with-kube 2021-01-08 08:34:49,09487 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 50|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:34:49,09504 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 51|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:34:49,09522 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 49|periodic kube consumer-offsets|1210026: Lock acquired 2021-01-08 08:34:49,09530 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 49|periodic kube consumer-offsets|1210026: Executing action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:34:49,09572 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:34:49,09595 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610094949095, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:34:49,09633 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=91 2021-01-08 08:34:49,09664 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=91) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:34:49,09875 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=91): org.apache.kafka.common.requests.CreateTopicsResponse@67cd3ab3 2021-01-08 08:34:49,09983 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610094949099, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:34:49,10006 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610094949099, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:34:49,10048 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@2f6c9f3b to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=92 2021-01-08 08:34:49,10069 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=92) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:34:49,10096 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=93 2021-01-08 08:34:49,10111 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=93) and timeout 30000 to node 1: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:34:49,10392 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 91,4 replyHeader:: 91,4294968670,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:34:49,10460 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=93): org.apache.kafka.common.requests.MetadataResponse@6fdcb5e3 2021-01-08 08:34:49,10686 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=92): org.apache.kafka.common.requests.DescribeConfigsResponse@60154608 2021-01-08 08:34:49,10821 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 49|periodic kube consumer-offsets|1210026: Reconciling topic __consumer_offsets, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:null 2021-01-08 08:34:49,10844 DEBUG [vert.x-eventloop-thread-0] TopicOperator:638 - 49|periodic kube consumer-offsets|1210026: KafkaTopic created in k8s and topic created in kafka, but they're identical => just creating in topicStore 2021-01-08 08:34:49,10858 DEBUG [vert.x-eventloop-thread-0] TopicOperator:639 - 49|periodic kube consumer-offsets|1210026: k8s and kafka versions of topic '__consumer_offsets' are the same 2021-01-08 08:34:49,10873 DEBUG [vert.x-eventloop-thread-0] TopicOperator:769 - Enqueuing event CreateInTopicStore(topicName=__consumer_offsets,ctx=49|periodic kube consumer-offsets|1210026) 2021-01-08 08:34:49,10886 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1171 - Executing CreateInTopicStore(topicName=__consumer_offsets,ctx=49|periodic kube consumer-offsets|1210026) 2021-01-08 08:34:49,10973 DEBUG [vert.x-eventloop-thread-0] ZkTopicStore:90 - create znode /strimzi/topics/__consumer_offsets 2021-01-08 08:34:49,12559 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 92,1 replyHeader:: 92,4294968671,0 request:: '/strimzi/topics/__consumer_offsets,#7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,v{s{15,s{'world,'anyone}}},0 response:: '/strimzi/topics/__consumer_offsets 2021-01-08 08:34:49,12628 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1173 - Completing CreateInTopicStore(topicName=__consumer_offsets,ctx=49|periodic kube consumer-offsets|1210026) 2021-01-08 08:34:49,12644 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1178 - CreateInTopicStore(topicName=__consumer_offsets,ctx=49|periodic kube consumer-offsets|1210026) succeeded 2021-01-08 08:34:49,12656 DEBUG [vert.x-eventloop-thread-0] TopicOperator:973 - succeeded reconciliation reconcile-with-kube 2021-01-08 08:34:49,12671 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 49|periodic kube consumer-offsets|1210026: Executing handler for action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:34:49,12682 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 49|periodic kube consumer-offsets|1210026: There is a KafkaTopic to set status on, rv=1210026, generation=1 2021-01-08 08:34:49,12808 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:34:49.126890Z"} 2021-01-08 08:34:49,12839 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 49|periodic kube consumer-offsets|1210026: Lock released 2021-01-08 08:34:49,12852 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 49|periodic kube consumer-offsets|1210026: Removing last waiter reconcile-with-kube 2021-01-08 08:34:55,78460 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:35:02,45615 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:35:09,12909 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:35:15,80232 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:35:22,47099 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:35:29,14401 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:35:35,81559 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:35:41,42558 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[], allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=94 2021-01-08 08:35:41,42586 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=94) and timeout 30000 to node 1: {topics=[],allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:35:41,42811 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=94): org.apache.kafka.common.requests.MetadataResponse@6e83e01b 2021-01-08 08:35:41,42849 DEBUG [kafka-admin-client-thread | adminclient-1] AdminMetadataManager:247 - [AdminClient clientId=adminclient-1] Updating cluster metadata to Cluster(id = 24eqSYNVTzaDk8st3RVFrQ, nodes = [my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null), my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null), my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null)], partitions = [], controller = my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null)) 2021-01-08 08:35:42,48878 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:35:49,16211 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:35:55,83485 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:36:02,50422 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:36:09,17499 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:36:15,84822 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:36:19,13091 INFO [vert.x-eventloop-thread-0] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:36:19,13114 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:127 - Listing topics 2021-01-08 08:36:19,13135 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610095039131, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:19,13192 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=95 2021-01-08 08:36:19,13217 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=95) and timeout 30000 to node 1: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:36:19,13688 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=95): org.apache.kafka.common.requests.MetadataResponse@25dd3fb7 2021-01-08 08:36:19,13830 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:36:19,13868 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 52|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:36:19,13883 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 52|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:36:19,13916 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 53|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:36:19,13929 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 53|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:36:19,13948 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 54|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:36:19,13960 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 54|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:36:19,13983 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 52|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:36:19,13994 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 52|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:36:19,14065 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 53|periodic kafka my-topic: Lock acquired 2021-01-08 08:36:19,14076 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 53|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:36:19,14098 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 54|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:36:19,14109 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 54|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:36:19,14283 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 93,4 replyHeader:: 93,4294968673,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:36:19,14449 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 94,4 replyHeader:: 94,4294968673,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:36:19,14486 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 52|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:36:19,14620 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 95,4 replyHeader:: 95,4294968673,0 request:: '/strimzi/topics/__consumer_offsets,F response:: #7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294968671,4294968671,1610094889111,1610094889111,0,0,0,0,233,0,4294968671} 2021-01-08 08:36:19,14667 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 53|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:36:19,14785 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 54|periodic kafka __consumer_offsets: Have private topic for topic __consumer_offsets in Kafka 2021-01-08 08:36:19,15887 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 52|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:36:19,15913 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:36:19,15945 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095039159, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:19,15997 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=96 2021-01-08 08:36:19,16024 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=96) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:36:19,16313 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=96): org.apache.kafka.common.requests.CreateTopicsResponse@2f24f542 2021-01-08 08:36:19,16503 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 53|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:36:19,16531 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:36:19,16560 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095039165, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:19,16658 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095039166, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:19,16661 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=97 2021-01-08 08:36:19,16683 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095039166, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:19,16694 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=97) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:36:19,16813 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=98 2021-01-08 08:36:19,16833 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=98) and timeout 30000 to node 1: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:36:19,16942 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=97): org.apache.kafka.common.requests.CreateTopicsResponse@654bd3a8 2021-01-08 08:36:19,17351 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=98): org.apache.kafka.common.requests.MetadataResponse@14b96422 2021-01-08 08:36:19,17440 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@58d5c185 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=99 2021-01-08 08:36:19,17465 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=99) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:36:19,17493 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095039174, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:19,17518 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095039175, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:19,17546 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:36:19,17557 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=100 2021-01-08 08:36:19,17570 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095039175, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:19,17578 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=100) and timeout 30000 to node 2: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:36:19,17608 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@5053a138 to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=101 2021-01-08 08:36:19,17623 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=101) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:36:19,17917 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=100): org.apache.kafka.common.requests.MetadataResponse@6bbeee09 2021-01-08 08:36:19,18062 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=102 2021-01-08 08:36:19,18094 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=102) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:36:19,18314 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=102): org.apache.kafka.common.requests.CreateTopicsResponse@34f90f28 2021-01-08 08:36:19,18536 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095039185, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:19,18561 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095039185, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:19,18621 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=101): org.apache.kafka.common.requests.DescribeConfigsResponse@42d629db 2021-01-08 08:36:19,18648 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=99): org.apache.kafka.common.requests.DescribeConfigsResponse@f411ad8 2021-01-08 08:36:19,18735 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 53|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:36:19,18745 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 52|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:36:19,18753 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 53|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:36:19,18747 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=103 2021-01-08 08:36:19,18767 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 52|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:36:19,18777 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 53|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:36:19,18789 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 52|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:36:19,18787 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=103) and timeout 30000 to node 1: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:36:19,18793 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 53|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:36:19,18806 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 52|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:36:19,18809 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 53|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:36:19,18822 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 52|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:36:19,18825 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 53|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:36:19,18834 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 52|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:36:19,18841 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:36:19,18850 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:36:19,18858 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:36:19,18864 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:36:19,18875 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 53|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:36:19,18880 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 52|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:36:19,18890 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 53|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:36:19,18895 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 52|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:36:19,19011 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:36:19.188988Z"} 2021-01-08 08:36:19,19025 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:36:19.189041Z"} 2021-01-08 08:36:19,19033 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 53|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:36:19,19101 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 52|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:36:19,19119 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 53|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:36:19,19127 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 52|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:36:19,19220 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=103): org.apache.kafka.common.requests.MetadataResponse@1313639f 2021-01-08 08:36:19,19380 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@6fefae9a to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=104 2021-01-08 08:36:19,19433 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=104) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:36:19,19992 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=104): org.apache.kafka.common.requests.DescribeConfigsResponse@1c84ca4d 2021-01-08 08:36:19,20102 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 54|periodic kafka __consumer_offsets: Reconciling topic __consumer_offsets, k8sTopic:null, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:36:19,20118 DEBUG [vert.x-eventloop-thread-0] TopicOperator:584 - 54|periodic kafka __consumer_offsets: KafkaTopic deleted in k8s => delete topic from kafka and from topicStore 2021-01-08 08:36:19,20132 DEBUG [vert.x-eventloop-thread-0] TopicOperator:769 - Enqueuing event DeleteKafkaTopic(topicName=__consumer_offsets,ctx=54|periodic kafka __consumer_offsets) 2021-01-08 08:36:19,20146 INFO [vert.x-eventloop-thread-0] TopicOperator:364 - 54|periodic kafka __consumer_offsets: Deleting topic '__consumer_offsets' 2021-01-08 08:36:19,20157 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:57 - Deleting topic __consumer_offsets 2021-01-08 08:36:19,20180 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=deleteTopics, deadlineMs=1610095039201, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:19,20211 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending DeleteTopicsRequestData(topicNames=[__consumer_offsets], timeoutMs=30000) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=105 2021-01-08 08:36:19,20230 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DELETE_TOPICS request with header RequestHeader(apiKey=DELETE_TOPICS, apiVersion=5, clientId=adminclient-1, correlationId=105) and timeout 30000 to node 2: {topic_names=[__consumer_offsets],timeout_ms=30000,_tagged_fields={}} 2021-01-08 08:36:19,28296 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DELETE_TOPICS response from node 2 for request with header RequestHeader(apiKey=DELETE_TOPICS, apiVersion=5, clientId=adminclient-1, correlationId=105): org.apache.kafka.common.requests.DeleteTopicsResponse@1b9494df 2021-01-08 08:36:19,28395 DEBUG [vert.x-eventloop-thread-1] TopicOperator:769 - Enqueuing event DeleteFromTopicStore(topicName=__consumer_offsets,ctx=54|periodic kafka __consumer_offsets) 2021-01-08 08:36:19,28415 DEBUG [vert.x-eventloop-thread-1] ZkTopicStore:117 - delete znode /strimzi/topics/__consumer_offsets 2021-01-08 08:36:19,29184 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 96,2 replyHeader:: 96,4294968675,0 request:: '/strimzi/topics/__consumer_offsets,-1 response:: null 2021-01-08 08:36:19,29611 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic null 2021-01-08 08:36:19,29630 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - __consumer_offsets reconcile success -> succeeded 2021-01-08 08:36:19,29646 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 54|periodic kafka __consumer_offsets: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:36:19,29660 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 54|periodic kafka __consumer_offsets: No KafkaTopic to set status 2021-01-08 08:36:19,29714 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 54|periodic kafka __consumer_offsets: Lock released 2021-01-08 08:36:19,29728 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 54|periodic kafka __consumer_offsets: Removing last waiter reconcile-from-kafka 2021-01-08 08:36:19,30579 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 55|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:36:19,30609 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 56|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:36:19,30624 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 57|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:36:20,63198 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:36:20,63221 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/brokers/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:36:20,63244 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:36:20,63255 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics for sessionid 0x101a07e36350011 2021-01-08 08:36:20,63256 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeDeleted path:/brokers/topics/__consumer_offsets 2021-01-08 08:36:20,63275 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Data of /brokers/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@6a1c90f7] 2021-01-08 08:36:20,63286 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:36:20,63294 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics 2021-01-08 08:36:20,63305 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:36:20,63304 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #13 ZkEvent[Data of /brokers/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@6a1c90f7] 2021-01-08 08:36:20,63315 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:36:20,63575 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:36:20,63588 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/config/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:36:20,63605 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeDeleted path:/config/topics/__consumer_offsets 2021-01-08 08:36:20,63617 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Data of /config/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@3aa30a1b] 2021-01-08 08:36:20,63624 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:36:20,63642 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 97,3 replyHeader:: 97,4294968928,-101 request:: '/brokers/topics/__consumer_offsets,T response:: 2021-01-08 08:36:20,64017 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 98,4 replyHeader:: 98,4294968929,-101 request:: '/brokers/topics/__consumer_offsets,T response:: 2021-01-08 08:36:20,64039 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #13 done 2021-01-08 08:36:20,64048 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #14 ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:36:20,64197 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 99,3 replyHeader:: 99,4294968929,0 request:: '/brokers/topics,T response:: s{4294967304,4294967304,1610094013731,1610094013731,0,10,0,0,0,2,4294968927} 2021-01-08 08:36:20,64326 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 100,8 replyHeader:: 100,4294968929,0 request:: '/brokers/topics,T response:: v{'my-topic,'my-topic-reversed} 2021-01-08 08:36:20,64357 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:115 - 7: znode /brokers/topics now has children [my-topic, my-topic-reversed], previous children [my-topic, __consumer_offsets, my-topic-reversed] 2021-01-08 08:36:20,64373 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:127 - Topics deleted from ZK for watch 7: [__consumer_offsets] 2021-01-08 08:36:20,64383 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicConfigsWatcher:98 - Unwatching znode __consumer_offsets for changes 2021-01-08 08:36:20,64413 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicWatcher:98 - Unwatching znode __consumer_offsets for changes 2021-01-08 08:36:20,64547 DEBUG [ZkClient-EventThread-19-localhost:2181] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095040645, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:20,64578 DEBUG [ZkClient-EventThread-19-localhost:2181] Util:94 - Waiting for 58|/brokers/topics 7:-__consumer_offsets to get confirmed non-existence 2021-01-08 08:36:20,64605 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=106 2021-01-08 08:36:20,64635 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=106) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:36:20,64639 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:143 - Topics created in ZK for watch 7: [] 2021-01-08 08:36:20,64694 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #14 done 2021-01-08 08:36:20,64704 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #15 ZkEvent[Data of /config/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@3aa30a1b] 2021-01-08 08:36:20,64813 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 101,3 replyHeader:: 101,4294968929,-101 request:: '/config/topics/__consumer_offsets,T response:: 2021-01-08 08:36:20,64916 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 102,4 replyHeader:: 102,4294968929,-101 request:: '/config/topics/__consumer_offsets,T response:: 2021-01-08 08:36:20,64936 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #15 done 2021-01-08 08:36:20,65281 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=106): org.apache.kafka.common.requests.CreateTopicsResponse@131303a9 2021-01-08 08:36:20,68390 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:36:20,68407 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeCreated path:/config/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:36:20,68438 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeCreated path:/config/topics/__consumer_offsets 2021-01-08 08:36:20,68452 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:36:20,69170 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:36:20,69182 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeCreated path:/brokers/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:36:20,69209 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:36:20,69219 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics for sessionid 0x101a07e36350011 2021-01-08 08:36:20,69232 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeCreated path:/brokers/topics/__consumer_offsets 2021-01-08 08:36:20,69239 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:36:20,69247 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics 2021-01-08 08:36:20,69258 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:36:20,69267 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:36:20,69277 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #16 ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:36:20,69413 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 103,3 replyHeader:: 103,4294968932,0 request:: '/brokers/topics,T response:: s{4294967304,4294967304,1610094013731,1610094013731,0,11,0,0,0,3,4294968932} 2021-01-08 08:36:20,69534 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 104,8 replyHeader:: 104,4294968932,0 request:: '/brokers/topics,T response:: v{'my-topic,'__consumer_offsets,'my-topic-reversed} 2021-01-08 08:36:20,69565 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:115 - 8: znode /brokers/topics now has children [my-topic, __consumer_offsets, my-topic-reversed], previous children [my-topic, my-topic-reversed] 2021-01-08 08:36:20,69580 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:127 - Topics deleted from ZK for watch 8: [] 2021-01-08 08:36:20,69588 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:143 - Topics created in ZK for watch 8: [__consumer_offsets] 2021-01-08 08:36:20,69599 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicConfigsWatcher:73 - Watching znode /config/topics/__consumer_offsets for changes 2021-01-08 08:36:20,69632 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicWatcher:73 - Watching znode /brokers/topics/__consumer_offsets for changes 2021-01-08 08:36:20,69811 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:957 - Metric onTopicCreated triggered 2021-01-08 08:36:20,69823 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:460 - 59|/brokers/topics 8:+__consumer_offsets: Queuing action onTopicCreated on topic __consumer_offsets 2021-01-08 08:36:20,69834 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:463 - 59|/brokers/topics 8:+__consumer_offsets: Adding first waiter onTopicCreated 2021-01-08 08:36:20,69861 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #16 done 2021-01-08 08:36:20,69883 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 59|/brokers/topics 8:+__consumer_offsets: Lock acquired 2021-01-08 08:36:20,69892 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 59|/brokers/topics 8:+__consumer_offsets: Executing action onTopicCreated on topic __consumer_offsets 2021-01-08 08:36:20,69919 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095040699, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:20,69938 DEBUG [vert.x-eventloop-thread-1] Util:94 - Waiting for 59|/brokers/topics 8:+__consumer_offsets to get confirmed existence 2021-01-08 08:36:20,70036 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=107 2021-01-08 08:36:20,70094 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=107) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:36:20,70258 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 105,3 replyHeader:: 105,4294968932,0 request:: '/config/topics/__consumer_offsets,T response:: s{4294968931,4294968931,1610094980679,1610094980679,0,0,0,0,109,0,4294968931} 2021-01-08 08:36:20,70284 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-0] ZkClient:196 - Subscribed data changes for /config/topics/__consumer_offsets 2021-01-08 08:36:20,70494 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 106,3 replyHeader:: 106,4294968932,0 request:: '/brokers/topics/__consumer_offsets,T response:: s{4294968932,4294968932,1610094980686,1610094980686,0,0,0,0,712,0,4294968932} 2021-01-08 08:36:20,70519 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-1] ZkClient:196 - Subscribed data changes for /brokers/topics/__consumer_offsets 2021-01-08 08:36:20,70605 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 107,4 replyHeader:: 107,4294968932,0 request:: '/config/topics/__consumer_offsets,T response:: #7b2276657273696f6e223a312c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294968931,4294968931,1610094980679,1610094980679,0,0,0,0,109,0,4294968931} 2021-01-08 08:36:20,70835 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 108,4 replyHeader:: 108,4294968933,0 request:: '/brokers/topics/__consumer_offsets,T response:: #7b2276657273696f6e223a322c22706172746974696f6e73223a7b223330223a5b302c312c325d2c223339223a5b302c322c315d2c223435223a5b302c322c315d2c2232223a5b322c302c315d2c2235223a5b322c312c305d2c223438223a5b302c312c325d2c223333223a5b302c322c315d2c223237223a5b302c322c315d2c223132223a5b302c312c325d2c2238223a5b322c302c315d2c223135223a5b302c322c315d2c223432223a5b302c312c325d2c223336223a5b302c312c325d2c223231223a5b302c322c315d2c223138223a5b302c312c325d2c223234223a5b302c312c325d2c223335223a5b322c312c305d2c223431223a5b322c312c305d2c2237223a5b312c322c305d2c223137223a5b322c312c305d2c2231223a5b312c322c305d2c223434223a5b322c302c315d2c223233223a5b322c312c305d2c223338223a5b322c302c315d2c223437223a5b322c312c305d2c2234223a5b312c302c325d2c223236223a5b322c302c315d2c223131223a5b322c312c305d2c223332223a5b322c302c315d2c223134223a5b322c302c315d2c223230223a5b322c302c315d2c223239223a5b322c312c305d2c223436223a5b312c302c325d2c223334223a5b312c302c325d2c223238223a5b312c302c325d2c2236223a5b302c312c325d2c223430223a5b312c302c325d2c223439223a5b312c322c305d2c2239223a5b302c322c315d2c223433223a5b312c322c305d2c2230223a5b302c312c325d2c223232223a5b312c302c325d2c223136223a5b312c302c325d2c223337223a5b312c322c305d2c223139223a5b312c322c305d2c2233223a5b302c322c315d2c223130223a5b312c302c325d2c223331223a5b312c322c305d2c223235223a5b312c322c305d2c223133223a5b312c322c305d7d2c22616464696e675f7265706c69636173223a7b7d2c2272656d6f76696e675f7265706c69636173223a7b7d7d,s{4294968932,4294968932,1610094980686,1610094980686,0,1,0,0,712,1,4294968933} 2021-01-08 08:36:20,70922 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=107): org.apache.kafka.common.requests.CreateTopicsResponse@19af42f 2021-01-08 08:36:21,65317 DEBUG [vert.x-eventloop-thread-1] Util:116 - 58|/brokers/topics 7:-__consumer_offsets is confirmed non-existence 2021-01-08 08:36:21,65340 DEBUG [vert.x-eventloop-thread-1] TopicOperator:957 - Metric onTopicDeleted triggered 2021-01-08 08:36:21,65352 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 58|/brokers/topics 7:-__consumer_offsets: Queuing action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:36:21,65364 DEBUG [vert.x-eventloop-thread-1] TopicOperator:466 - 58|/brokers/topics 7:-__consumer_offsets: Adding waiter onTopicDeleted: 2 2021-01-08 08:36:21,70444 DEBUG [vert.x-eventloop-thread-1] Util:116 - 59|/brokers/topics 8:+__consumer_offsets is confirmed existence 2021-01-08 08:36:21,70467 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:36:21,70494 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095041704, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:21,70549 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=108 2021-01-08 08:36:21,70585 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=108) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:36:21,70819 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=108): org.apache.kafka.common.requests.CreateTopicsResponse@3bf690f3 2021-01-08 08:36:21,70948 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095041709, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:21,70981 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095041709, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:36:21,71049 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=109 2021-01-08 08:36:21,71075 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=109) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:36:21,71107 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@47bc93e to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=110 2021-01-08 08:36:21,71121 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=110) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:36:21,71331 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=110): org.apache.kafka.common.requests.DescribeConfigsResponse@f7e9c53 2021-01-08 08:36:21,71432 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 59|/brokers/topics 8:+__consumer_offsets: Executing handler for action onTopicCreated on topic __consumer_offsets 2021-01-08 08:36:21,71444 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 59|/brokers/topics 8:+__consumer_offsets: No KafkaTopic to set status 2021-01-08 08:36:21,71458 WARN [vert.x-eventloop-thread-1] ZkTopicsWatcher:153 - 59|/brokers/topics 8:+__consumer_offsets: Error responding to creation of topic __consumer_offsets org.apache.kafka.common.errors.UnknownTopicOrPartitionException: 2021-01-08 08:36:21,71490 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 59|/brokers/topics 8:+__consumer_offsets: Lock released 2021-01-08 08:36:21,71500 DEBUG [vert.x-eventloop-thread-1] TopicOperator:452 - 59|/brokers/topics 8:+__consumer_offsets: Removing waiter onTopicCreated, 1 waiters left 2021-01-08 08:36:21,71509 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 58|/brokers/topics 7:-__consumer_offsets: Lock acquired 2021-01-08 08:36:21,71515 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 58|/brokers/topics 7:-__consumer_offsets: Executing action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:36:21,71767 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 109,4 replyHeader:: 109,4294969033,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:36:21,71906 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=109): org.apache.kafka.common.requests.MetadataResponse@59133a1f 2021-01-08 08:36:21,73418 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 58|/brokers/topics 7:-__consumer_offsets: Reconciling topic null, k8sTopic:null, kafkaTopic:null, privateTopic:null 2021-01-08 08:36:21,73436 DEBUG [vert.x-eventloop-thread-1] TopicOperator:544 - 58|/brokers/topics 7:-__consumer_offsets: All three topics null during reconciliation. 2021-01-08 08:36:21,73446 DEBUG [vert.x-eventloop-thread-1] TopicOperator:973 - succeeded reconciliation onTopicDeleted 2021-01-08 08:36:21,73462 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 58|/brokers/topics 7:-__consumer_offsets: Executing handler for action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:36:21,73471 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 58|/brokers/topics 7:-__consumer_offsets: No KafkaTopic to set status 2021-01-08 08:36:21,73484 DEBUG [vert.x-eventloop-thread-1] ZkTopicsWatcher:135 - 58|/brokers/topics 7:-__consumer_offsets: Success responding to deletion of topic __consumer_offsets 2021-01-08 08:36:21,73496 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 58|/brokers/topics 7:-__consumer_offsets: Lock released 2021-01-08 08:36:21,73506 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 58|/brokers/topics 7:-__consumer_offsets: Removing last waiter onTopicDeleted 2021-01-08 08:36:28,39034 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:36:35,05900 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:36:41,73194 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:36:48,40372 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:36:55,07600 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:37:01,74831 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:37:08,41772 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:37:15,09155 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:37:21,76410 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:37:28,43387 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:37:35,10680 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:37:41,77899 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:37:48,45045 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:37:49,39465 INFO [vert.x-eventloop-thread-1] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:37:49,39488 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:127 - Listing topics 2021-01-08 08:37:49,39509 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610095129394, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:37:49,39566 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=111 2021-01-08 08:37:49,39592 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=111) and timeout 30000 to node 0: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:37:49,40090 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=111): org.apache.kafka.common.requests.MetadataResponse@56ed58c6 2021-01-08 08:37:49,40212 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:37:49,40237 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 60|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:37:49,40249 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 60|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:37:49,40276 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 61|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:37:49,40287 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 61|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:37:49,40304 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 62|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:37:49,40314 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 62|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:37:49,40333 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 60|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:37:49,40342 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 60|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:37:49,40400 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 61|periodic kafka my-topic: Lock acquired 2021-01-08 08:37:49,40411 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 61|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:37:49,40432 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 62|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:37:49,40441 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 62|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:37:49,40600 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 110,4 replyHeader:: 110,4294969035,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:37:49,40793 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 60|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:37:49,40815 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 111,4 replyHeader:: 111,4294969035,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:37:49,41120 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 61|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:37:49,41138 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 112,4 replyHeader:: 112,4294969035,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:37:49,41260 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1364 - 62|periodic kafka __consumer_offsets: No private topic for topic __consumer_offsets in Kafka -> undetermined 2021-01-08 08:37:49,41273 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 62|periodic kafka __consumer_offsets: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:37:49,41282 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 62|periodic kafka __consumer_offsets: No KafkaTopic to set status 2021-01-08 08:37:49,41294 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 62|periodic kafka __consumer_offsets: Lock released 2021-01-08 08:37:49,41304 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 62|periodic kafka __consumer_offsets: Removing last waiter reconcile-from-kafka 2021-01-08 08:37:49,41967 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 60|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:37:49,41990 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:37:49,42017 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095129420, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:37:49,42064 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=112 2021-01-08 08:37:49,42096 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=112) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:37:49,42366 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=112): org.apache.kafka.common.requests.CreateTopicsResponse@c85591c 2021-01-08 08:37:49,42492 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095129424, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:37:49,42520 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095129425, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:37:49,42552 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=113 2021-01-08 08:37:49,42570 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 61|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:37:49,42580 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=113) and timeout 30000 to node 1: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:37:49,42591 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:37:49,42613 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095129426, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:37:49,42683 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=114 2021-01-08 08:37:49,42707 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=114) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:37:49,42734 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@7762c0db to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=115 2021-01-08 08:37:49,42767 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=115) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:37:49,42925 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=113): org.apache.kafka.common.requests.MetadataResponse@37e1cfea 2021-01-08 08:37:49,43057 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=114): org.apache.kafka.common.requests.CreateTopicsResponse@bf68ab8 2021-01-08 08:37:49,43139 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095129431, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:37:49,43166 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095129431, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:37:49,43193 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=116 2021-01-08 08:37:49,43216 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=116) and timeout 30000 to node 2: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:37:49,43312 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@2d9d7f63 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=117 2021-01-08 08:37:49,43329 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=117) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:37:49,43485 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=116): org.apache.kafka.common.requests.MetadataResponse@4c980613 2021-01-08 08:37:49,43628 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=115): org.apache.kafka.common.requests.DescribeConfigsResponse@753f0ef5 2021-01-08 08:37:49,43737 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 60|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:37:49,43759 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 60|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:37:49,43777 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 60|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:37:49,43789 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 60|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:37:49,43801 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 60|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:37:49,43810 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 60|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:37:49,43824 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:37:49,43834 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:37:49,43848 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 60|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:37:49,43860 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 60|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:37:49,43970 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:37:49.438669Z"} 2021-01-08 08:37:49,44037 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 60|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:37:49,44062 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 60|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:37:49,44098 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=117): org.apache.kafka.common.requests.DescribeConfigsResponse@4a23e9a0 2021-01-08 08:37:49,44174 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 61|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:37:49,44187 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 61|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:37:49,44205 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 61|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:37:49,44218 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 61|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:37:49,44233 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 61|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:37:49,44242 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 61|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:37:49,44258 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:37:49,44270 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:37:49,44283 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 61|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:37:49,44295 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 61|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:37:49,44408 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:37:49.443011Z"} 2021-01-08 08:37:49,44503 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 61|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:37:49,44526 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 61|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:37:49,45402 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 63|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:37:49,45432 DEBUG [vert.x-eventloop-thread-1] TopicOperator:957 - Metric reconcile-with-kube triggered 2021-01-08 08:37:49,45445 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 63|periodic kube consumer-offsets|1210026: Queuing action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:37:49,45461 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 63|periodic kube consumer-offsets|1210026: Adding first waiter reconcile-with-kube 2021-01-08 08:37:49,45491 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 64|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:37:49,45507 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 65|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:37:49,45525 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 63|periodic kube consumer-offsets|1210026: Lock acquired 2021-01-08 08:37:49,45534 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 63|periodic kube consumer-offsets|1210026: Executing action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:37:49,45578 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:37:49,45604 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095129455, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:37:49,45653 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=118 2021-01-08 08:37:49,45683 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=118) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:37:49,46088 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=118): org.apache.kafka.common.requests.CreateTopicsResponse@c83f1e0 2021-01-08 08:37:49,46266 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095129462, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:37:49,46290 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095129462, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:37:49,46337 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=119 2021-01-08 08:37:49,46357 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=119) and timeout 30000 to node 0: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:37:49,46383 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@a7c6fd to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=120 2021-01-08 08:37:49,46397 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=120) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:37:49,46577 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 113,4 replyHeader:: 113,4294969035,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:37:49,46822 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=119): org.apache.kafka.common.requests.MetadataResponse@d6238ba 2021-01-08 08:37:49,47020 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=120): org.apache.kafka.common.requests.DescribeConfigsResponse@58d9be5f 2021-01-08 08:37:49,47159 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 63|periodic kube consumer-offsets|1210026: Reconciling topic __consumer_offsets, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:null 2021-01-08 08:37:49,47181 DEBUG [vert.x-eventloop-thread-0] TopicOperator:638 - 63|periodic kube consumer-offsets|1210026: KafkaTopic created in k8s and topic created in kafka, but they're identical => just creating in topicStore 2021-01-08 08:37:49,47193 DEBUG [vert.x-eventloop-thread-0] TopicOperator:639 - 63|periodic kube consumer-offsets|1210026: k8s and kafka versions of topic '__consumer_offsets' are the same 2021-01-08 08:37:49,47208 DEBUG [vert.x-eventloop-thread-0] TopicOperator:769 - Enqueuing event CreateInTopicStore(topicName=__consumer_offsets,ctx=63|periodic kube consumer-offsets|1210026) 2021-01-08 08:37:49,47221 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1171 - Executing CreateInTopicStore(topicName=__consumer_offsets,ctx=63|periodic kube consumer-offsets|1210026) 2021-01-08 08:37:49,47304 DEBUG [vert.x-eventloop-thread-0] ZkTopicStore:90 - create znode /strimzi/topics/__consumer_offsets 2021-01-08 08:37:49,48448 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 114,1 replyHeader:: 114,4294969036,0 request:: '/strimzi/topics/__consumer_offsets,#7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,v{s{15,s{'world,'anyone}}},0 response:: '/strimzi/topics/__consumer_offsets 2021-01-08 08:37:49,48525 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1173 - Completing CreateInTopicStore(topicName=__consumer_offsets,ctx=63|periodic kube consumer-offsets|1210026) 2021-01-08 08:37:49,48547 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1178 - CreateInTopicStore(topicName=__consumer_offsets,ctx=63|periodic kube consumer-offsets|1210026) succeeded 2021-01-08 08:37:49,48560 DEBUG [vert.x-eventloop-thread-0] TopicOperator:973 - succeeded reconciliation reconcile-with-kube 2021-01-08 08:37:49,48575 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 63|periodic kube consumer-offsets|1210026: Executing handler for action reconcile-with-kube on topic __consumer_offsets 2021-01-08 08:37:49,48585 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 63|periodic kube consumer-offsets|1210026: There is a KafkaTopic to set status on, rv=1210026, generation=1 2021-01-08 08:37:49,48702 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:37:49.485916Z"} 2021-01-08 08:37:49,48736 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 63|periodic kube consumer-offsets|1210026: Lock released 2021-01-08 08:37:49,48752 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 63|periodic kube consumer-offsets|1210026: Removing last waiter reconcile-with-kube 2021-01-08 08:37:56,14621 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:38:02,81540 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:38:09,48798 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:38:16,16110 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:38:22,82923 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:38:29,50159 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:38:36,17397 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:38:42,84613 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:38:49,51825 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:38:56,19156 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:39:02,86089 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:39:09,53373 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:39:16,20480 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:39:19,48996 INFO [vert.x-eventloop-thread-0] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:39:19,49018 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:127 - Listing topics 2021-01-08 08:39:19,49047 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610095219490, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:19,49112 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=121 2021-01-08 08:39:19,49140 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=121) and timeout 30000 to node 0: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:39:19,49583 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=121): org.apache.kafka.common.requests.MetadataResponse@4bc65855 2021-01-08 08:39:19,49757 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:39:19,49802 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 66|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:39:19,49850 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 66|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:39:19,49905 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 67|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:39:19,49920 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 67|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:39:19,49938 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 68|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:39:19,49950 DEBUG [vert.x-eventloop-thread-1] TopicOperator:463 - 68|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:39:19,49980 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 66|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:39:19,49990 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 66|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:39:19,50035 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 67|periodic kafka my-topic: Lock acquired 2021-01-08 08:39:19,50082 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 67|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:39:19,50107 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 68|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:39:19,50117 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 68|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:39:19,50282 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 115,4 replyHeader:: 115,4294969038,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:39:19,50447 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 116,4 replyHeader:: 116,4294969038,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:39:19,50615 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 117,4 replyHeader:: 117,4294969038,0 request:: '/strimzi/topics/__consumer_offsets,F response:: #7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294969036,4294969036,1610095069474,1610095069474,0,0,0,0,233,0,4294969036} 2021-01-08 08:39:19,50617 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 66|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:39:19,50809 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 67|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:39:19,51035 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1368 - 68|periodic kafka __consumer_offsets: Have private topic for topic __consumer_offsets in Kafka 2021-01-08 08:39:19,52509 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 66|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:39:19,52557 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:39:19,52598 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095219525, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:19,52658 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=122 2021-01-08 08:39:19,52692 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=122) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:39:19,52912 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 67|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:39:19,52937 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:39:19,52962 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095219529, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:19,52969 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=122): org.apache.kafka.common.requests.CreateTopicsResponse@75e8fffe 2021-01-08 08:39:19,53252 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095219532, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:19,53252 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=123 2021-01-08 08:39:19,53280 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095219532, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:19,53289 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=123) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:39:19,53389 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=124 2021-01-08 08:39:19,53409 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=124) and timeout 30000 to node 1: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:39:19,53670 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:39:19,53703 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095219536, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:19,53712 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=124): org.apache.kafka.common.requests.MetadataResponse@6953be11 2021-01-08 08:39:19,53829 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@7c6dde22 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=125 2021-01-08 08:39:19,53850 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=125) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:39:19,53944 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=123): org.apache.kafka.common.requests.CreateTopicsResponse@590250a 2021-01-08 08:39:19,54131 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=126 2021-01-08 08:39:19,54153 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095219541, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:19,54164 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=126) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:39:19,54186 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095219541, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:19,54300 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=127 2021-01-08 08:39:19,54336 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=127) and timeout 30000 to node 0: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:39:19,54436 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=126): org.apache.kafka.common.requests.CreateTopicsResponse@37b52d49 2021-01-08 08:39:19,54496 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095219544, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:19,54518 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095219545, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:19,54602 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=127): org.apache.kafka.common.requests.MetadataResponse@43e2ae98 2021-01-08 08:39:19,54716 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@516cf408 to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=128 2021-01-08 08:39:19,54734 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=128) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:39:19,54759 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@55d0335a to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=129 2021-01-08 08:39:19,54773 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=129) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:39:19,54893 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=125): org.apache.kafka.common.requests.DescribeConfigsResponse@4b1183cc 2021-01-08 08:39:19,54968 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 66|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:39:19,54982 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 66|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:39:19,55000 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 66|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:39:19,55013 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 66|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:39:19,55026 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 66|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:39:19,55035 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 66|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:39:19,55048 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:39:19,55106 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:39:19,55124 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 66|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:39:19,55135 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 66|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:39:19,55299 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:39:19.551451Z"} 2021-01-08 08:39:19,55320 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 66|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:39:19,55334 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 66|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:39:19,55363 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=128): org.apache.kafka.common.requests.DescribeConfigsResponse@1dd17258 2021-01-08 08:39:19,55483 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=129): org.apache.kafka.common.requests.DescribeConfigsResponse@2b2b095e 2021-01-08 08:39:19,55560 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=130 2021-01-08 08:39:19,55573 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 67|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:39:19,55579 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=130) and timeout 30000 to node 0: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:39:19,55588 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 67|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:39:19,55607 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 67|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:39:19,55621 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 67|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:39:19,55633 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 67|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:39:19,55643 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 67|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:39:19,55657 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:39:19,55668 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:39:19,55682 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 67|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:39:19,55694 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 67|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:39:19,55792 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:39:19.557002Z"} 2021-01-08 08:39:19,55810 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 67|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:39:19,55824 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 67|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:39:19,56286 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=130): org.apache.kafka.common.requests.MetadataResponse@7fb5a582 2021-01-08 08:39:19,56464 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 68|periodic kafka __consumer_offsets: Reconciling topic __consumer_offsets, k8sTopic:null, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:39:19,56483 DEBUG [vert.x-eventloop-thread-0] TopicOperator:584 - 68|periodic kafka __consumer_offsets: KafkaTopic deleted in k8s => delete topic from kafka and from topicStore 2021-01-08 08:39:19,56498 DEBUG [vert.x-eventloop-thread-0] TopicOperator:769 - Enqueuing event DeleteKafkaTopic(topicName=__consumer_offsets,ctx=68|periodic kafka __consumer_offsets) 2021-01-08 08:39:19,56513 INFO [vert.x-eventloop-thread-0] TopicOperator:364 - 68|periodic kafka __consumer_offsets: Deleting topic '__consumer_offsets' 2021-01-08 08:39:19,56521 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:57 - Deleting topic __consumer_offsets 2021-01-08 08:39:19,56540 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=deleteTopics, deadlineMs=1610095219565, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:19,56576 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending DeleteTopicsRequestData(topicNames=[__consumer_offsets], timeoutMs=30000) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=131 2021-01-08 08:39:19,56608 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DELETE_TOPICS request with header RequestHeader(apiKey=DELETE_TOPICS, apiVersion=5, clientId=adminclient-1, correlationId=131) and timeout 30000 to node 2: {topic_names=[__consumer_offsets],timeout_ms=30000,_tagged_fields={}} 2021-01-08 08:39:19,59383 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DELETE_TOPICS response from node 2 for request with header RequestHeader(apiKey=DELETE_TOPICS, apiVersion=5, clientId=adminclient-1, correlationId=131): org.apache.kafka.common.requests.DeleteTopicsResponse@387d565b 2021-01-08 08:39:19,59551 DEBUG [vert.x-eventloop-thread-1] TopicOperator:769 - Enqueuing event DeleteFromTopicStore(topicName=__consumer_offsets,ctx=68|periodic kafka __consumer_offsets) 2021-01-08 08:39:19,59596 DEBUG [vert.x-eventloop-thread-1] ZkTopicStore:117 - delete znode /strimzi/topics/__consumer_offsets 2021-01-08 08:39:19,60770 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 118,2 replyHeader:: 118,4294969040,0 request:: '/strimzi/topics/__consumer_offsets,-1 response:: null 2021-01-08 08:39:19,60830 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic null 2021-01-08 08:39:19,60847 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - __consumer_offsets reconcile success -> succeeded 2021-01-08 08:39:19,60864 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 68|periodic kafka __consumer_offsets: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:39:19,60877 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 68|periodic kafka __consumer_offsets: No KafkaTopic to set status 2021-01-08 08:39:19,60932 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 68|periodic kafka __consumer_offsets: Lock released 2021-01-08 08:39:19,60947 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 68|periodic kafka __consumer_offsets: Removing last waiter reconcile-from-kafka 2021-01-08 08:39:19,61730 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 69|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:39:19,61763 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 70|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:39:19,61779 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 71|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:39:20,99501 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:39:20,99542 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/brokers/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:39:20,99573 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:39:20,99582 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics for sessionid 0x101a07e36350011 2021-01-08 08:39:20,99586 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeDeleted path:/brokers/topics/__consumer_offsets 2021-01-08 08:39:20,99610 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Data of /brokers/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@14e4e9d3] 2021-01-08 08:39:20,99620 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:39:20,99626 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics 2021-01-08 08:39:20,99634 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:39:20,99634 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #17 ZkEvent[Data of /brokers/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@14e4e9d3] 2021-01-08 08:39:20,99640 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:39:20,99838 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:39:20,99849 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeDeleted path:/config/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:39:20,99869 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeDeleted path:/config/topics/__consumer_offsets 2021-01-08 08:39:20,99880 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Data of /config/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@25fc4add] 2021-01-08 08:39:20,99886 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:39:20,99895 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 119,3 replyHeader:: 119,4294969293,-101 request:: '/brokers/topics/__consumer_offsets,T response:: 2021-01-08 08:39:21,01309 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 120,4 replyHeader:: 120,4294969294,-101 request:: '/brokers/topics/__consumer_offsets,T response:: 2021-01-08 08:39:21,01357 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #17 done 2021-01-08 08:39:21,01373 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #18 ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:39:21,01480 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 121,3 replyHeader:: 121,4294969294,0 request:: '/brokers/topics,T response:: s{4294967304,4294967304,1610094013731,1610094013731,0,12,0,0,0,2,4294969292} 2021-01-08 08:39:21,01577 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 122,8 replyHeader:: 122,4294969294,0 request:: '/brokers/topics,T response:: v{'my-topic,'my-topic-reversed} 2021-01-08 08:39:21,01604 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:115 - 9: znode /brokers/topics now has children [my-topic, my-topic-reversed], previous children [my-topic, __consumer_offsets, my-topic-reversed] 2021-01-08 08:39:21,01618 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:127 - Topics deleted from ZK for watch 9: [__consumer_offsets] 2021-01-08 08:39:21,01627 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicConfigsWatcher:98 - Unwatching znode __consumer_offsets for changes 2021-01-08 08:39:21,01668 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicWatcher:98 - Unwatching znode __consumer_offsets for changes 2021-01-08 08:39:21,01711 DEBUG [ZkClient-EventThread-19-localhost:2181] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095221016, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:21,01738 DEBUG [ZkClient-EventThread-19-localhost:2181] Util:94 - Waiting for 72|/brokers/topics 9:-__consumer_offsets to get confirmed non-existence 2021-01-08 08:39:21,01765 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=132 2021-01-08 08:39:21,02169 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=132) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:39:21,02187 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:143 - Topics created in ZK for watch 9: [] 2021-01-08 08:39:21,02204 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #18 done 2021-01-08 08:39:21,02212 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #19 ZkEvent[Data of /config/topics/__consumer_offsets changed sent to io.strimzi.operator.topic.zk.ZkImpl$DataWatchAdapter@25fc4add] 2021-01-08 08:39:21,02666 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 123,3 replyHeader:: 123,4294969295,-101 request:: '/config/topics/__consumer_offsets,T response:: 2021-01-08 08:39:21,02762 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 124,4 replyHeader:: 124,4294969295,-101 request:: '/config/topics/__consumer_offsets,T response:: 2021-01-08 08:39:21,02784 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #19 done 2021-01-08 08:39:21,03007 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=132): org.apache.kafka.common.requests.CreateTopicsResponse@2ea8a0be 2021-01-08 08:39:21,03290 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:39:21,03303 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeCreated path:/config/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:39:21,03326 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeCreated path:/config/topics/__consumer_offsets 2021-01-08 08:39:21,03338 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:39:21,04103 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:39:21,04119 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeCreated path:/brokers/topics/__consumer_offsets for sessionid 0x101a07e36350011 2021-01-08 08:39:21,04141 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:846 - Got notification sessionid:0x101a07e36350011 2021-01-08 08:39:21,04149 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:868 - Got WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics for sessionid 0x101a07e36350011 2021-01-08 08:39:21,04182 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeCreated path:/brokers/topics/__consumer_offsets 2021-01-08 08:39:21,04200 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:39:21,04207 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:615 - Received event: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/brokers/topics 2021-01-08 08:39:21,04216 DEBUG [vert.x-worker-thread-0-EventThread] ZkEventThread:89 - New event: ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:39:21,04227 DEBUG [vert.x-worker-thread-0-EventThread] ZkClient:659 - Leaving process event 2021-01-08 08:39:21,04242 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:70 - Delivering event #20 ZkEvent[Children of /brokers/topics changed sent to io.strimzi.operator.topic.zk.ZkImpl$$Lambda$371/0x0000000840389840@9e6be5d] 2021-01-08 08:39:21,04374 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 125,3 replyHeader:: 125,4294969297,0 request:: '/brokers/topics,T response:: s{4294967304,4294967304,1610094013731,1610094013731,0,13,0,0,0,3,4294969297} 2021-01-08 08:39:21,04475 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 126,8 replyHeader:: 126,4294969297,0 request:: '/brokers/topics,T response:: v{'my-topic,'__consumer_offsets,'my-topic-reversed} 2021-01-08 08:39:21,04504 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:115 - 10: znode /brokers/topics now has children [my-topic, __consumer_offsets, my-topic-reversed], previous children [my-topic, my-topic-reversed] 2021-01-08 08:39:21,04518 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:127 - Topics deleted from ZK for watch 10: [] 2021-01-08 08:39:21,04526 INFO [ZkClient-EventThread-19-localhost:2181] ZkTopicsWatcher:143 - Topics created in ZK for watch 10: [__consumer_offsets] 2021-01-08 08:39:21,04537 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicConfigsWatcher:73 - Watching znode /config/topics/__consumer_offsets for changes 2021-01-08 08:39:21,04573 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkTopicWatcher:73 - Watching znode /brokers/topics/__consumer_offsets for changes 2021-01-08 08:39:21,04665 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:957 - Metric onTopicCreated triggered 2021-01-08 08:39:21,04680 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:460 - 73|/brokers/topics 10:+__consumer_offsets: Queuing action onTopicCreated on topic __consumer_offsets 2021-01-08 08:39:21,04688 DEBUG [ZkClient-EventThread-19-localhost:2181] TopicOperator:463 - 73|/brokers/topics 10:+__consumer_offsets: Adding first waiter onTopicCreated 2021-01-08 08:39:21,04716 DEBUG [ZkClient-EventThread-19-localhost:2181] ZkEventThread:80 - Delivering event #20 done 2021-01-08 08:39:21,04728 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 73|/brokers/topics 10:+__consumer_offsets: Lock acquired 2021-01-08 08:39:21,04741 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 73|/brokers/topics 10:+__consumer_offsets: Executing action onTopicCreated on topic __consumer_offsets 2021-01-08 08:39:21,04769 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 127,3 replyHeader:: 127,4294969297,0 request:: '/config/topics/__consumer_offsets,T response:: s{4294969296,4294969296,1610095161027,1610095161027,0,0,0,0,109,0,4294969296} 2021-01-08 08:39:21,04775 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095221047, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:21,04796 DEBUG [vert.x-eventloop-thread-1] Util:94 - Waiting for 73|/brokers/topics 10:+__consumer_offsets to get confirmed existence 2021-01-08 08:39:21,04817 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 128,3 replyHeader:: 128,4294969297,0 request:: '/brokers/topics/__consumer_offsets,T response:: s{4294969297,4294969297,1610095161036,1610095161036,0,0,0,0,712,0,4294969297} 2021-01-08 08:39:21,04832 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-0] ZkClient:196 - Subscribed data changes for /config/topics/__consumer_offsets 2021-01-08 08:39:21,04835 DEBUG [io.strimzi.operator.topic.zk.ZkImpl-1] ZkClient:196 - Subscribed data changes for /brokers/topics/__consumer_offsets 2021-01-08 08:39:21,04868 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=133 2021-01-08 08:39:21,04941 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=133) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:39:21,05125 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 129,4 replyHeader:: 129,4294969297,0 request:: '/config/topics/__consumer_offsets,T response:: #7b2276657273696f6e223a312c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294969296,4294969296,1610095161027,1610095161027,0,0,0,0,109,0,4294969296} 2021-01-08 08:39:21,05517 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 130,4 replyHeader:: 130,4294969298,0 request:: '/brokers/topics/__consumer_offsets,T response:: #7b2276657273696f6e223a322c22706172746974696f6e73223a7b223330223a5b322c302c315d2c223339223a5b322c312c305d2c223435223a5b322c312c305d2c2232223a5b312c322c305d2c2235223a5b312c302c325d2c223438223a5b322c302c315d2c223333223a5b322c312c305d2c223237223a5b322c312c305d2c223132223a5b322c302c315d2c2238223a5b312c322c305d2c223135223a5b322c312c305d2c223432223a5b322c302c315d2c223336223a5b322c302c315d2c223231223a5b322c312c305d2c223138223a5b322c302c315d2c223234223a5b322c302c315d2c223335223a5b312c302c325d2c223431223a5b312c302c325d2c2237223a5b302c312c325d2c223137223a5b312c302c325d2c2231223a5b302c312c325d2c223434223a5b312c322c305d2c223233223a5b312c302c325d2c223338223a5b312c322c305d2c223437223a5b312c302c325d2c2234223a5b302c322c315d2c223236223a5b312c322c305d2c223131223a5b312c302c325d2c223332223a5b312c322c305d2c223134223a5b312c322c305d2c223230223a5b312c322c305d2c223239223a5b312c302c325d2c223436223a5b302c322c315d2c223334223a5b302c322c315d2c223238223a5b302c322c315d2c2236223a5b322c302c315d2c223430223a5b302c322c315d2c223439223a5b302c312c325d2c2239223a5b322c312c305d2c223433223a5b302c312c325d2c2230223a5b322c302c315d2c223232223a5b302c322c315d2c223136223a5b302c322c315d2c223337223a5b302c312c325d2c223139223a5b302c312c325d2c2233223a5b322c312c305d2c223130223a5b302c322c315d2c223331223a5b302c312c325d2c223235223a5b302c312c325d2c223133223a5b302c312c325d7d2c22616464696e675f7265706c69636173223a7b7d2c2272656d6f76696e675f7265706c69636173223a7b7d7d,s{4294969297,4294969297,1610095161036,1610095161036,0,1,0,0,712,1,4294969298} 2021-01-08 08:39:21,05565 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=133): org.apache.kafka.common.requests.CreateTopicsResponse@5dba2dcc 2021-01-08 08:39:22,03042 DEBUG [vert.x-eventloop-thread-1] Util:116 - 72|/brokers/topics 9:-__consumer_offsets is confirmed non-existence 2021-01-08 08:39:22,03067 DEBUG [vert.x-eventloop-thread-1] TopicOperator:957 - Metric onTopicDeleted triggered 2021-01-08 08:39:22,03079 DEBUG [vert.x-eventloop-thread-1] TopicOperator:460 - 72|/brokers/topics 9:-__consumer_offsets: Queuing action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:39:22,03091 DEBUG [vert.x-eventloop-thread-1] TopicOperator:466 - 72|/brokers/topics 9:-__consumer_offsets: Adding waiter onTopicDeleted: 2 2021-01-08 08:39:22,07083 DEBUG [vert.x-eventloop-thread-1] Util:116 - 73|/brokers/topics 10:+__consumer_offsets is confirmed existence 2021-01-08 08:39:22,07103 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:39:22,07133 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095222071, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:22,07195 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=134 2021-01-08 08:39:22,07225 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=134) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:39:22,07435 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=134): org.apache.kafka.common.requests.CreateTopicsResponse@f711a4c 2021-01-08 08:39:22,08002 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095222079, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:22,08032 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095222080, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:22,08092 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@5d6c4a77 to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=135 2021-01-08 08:39:22,08110 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=135) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:39:22,08137 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=136 2021-01-08 08:39:22,08152 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=136) and timeout 30000 to node 0: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:39:22,08712 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=136): org.apache.kafka.common.requests.MetadataResponse@362abc34 2021-01-08 08:39:22,09031 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=135): org.apache.kafka.common.requests.DescribeConfigsResponse@3af23370 2021-01-08 08:39:22,09543 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 131,4 replyHeader:: 131,4294969398,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:39:22,10732 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 73|/brokers/topics 10:+__consumer_offsets: Reconciling topic __consumer_offsets, k8sTopic:null, kafkaTopic:nonnull, privateTopic:null 2021-01-08 08:39:22,10753 DEBUG [vert.x-eventloop-thread-0] TopicOperator:548 - 73|/brokers/topics 10:+__consumer_offsets: topic created in kafka, will create KafkaTopic in k8s and topicStore 2021-01-08 08:39:22,10987 DEBUG [vert.x-eventloop-thread-0] TopicOperator:769 - Enqueuing event CreateResource(topicName=__consumer_offsets,ctx=73|/brokers/topics 10:+__consumer_offsets) 2021-01-08 08:39:22,12771 INFO [OkHttp https://172.30.0.1/...] K8sTopicWatcher:45 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: event ADDED on resource consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a generation=1, labels={strimzi.io/cluster=my-cluster} 2021-01-08 08:39:22,12774 DEBUG [vert.x-worker-thread-1] K8sImpl:54 - KafkaTopic consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a created with version null->1213768 2021-01-08 08:39:22,12928 DEBUG [vert.x-eventloop-thread-0] TopicOperator:769 - Enqueuing event CreateInTopicStore(topicName=__consumer_offsets,ctx=73|/brokers/topics 10:+__consumer_offsets) 2021-01-08 08:39:22,12943 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1171 - Executing CreateInTopicStore(topicName=__consumer_offsets,ctx=73|/brokers/topics 10:+__consumer_offsets) 2021-01-08 08:39:22,12986 DEBUG [OkHttp https://172.30.0.1/...] TopicOperator:460 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: Queuing action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:22,13004 DEBUG [OkHttp https://172.30.0.1/...] TopicOperator:466 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: Adding waiter onResourceEvent: 3 2021-01-08 08:39:22,13031 DEBUG [vert.x-eventloop-thread-0] ZkTopicStore:90 - create znode /strimzi/topics/__consumer_offsets 2021-01-08 08:39:22,14001 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 132,1 replyHeader:: 132,4294969399,0 request:: '/strimzi/topics/__consumer_offsets,#7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,v{s{15,s{'world,'anyone}}},0 response:: '/strimzi/topics/__consumer_offsets 2021-01-08 08:39:22,14087 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1173 - Completing CreateInTopicStore(topicName=__consumer_offsets,ctx=73|/brokers/topics 10:+__consumer_offsets) 2021-01-08 08:39:22,14097 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1178 - CreateInTopicStore(topicName=__consumer_offsets,ctx=73|/brokers/topics 10:+__consumer_offsets) succeeded 2021-01-08 08:39:22,14108 DEBUG [vert.x-eventloop-thread-0] TopicOperator:973 - succeeded reconciliation onTopicCreated 2021-01-08 08:39:22,14122 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 73|/brokers/topics 10:+__consumer_offsets: Executing handler for action onTopicCreated on topic __consumer_offsets 2021-01-08 08:39:22,14133 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 73|/brokers/topics 10:+__consumer_offsets: There is a KafkaTopic to set status on, rv=1213768, generation=1 2021-01-08 08:39:22,14222 DEBUG [vert.x-eventloop-thread-0] StatusDiff:48 - Status differs: {"op":"replace","path":"/","value":{"conditions":[{"type":"Ready","status":"True","lastTransitionTime":"2021-01-08T08:39:22.141397Z"}],"observedGeneration":1}} 2021-01-08 08:39:22,14237 DEBUG [vert.x-eventloop-thread-0] StatusDiff:49 - Current Status path / has value 2021-01-08 08:39:22,14246 DEBUG [vert.x-eventloop-thread-0] StatusDiff:50 - Desired Status path / has value 2021-01-08 08:39:22,15370 DEBUG [kubernetes-ops-pool-0] CrdOperator:117 - Status of KafkaTopic consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a in namespace myproject has been updated 2021-01-08 08:39:22,15436 INFO [OkHttp https://172.30.0.1/...] K8sTopicWatcher:45 - 75|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213769: event MODIFIED on resource consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a generation=1, labels={strimzi.io/cluster=my-cluster} 2021-01-08 08:39:22,15447 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1014 - 73|/brokers/topics 10:+__consumer_offsets: status was set rv=1213769, generation=1, observedGeneration=1 2021-01-08 08:39:22,15466 DEBUG [OkHttp https://172.30.0.1/...] TopicOperator:460 - 75|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213769: Queuing action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:22,15472 DEBUG [vert.x-eventloop-thread-0] ZkTopicsWatcher:151 - 73|/brokers/topics 10:+__consumer_offsets: Success responding to creation of topic __consumer_offsets 2021-01-08 08:39:22,15480 DEBUG [OkHttp https://172.30.0.1/...] TopicOperator:466 - 75|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213769: Adding waiter onResourceEvent: 4 2021-01-08 08:39:22,15544 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 73|/brokers/topics 10:+__consumer_offsets: Lock released 2021-01-08 08:39:22,15562 DEBUG [vert.x-eventloop-thread-0] TopicOperator:452 - 73|/brokers/topics 10:+__consumer_offsets: Removing waiter onTopicCreated, 3 waiters left 2021-01-08 08:39:22,15585 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 72|/brokers/topics 9:-__consumer_offsets: Lock acquired 2021-01-08 08:39:22,15594 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 72|/brokers/topics 9:-__consumer_offsets: Executing action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:39:22,15853 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 133,4 replyHeader:: 133,4294969399,0 request:: '/strimzi/topics/__consumer_offsets,F response:: #7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294969399,4294969399,1610095162131,1610095162131,0,0,0,0,233,0,4294969399} 2021-01-08 08:39:22,16884 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 72|/brokers/topics 9:-__consumer_offsets: Concurrent modification in kube: new version 1213769 2021-01-08 08:39:22,16904 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 72|/brokers/topics 9:-__consumer_offsets|1213769: Reconciling topic __consumer_offsets, k8sTopic:nonnull, kafkaTopic:null, privateTopic:nonnull 2021-01-08 08:39:22,16914 DEBUG [vert.x-eventloop-thread-1] TopicOperator:590 - 72|/brokers/topics 9:-__consumer_offsets|1213769: topic deleted in kafkas => delete KafkaTopic from k8s and from topicStore 2021-01-08 08:39:22,17069 DEBUG [vert.x-eventloop-thread-1] TopicOperator:769 - Enqueuing event DeleteResource(mapName=consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a,ctx=72|/brokers/topics 9:-__consumer_offsets|1213769) 2021-01-08 08:39:22,19216 DEBUG [vert.x-worker-thread-2] Util:94 - Waiting for sync resource deletion consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a to get deleted 2021-01-08 08:39:22,19647 INFO [OkHttp https://172.30.0.1/...] K8sTopicWatcher:45 - 76|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213770: event MODIFIED on resource consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a generation=2, labels={strimzi.io/cluster=my-cluster} 2021-01-08 08:39:22,19672 DEBUG [OkHttp https://172.30.0.1/...] TopicOperator:460 - 76|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213770: Queuing action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:22,19683 DEBUG [OkHttp https://172.30.0.1/...] TopicOperator:466 - 76|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213770: Adding waiter onResourceEvent: 4 2021-01-08 08:39:22,25593 DEBUG [kubernetes-ops-pool-1] K8sImpl:102 - KafkaTopic consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a deleted false 2021-01-08 08:39:22,26482 INFO [OkHttp https://172.30.0.1/...] K8sTopicWatcher:45 - 77|kube -consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213771: event DELETED on resource consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a generation=2, labels={strimzi.io/cluster=my-cluster} 2021-01-08 08:39:22,26505 DEBUG [OkHttp https://172.30.0.1/...] TopicOperator:460 - 77|kube -consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213771: Queuing action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:22,26516 DEBUG [OkHttp https://172.30.0.1/...] TopicOperator:466 - 77|kube -consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213771: Adding waiter onResourceEvent: 5 2021-01-08 08:39:23,27107 DEBUG [kubernetes-ops-pool-2] K8sImpl:102 - KafkaTopic consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a deleted true 2021-01-08 08:39:23,27196 DEBUG [vert.x-eventloop-thread-1] Util:116 - sync resource deletion consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a is deleted 2021-01-08 08:39:23,27221 DEBUG [vert.x-eventloop-thread-1] TopicOperator:769 - Enqueuing event DeleteFromTopicStore(topicName=__consumer_offsets,ctx=72|/brokers/topics 9:-__consumer_offsets|1213769) 2021-01-08 08:39:23,27232 DEBUG [vert.x-eventloop-thread-1] ZkTopicStore:117 - delete znode /strimzi/topics/__consumer_offsets 2021-01-08 08:39:23,28334 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 134,2 replyHeader:: 134,4294969400,0 request:: '/strimzi/topics/__consumer_offsets,-1 response:: null 2021-01-08 08:39:23,28392 DEBUG [vert.x-eventloop-thread-1] TopicOperator:973 - succeeded reconciliation onTopicDeleted 2021-01-08 08:39:23,28414 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 72|/brokers/topics 9:-__consumer_offsets|1213769: Executing handler for action onTopicDeleted on topic __consumer_offsets 2021-01-08 08:39:23,28425 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 72|/brokers/topics 9:-__consumer_offsets|1213769: No KafkaTopic to set status 2021-01-08 08:39:23,28439 DEBUG [vert.x-eventloop-thread-1] ZkTopicsWatcher:135 - 72|/brokers/topics 9:-__consumer_offsets|1213769: Success responding to deletion of topic __consumer_offsets 2021-01-08 08:39:23,28487 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 72|/brokers/topics 9:-__consumer_offsets|1213769: Lock released 2021-01-08 08:39:23,28500 DEBUG [vert.x-eventloop-thread-1] TopicOperator:452 - 72|/brokers/topics 9:-__consumer_offsets|1213769: Removing waiter onTopicDeleted, 4 waiters left 2021-01-08 08:39:23,28511 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: Lock acquired 2021-01-08 08:39:23,28518 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: Executing action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,29827 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:39:23,29856 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095223298, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:23,29899 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=137 2021-01-08 08:39:23,29928 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=137) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:39:23,30159 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 135,4 replyHeader:: 135,4294969400,-101 request:: '/strimzi/topics/__consumer_offsets,F response:: 2021-01-08 08:39:23,30189 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=137): org.apache.kafka.common.requests.CreateTopicsResponse@3d56fd18 2021-01-08 08:39:23,30336 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095223303, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:23,30366 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095223303, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:39:23,30394 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=138 2021-01-08 08:39:23,30418 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=138) and timeout 30000 to node 0: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:39:23,30503 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@2562737 to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=139 2021-01-08 08:39:23,30519 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=139) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:39:23,30714 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=138): org.apache.kafka.common.requests.MetadataResponse@617ee055 2021-01-08 08:39:23,30996 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=139): org.apache.kafka.common.requests.DescribeConfigsResponse@4a3c36b9 2021-01-08 08:39:23,31105 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: Reconciling topic __consumer_offsets, k8sTopic:null, kafkaTopic:nonnull, privateTopic:null 2021-01-08 08:39:23,31121 DEBUG [vert.x-eventloop-thread-1] TopicOperator:548 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: topic created in kafka, will create KafkaTopic in k8s and topicStore 2021-01-08 08:39:23,31135 DEBUG [vert.x-eventloop-thread-1] TopicOperator:769 - Enqueuing event CreateResource(topicName=__consumer_offsets,ctx=74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768) 2021-01-08 08:39:23,32081 INFO [OkHttp https://172.30.0.1/...] K8sTopicWatcher:45 - 78|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213781: event ADDED on resource consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a generation=1, labels={strimzi.io/cluster=my-cluster} 2021-01-08 08:39:23,32043 DEBUG [vert.x-worker-thread-3] K8sImpl:54 - KafkaTopic consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a created with version null->1213781 2021-01-08 08:39:23,32104 DEBUG [OkHttp https://172.30.0.1/...] TopicOperator:460 - 78|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213781: Queuing action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,32115 DEBUG [OkHttp https://172.30.0.1/...] TopicOperator:466 - 78|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213781: Adding waiter onResourceEvent: 5 2021-01-08 08:39:23,32156 DEBUG [vert.x-eventloop-thread-1] TopicOperator:769 - Enqueuing event CreateInTopicStore(topicName=__consumer_offsets,ctx=74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768) 2021-01-08 08:39:23,32179 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1171 - Executing CreateInTopicStore(topicName=__consumer_offsets,ctx=74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768) 2021-01-08 08:39:23,32264 DEBUG [vert.x-eventloop-thread-1] ZkTopicStore:90 - create znode /strimzi/topics/__consumer_offsets 2021-01-08 08:39:23,33537 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 136,1 replyHeader:: 136,4294969401,0 request:: '/strimzi/topics/__consumer_offsets,#7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,v{s{15,s{'world,'anyone}}},0 response:: '/strimzi/topics/__consumer_offsets 2021-01-08 08:39:23,33601 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1173 - Completing CreateInTopicStore(topicName=__consumer_offsets,ctx=74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768) 2021-01-08 08:39:23,33613 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1178 - CreateInTopicStore(topicName=__consumer_offsets,ctx=74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768) succeeded 2021-01-08 08:39:23,33627 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: Executing handler for action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,33637 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: There is a KafkaTopic to set status on, rv=1213781, generation=1 2021-01-08 08:39:23,33731 DEBUG [vert.x-eventloop-thread-1] StatusDiff:48 - Status differs: {"op":"replace","path":"/","value":{"conditions":[{"type":"Ready","status":"True","lastTransitionTime":"2021-01-08T08:39:23.336448Z"}],"observedGeneration":1}} 2021-01-08 08:39:23,33747 DEBUG [vert.x-eventloop-thread-1] StatusDiff:49 - Current Status path / has value 2021-01-08 08:39:23,33759 DEBUG [vert.x-eventloop-thread-1] StatusDiff:50 - Desired Status path / has value 2021-01-08 08:39:23,34715 INFO [OkHttp https://172.30.0.1/...] K8sTopicWatcher:45 - 79|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213783: event MODIFIED on resource consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a generation=1, labels={strimzi.io/cluster=my-cluster} 2021-01-08 08:39:23,34692 DEBUG [kubernetes-ops-pool-3] CrdOperator:117 - Status of KafkaTopic consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a in namespace myproject has been updated 2021-01-08 08:39:23,34739 DEBUG [OkHttp https://172.30.0.1/...] TopicOperator:460 - 79|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213783: Queuing action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,34750 DEBUG [OkHttp https://172.30.0.1/...] TopicOperator:466 - 79|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213783: Adding waiter onResourceEvent: 6 2021-01-08 08:39:23,34784 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1014 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: status was set rv=1213783, generation=1, observedGeneration=1 2021-01-08 08:39:23,34817 INFO [vert.x-eventloop-thread-1] K8sTopicWatcher:49 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: Success processing event ADDED on resource consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a with labels {strimzi.io/cluster=my-cluster} 2021-01-08 08:39:23,34926 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: Lock released 2021-01-08 08:39:23,34939 DEBUG [vert.x-eventloop-thread-1] TopicOperator:452 - 74|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213768: Removing waiter onResourceEvent, 5 waiters left 2021-01-08 08:39:23,34955 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 75|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213769: Lock acquired 2021-01-08 08:39:23,34963 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 75|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213769: Executing action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,36207 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1054 - 75|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213769: last updated generation=1 2021-01-08 08:39:23,36224 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1062 - 75|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213769: Ignoring modification event caused by my own status update on consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a 2021-01-08 08:39:23,36235 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 75|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213769: Executing handler for action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,36244 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 75|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213769: No KafkaTopic to set status 2021-01-08 08:39:23,36261 INFO [vert.x-eventloop-thread-1] K8sTopicWatcher:49 - 75|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213769: Success processing event MODIFIED on resource consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a with labels {strimzi.io/cluster=my-cluster} 2021-01-08 08:39:23,36310 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 75|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213769: Lock released 2021-01-08 08:39:23,36327 DEBUG [vert.x-eventloop-thread-1] TopicOperator:452 - 75|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213769: Removing waiter onResourceEvent, 4 waiters left 2021-01-08 08:39:23,36336 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 76|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213770: Lock acquired 2021-01-08 08:39:23,36345 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 76|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213770: Executing action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,37178 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1054 - 76|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213770: last updated generation=1 2021-01-08 08:39:23,37191 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1062 - 76|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213770: Ignoring modification event caused by my own status update on consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a 2021-01-08 08:39:23,37202 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 76|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213770: Executing handler for action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,37210 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 76|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213770: No KafkaTopic to set status 2021-01-08 08:39:23,37225 INFO [vert.x-eventloop-thread-1] K8sTopicWatcher:49 - 76|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213770: Success processing event MODIFIED on resource consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a with labels {strimzi.io/cluster=my-cluster} 2021-01-08 08:39:23,37267 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 76|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213770: Lock released 2021-01-08 08:39:23,37280 DEBUG [vert.x-eventloop-thread-1] TopicOperator:452 - 76|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213770: Removing waiter onResourceEvent, 3 waiters left 2021-01-08 08:39:23,37289 DEBUG [vert.x-eventloop-thread-1] TopicOperator:472 - 77|kube -consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213771: Lock acquired 2021-01-08 08:39:23,37296 DEBUG [vert.x-eventloop-thread-1] TopicOperator:473 - 77|kube -consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213771: Executing action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,39861 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1054 - 77|kube -consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213771: last updated generation=1 2021-01-08 08:39:23,39876 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1062 - 77|kube -consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213771: Ignoring modification event caused by my own status update on consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a 2021-01-08 08:39:23,39886 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 77|kube -consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213771: Executing handler for action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,39896 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1031 - 77|kube -consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213771: No KafkaTopic to set status 2021-01-08 08:39:23,39911 INFO [vert.x-eventloop-thread-1] K8sTopicWatcher:49 - 77|kube -consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213771: Success processing event DELETED on resource consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a with labels {strimzi.io/cluster=my-cluster} 2021-01-08 08:39:23,39958 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 77|kube -consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213771: Lock released 2021-01-08 08:39:23,39974 DEBUG [vert.x-eventloop-thread-1] TopicOperator:452 - 77|kube -consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213771: Removing waiter onResourceEvent, 2 waiters left 2021-01-08 08:39:23,39974 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 78|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213781: Lock acquired 2021-01-08 08:39:23,39987 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 78|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213781: Executing action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,40505 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1054 - 78|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213781: last updated generation=1 2021-01-08 08:39:23,40520 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1062 - 78|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213781: Ignoring modification event caused by my own status update on consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a 2021-01-08 08:39:23,40535 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 78|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213781: Executing handler for action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,40545 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1031 - 78|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213781: No KafkaTopic to set status 2021-01-08 08:39:23,40566 INFO [vert.x-eventloop-thread-0] K8sTopicWatcher:49 - 78|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213781: Success processing event ADDED on resource consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a with labels {strimzi.io/cluster=my-cluster} 2021-01-08 08:39:23,40606 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 78|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213781: Lock released 2021-01-08 08:39:23,40621 DEBUG [vert.x-eventloop-thread-0] TopicOperator:452 - 78|kube +consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213781: Removing waiter onResourceEvent, 1 waiters left 2021-01-08 08:39:23,40632 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 79|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213783: Lock acquired 2021-01-08 08:39:23,40639 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 79|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213783: Executing action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,41107 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1054 - 79|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213783: last updated generation=1 2021-01-08 08:39:23,41119 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1062 - 79|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213783: Ignoring modification event caused by my own status update on consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a 2021-01-08 08:39:23,41129 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 79|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213783: Executing handler for action onResourceEvent on topic __consumer_offsets 2021-01-08 08:39:23,41141 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1031 - 79|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213783: No KafkaTopic to set status 2021-01-08 08:39:23,41161 INFO [vert.x-eventloop-thread-0] K8sTopicWatcher:49 - 79|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213783: Success processing event MODIFIED on resource consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a with labels {strimzi.io/cluster=my-cluster} 2021-01-08 08:39:23,41173 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 79|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213783: Lock released 2021-01-08 08:39:23,41183 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 79|kube =consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a|1213783: Removing last waiter onResourceEvent 2021-01-08 08:39:29,99667 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:39:36,66812 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:39:43,34075 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:39:50,01511 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:39:56,68768 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:40:03,35921 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:40:10,03202 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:40:16,70505 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:40:23,37662 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:40:30,05010 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:40:36,72271 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:40:41,50679 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[], allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=140 2021-01-08 08:40:41,50704 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=140) and timeout 30000 to node 2: {topics=[],allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:40:41,50915 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=140): org.apache.kafka.common.requests.MetadataResponse@4088bcdb 2021-01-08 08:40:41,50961 DEBUG [kafka-admin-client-thread | adminclient-1] AdminMetadataManager:247 - [AdminClient clientId=adminclient-1] Updating cluster metadata to Cluster(id = 24eqSYNVTzaDk8st3RVFrQ, nodes = [my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null), my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null), my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null)], partitions = [], controller = my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null)) 2021-01-08 08:40:43,39573 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:40:49,67497 INFO [vert.x-eventloop-thread-1] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:40:49,67532 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:127 - Listing topics 2021-01-08 08:40:49,67569 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610095309675, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:40:49,67623 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=141 2021-01-08 08:40:49,67648 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=141) and timeout 30000 to node 0: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:40:49,68095 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=141): org.apache.kafka.common.requests.MetadataResponse@73d44462 2021-01-08 08:40:49,68225 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:40:49,68251 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 80|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:40:49,68265 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 80|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:40:49,68293 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 81|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:40:49,68304 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 81|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:40:49,68322 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 82|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:40:49,68332 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 82|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:40:49,68351 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 80|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:40:49,68362 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 80|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:40:49,68394 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 81|periodic kafka my-topic: Lock acquired 2021-01-08 08:40:49,68403 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 81|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:40:49,68426 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 82|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:40:49,68435 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 82|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:40:49,68805 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 137,4 replyHeader:: 137,4294969401,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:40:49,68828 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:40:49,68944 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 138,4 replyHeader:: 138,4294969401,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:40:49,69000 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 80|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:40:49,69116 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 139,4 replyHeader:: 139,4294969401,0 request:: '/strimzi/topics/__consumer_offsets,F response:: #7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294969401,4294969401,1610095163323,1610095163323,0,0,0,0,233,0,4294969401} 2021-01-08 08:40:49,69252 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 81|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:40:49,69404 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 82|periodic kafka __consumer_offsets: Have private topic for topic __consumer_offsets in Kafka 2021-01-08 08:40:49,72267 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 80|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:40:49,72293 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:40:49,72317 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095309723, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:40:49,72368 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=142 2021-01-08 08:40:49,72397 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=142) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:40:49,72651 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=142): org.apache.kafka.common.requests.CreateTopicsResponse@653e7e05 2021-01-08 08:40:49,72748 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 81|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:40:49,72761 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095309727, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:40:49,72770 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:40:49,72783 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095309727, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:40:49,72791 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095309727, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:40:49,72832 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=143 2021-01-08 08:40:49,72853 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=143) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:40:49,72882 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@4e1c15e5 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=144 2021-01-08 08:40:49,72895 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=144) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:40:49,73128 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=143): org.apache.kafka.common.requests.MetadataResponse@4b771699 2021-01-08 08:40:49,73276 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=145 2021-01-08 08:40:49,73309 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=145) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:40:49,73504 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=145): org.apache.kafka.common.requests.CreateTopicsResponse@1a51bcc1 2021-01-08 08:40:49,73659 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=144): org.apache.kafka.common.requests.DescribeConfigsResponse@c8937d8 2021-01-08 08:40:49,73712 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095309736, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:40:49,73730 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095309737, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:40:49,73795 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@2294a891 to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=146 2021-01-08 08:40:49,73811 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=146) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:40:49,73835 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=147 2021-01-08 08:40:49,73849 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=147) and timeout 30000 to node 1: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:40:49,73863 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 82|periodic kafka __consumer_offsets: Concurrent modification in kube: new version 1213783 2021-01-08 08:40:49,73891 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:40:49,73912 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095309739, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:40:49,73947 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 80|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:40:49,73960 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 80|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:40:49,73978 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 80|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:40:49,73992 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 80|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:40:49,74008 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 80|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:40:49,74018 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 80|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:40:49,74030 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:40:49,74039 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:40:49,74052 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 80|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:40:49,74067 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 80|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:40:49,74171 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:40:49.740725Z"} 2021-01-08 08:40:49,74190 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 80|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:40:49,74198 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=147): org.apache.kafka.common.requests.MetadataResponse@51a4b184 2021-01-08 08:40:49,74206 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 80|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:40:49,74960 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=146): org.apache.kafka.common.requests.DescribeConfigsResponse@4604a8c5 2021-01-08 08:40:49,75069 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=148 2021-01-08 08:40:49,75088 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 81|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:40:49,75096 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=148) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:40:49,75108 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 81|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:40:49,75127 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 81|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:40:49,75142 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 81|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:40:49,75160 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 81|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:40:49,75170 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 81|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:40:49,75183 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:40:49,75194 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:40:49,75210 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 81|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:40:49,75223 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 81|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:40:49,75334 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:40:49.752293Z"} 2021-01-08 08:40:49,75347 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=148): org.apache.kafka.common.requests.CreateTopicsResponse@3f6702a 2021-01-08 08:40:49,75356 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 81|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:40:49,75487 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095309754, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:40:49,75508 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095309754, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:40:49,75552 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=149 2021-01-08 08:40:49,75572 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=149) and timeout 30000 to node 0: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:40:49,75658 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@4a180e5 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=150 2021-01-08 08:40:49,75674 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=150) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:40:49,75763 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 81|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:40:49,75865 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=149): org.apache.kafka.common.requests.MetadataResponse@707d2964 2021-01-08 08:40:49,76205 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=150): org.apache.kafka.common.requests.DescribeConfigsResponse@13dce52e 2021-01-08 08:40:49,76330 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 82|periodic kafka __consumer_offsets|1213783: Reconciling topic __consumer_offsets, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:40:49,76344 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 82|periodic kafka __consumer_offsets|1213783: 3 way diff 2021-01-08 08:40:49,76366 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 82|periodic kafka __consumer_offsets|1213783: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:40:49,76378 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 82|periodic kafka __consumer_offsets|1213783: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:40:49,76393 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 82|periodic kafka __consumer_offsets|1213783: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:40:49,76403 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 82|periodic kafka __consumer_offsets|1213783: All three topics are identical 2021-01-08 08:40:49,76426 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a 2021-01-08 08:40:49,76450 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - __consumer_offsets reconcile success -> succeeded 2021-01-08 08:40:49,76470 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 82|periodic kafka __consumer_offsets|1213783: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:40:49,76483 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 82|periodic kafka __consumer_offsets|1213783: There is a KafkaTopic to set status on, rv=1213783, generation=1 2021-01-08 08:40:49,76594 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:40:49.764893Z"} 2021-01-08 08:40:49,76673 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 82|periodic kafka __consumer_offsets|1213783: Lock released 2021-01-08 08:40:49,76688 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 82|periodic kafka __consumer_offsets|1213783: Removing last waiter reconcile-from-kafka 2021-01-08 08:40:49,77561 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 83|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:40:49,77596 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 84|periodic kube consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a: Concurrent modification in kube: new version 1213783 2021-01-08 08:40:49,77613 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 85|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:40:49,77627 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 86|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:40:56,36418 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:41:03,03073 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:41:09,70206 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:41:16,37213 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:41:23,04357 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:41:29,71421 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:41:36,38669 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:41:43,06066 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:41:49,73477 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:41:56,40871 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:42:03,08037 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:42:09,75286 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:42:16,42541 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:42:19,86472 INFO [vert.x-eventloop-thread-1] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:42:19,86500 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:127 - Listing topics 2021-01-08 08:42:19,86519 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610095399865, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:42:19,86584 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=151 2021-01-08 08:42:19,86605 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=151) and timeout 30000 to node 2: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:42:19,86965 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=151): org.apache.kafka.common.requests.MetadataResponse@231ba05f 2021-01-08 08:42:19,87098 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:42:19,87123 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 87|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:42:19,87134 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 87|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:42:19,87164 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 88|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:42:19,87173 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 88|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:42:19,87190 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 89|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:42:19,87199 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 89|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:42:19,87218 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 87|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:42:19,87227 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 87|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:42:19,87300 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 88|periodic kafka my-topic: Lock acquired 2021-01-08 08:42:19,87310 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 88|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:42:19,87329 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 89|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:42:19,87338 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 89|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:42:19,87484 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 140,4 replyHeader:: 140,4294969403,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:42:19,87631 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 141,4 replyHeader:: 141,4294969403,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:42:19,87662 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 87|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:42:19,87776 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 88|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:42:19,87896 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 142,4 replyHeader:: 142,4294969403,0 request:: '/strimzi/topics/__consumer_offsets,F response:: #7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294969401,4294969401,1610095163323,1610095163323,0,0,0,0,233,0,4294969401} 2021-01-08 08:42:19,88063 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 89|periodic kafka __consumer_offsets: Have private topic for topic __consumer_offsets in Kafka 2021-01-08 08:42:19,89073 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 87|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:42:19,89096 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:42:19,89121 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095399891, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:42:19,89172 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=152 2021-01-08 08:42:19,89197 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=152) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:42:19,89429 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=152): org.apache.kafka.common.requests.CreateTopicsResponse@2b35aa71 2021-01-08 08:42:19,89771 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095399897, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:42:19,89798 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095399897, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:42:19,89841 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@329a807a to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=153 2021-01-08 08:42:19,89846 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 88|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:42:19,89866 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=153) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:42:19,89870 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:42:19,89892 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095399898, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:42:19,89893 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=154 2021-01-08 08:42:19,89908 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=154) and timeout 30000 to node 1: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:42:19,90037 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 89|periodic kafka __consumer_offsets: Concurrent modification in kube: new version 1213783 2021-01-08 08:42:19,90057 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:42:19,90073 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095399900, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:42:19,90314 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=154): org.apache.kafka.common.requests.MetadataResponse@461be692 2021-01-08 08:42:19,90449 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=153): org.apache.kafka.common.requests.DescribeConfigsResponse@38fcaf56 2021-01-08 08:42:19,90687 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=155 2021-01-08 08:42:19,90704 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 87|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:42:19,90712 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=155) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:42:19,90720 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 87|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:42:19,90766 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 87|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:42:19,90782 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 87|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:42:19,90795 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 87|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:42:19,90804 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 87|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:42:19,90816 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:42:19,90826 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:42:19,90840 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 87|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:42:19,90851 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 87|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:42:19,90881 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=155): org.apache.kafka.common.requests.CreateTopicsResponse@3e9cfc1a 2021-01-08 08:42:19,90914 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=156 2021-01-08 08:42:19,90927 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095399909, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:42:19,90933 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=156) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:42:19,90949 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095399909, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:42:19,90960 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:42:19.908585Z"} 2021-01-08 08:42:19,90983 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 87|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:42:19,90996 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 87|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:42:19,91006 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=157 2021-01-08 08:42:19,91023 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=157) and timeout 30000 to node 0: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:42:19,91100 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=156): org.apache.kafka.common.requests.CreateTopicsResponse@58e322e3 2021-01-08 08:42:19,91145 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095399911, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:42:19,91165 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095399911, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:42:19,91202 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@4155701b to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=158 2021-01-08 08:42:19,91217 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=158) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:42:19,91236 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=159 2021-01-08 08:42:19,91249 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=159) and timeout 30000 to node 1: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:42:19,91366 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=157): org.apache.kafka.common.requests.MetadataResponse@1c8ef178 2021-01-08 08:42:19,91473 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@45abb2df to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=160 2021-01-08 08:42:19,91496 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=160) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:42:19,91627 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=159): org.apache.kafka.common.requests.MetadataResponse@41e64b3 2021-01-08 08:42:19,92070 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=158): org.apache.kafka.common.requests.DescribeConfigsResponse@6ea10983 2021-01-08 08:42:19,92310 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 89|periodic kafka __consumer_offsets|1213783: Reconciling topic __consumer_offsets, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:42:19,92325 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 89|periodic kafka __consumer_offsets|1213783: 3 way diff 2021-01-08 08:42:19,92335 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=160): org.apache.kafka.common.requests.DescribeConfigsResponse@54f201e 2021-01-08 08:42:19,92346 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 89|periodic kafka __consumer_offsets|1213783: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:42:19,92364 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 89|periodic kafka __consumer_offsets|1213783: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:42:19,92405 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 89|periodic kafka __consumer_offsets|1213783: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:42:19,92460 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 89|periodic kafka __consumer_offsets|1213783: All three topics are identical 2021-01-08 08:42:19,92472 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 88|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:42:19,92479 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a 2021-01-08 08:42:19,92487 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 88|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:42:19,92494 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - __consumer_offsets reconcile success -> succeeded 2021-01-08 08:42:19,92504 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 88|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:42:19,92510 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 89|periodic kafka __consumer_offsets|1213783: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:42:19,92517 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 88|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:42:19,92525 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 89|periodic kafka __consumer_offsets|1213783: There is a KafkaTopic to set status on, rv=1213783, generation=1 2021-01-08 08:42:19,92530 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 88|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:42:19,92540 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 88|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:42:19,92551 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:42:19,92565 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:42:19,92578 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 88|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:42:19,92589 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 88|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:42:19,92638 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:42:19.925324Z"} 2021-01-08 08:42:19,92864 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 89|periodic kafka __consumer_offsets|1213783: Lock released 2021-01-08 08:42:19,92880 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 89|periodic kafka __consumer_offsets|1213783: Removing last waiter reconcile-from-kafka 2021-01-08 08:42:19,92946 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:42:19.925940Z"} 2021-01-08 08:42:19,93081 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 88|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:42:19,93099 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 88|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:42:19,93818 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 90|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:42:19,93846 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 91|periodic kube consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a: Concurrent modification in kube: new version 1213783 2021-01-08 08:42:19,93865 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 92|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:42:19,93880 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 93|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:42:26,55190 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:42:33,22188 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:42:39,88810 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:42:46,56072 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:42:53,23449 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:42:59,90619 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:43:06,57888 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:43:13,24582 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:43:19,91549 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:43:26,58900 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:43:33,25923 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:43:39,93015 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:43:46,60297 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:43:50,02405 INFO [vert.x-eventloop-thread-1] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:43:50,02426 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:127 - Listing topics 2021-01-08 08:43:50,02446 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610095490024, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:43:50,02535 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=161 2021-01-08 08:43:50,02567 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=161) and timeout 30000 to node 0: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:43:50,03090 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=161): org.apache.kafka.common.requests.MetadataResponse@1dea5722 2021-01-08 08:43:50,03228 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:43:50,03261 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 94|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:43:50,03272 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 94|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:43:50,03297 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 95|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:43:50,03306 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 95|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:43:50,03323 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 96|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:43:50,03332 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 96|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:43:50,03349 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 94|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:43:50,03359 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 94|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:43:50,03418 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 95|periodic kafka my-topic: Lock acquired 2021-01-08 08:43:50,03427 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 95|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:43:50,03447 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 96|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:43:50,03458 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 96|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:43:50,03630 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 143,4 replyHeader:: 143,4294969405,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:43:50,03767 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 144,4 replyHeader:: 144,4294969405,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:43:50,03805 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 94|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:43:50,03890 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 145,4 replyHeader:: 145,4294969405,0 request:: '/strimzi/topics/__consumer_offsets,F response:: #7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294969401,4294969401,1610095163323,1610095163323,0,0,0,0,233,0,4294969401} 2021-01-08 08:43:50,03920 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 95|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:43:50,04026 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 96|periodic kafka __consumer_offsets: Have private topic for topic __consumer_offsets in Kafka 2021-01-08 08:43:50,04522 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 94|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:43:50,04542 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:43:50,04566 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095490045, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:43:50,04620 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=162 2021-01-08 08:43:50,04651 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=162) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:43:50,04892 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=162): org.apache.kafka.common.requests.CreateTopicsResponse@6aa683d0 2021-01-08 08:43:50,04999 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095490049, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:43:50,05025 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095490050, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:43:50,05023 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 95|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:43:50,05037 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=163 2021-01-08 08:43:50,05046 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:43:50,05061 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=163) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:43:50,05068 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095490050, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:43:50,05141 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@cad6216 to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=164 2021-01-08 08:43:50,05171 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=164) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:43:50,05324 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=163): org.apache.kafka.common.requests.MetadataResponse@709efc4d 2021-01-08 08:43:50,05558 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 96|periodic kafka __consumer_offsets: Concurrent modification in kube: new version 1213783 2021-01-08 08:43:50,05566 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=165 2021-01-08 08:43:50,05581 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:43:50,05596 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=165) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:43:50,05603 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095490055, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:43:50,05869 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=165): org.apache.kafka.common.requests.CreateTopicsResponse@3f02e63b 2021-01-08 08:43:50,05971 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=166 2021-01-08 08:43:50,05973 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095490059, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:43:50,06000 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=166) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:43:50,06117 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=164): org.apache.kafka.common.requests.DescribeConfigsResponse@61644a08 2021-01-08 08:43:50,06000 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095490059, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:43:50,06210 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=167 2021-01-08 08:43:50,06213 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 94|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:43:50,06229 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 94|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:43:50,06228 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=167) and timeout 30000 to node 1: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:43:50,06250 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 94|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:43:50,06265 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 94|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:43:50,06277 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 94|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:43:50,06286 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 94|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:43:50,06302 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:43:50,06313 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:43:50,06315 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=166): org.apache.kafka.common.requests.CreateTopicsResponse@6be29e0 2021-01-08 08:43:50,06327 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 94|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:43:50,06405 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 94|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:43:50,06425 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095490064, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:43:50,06443 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095490064, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:43:50,06530 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:43:50.064148Z"} 2021-01-08 08:43:50,06551 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 94|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:43:50,06568 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 94|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:43:50,06602 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=167): org.apache.kafka.common.requests.MetadataResponse@145c1ee 2021-01-08 08:43:50,06723 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@29dedcc3 to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=168 2021-01-08 08:43:50,06744 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=168) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:43:50,06774 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@6aa9e7fa to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=169 2021-01-08 08:43:50,06787 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=169) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:43:50,07298 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=168): org.apache.kafka.common.requests.DescribeConfigsResponse@1daf3266 2021-01-08 08:43:50,07551 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=169): org.apache.kafka.common.requests.DescribeConfigsResponse@324ada8c 2021-01-08 08:43:50,07624 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=170 2021-01-08 08:43:50,07639 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 95|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:43:50,07649 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=170) and timeout 30000 to node 1: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:43:50,07663 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 95|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:43:50,07690 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 95|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:43:50,07705 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 95|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:43:50,07719 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 95|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:43:50,07728 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 95|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:43:50,07740 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:43:50,07750 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:43:50,07767 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 95|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:43:50,07780 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 95|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:43:50,07898 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:43:50.077863Z"} 2021-01-08 08:43:50,07911 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=170): org.apache.kafka.common.requests.MetadataResponse@6dad20c2 2021-01-08 08:43:50,07917 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 95|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:43:50,07931 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 95|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:43:50,08076 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 96|periodic kafka __consumer_offsets|1213783: Reconciling topic __consumer_offsets, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:43:50,08088 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 96|periodic kafka __consumer_offsets|1213783: 3 way diff 2021-01-08 08:43:50,08108 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 96|periodic kafka __consumer_offsets|1213783: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:43:50,08120 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 96|periodic kafka __consumer_offsets|1213783: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:43:50,08132 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 96|periodic kafka __consumer_offsets|1213783: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:43:50,08140 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 96|periodic kafka __consumer_offsets|1213783: All three topics are identical 2021-01-08 08:43:50,08156 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a 2021-01-08 08:43:50,08169 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - __consumer_offsets reconcile success -> succeeded 2021-01-08 08:43:50,08182 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 96|periodic kafka __consumer_offsets|1213783: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:43:50,08194 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 96|periodic kafka __consumer_offsets|1213783: There is a KafkaTopic to set status on, rv=1213783, generation=1 2021-01-08 08:43:50,08287 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:43:50.081992Z"} 2021-01-08 08:43:50,08407 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 96|periodic kafka __consumer_offsets|1213783: Lock released 2021-01-08 08:43:50,08427 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 96|periodic kafka __consumer_offsets|1213783: Removing last waiter reconcile-from-kafka 2021-01-08 08:43:50,09157 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 97|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:43:50,09182 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 98|periodic kube consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a: Concurrent modification in kube: new version 1213783 2021-01-08 08:43:50,09197 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 99|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:43:50,09213 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 100|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:43:56,71283 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 1ms 2021-01-08 08:44:03,37948 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:44:10,05175 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:44:16,72452 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:44:23,39795 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:44:30,07076 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:44:36,74237 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:44:43,41202 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:44:50,08315 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:44:56,75356 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:45:03,42544 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:45:10,09834 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:45:16,77114 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:45:20,17702 INFO [vert.x-eventloop-thread-1] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:45:20,17724 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:127 - Listing topics 2021-01-08 08:45:20,17742 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610095580177, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:45:20,17804 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=171 2021-01-08 08:45:20,17827 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=171) and timeout 30000 to node 0: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:45:20,18189 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=171): org.apache.kafka.common.requests.MetadataResponse@481e9cba 2021-01-08 08:45:20,18311 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:45:20,18334 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 101|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:45:20,18349 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 101|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:45:20,18378 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 102|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:45:20,18388 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 102|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:45:20,18404 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 103|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:45:20,18413 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 103|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:45:20,18430 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 101|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:45:20,18438 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 101|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:45:20,18469 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 102|periodic kafka my-topic: Lock acquired 2021-01-08 08:45:20,18525 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 102|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:45:20,18544 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 103|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:45:20,18556 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 103|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:45:20,18693 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 146,4 replyHeader:: 146,4294969407,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:45:20,18808 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 147,4 replyHeader:: 147,4294969407,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:45:20,18861 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 101|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:45:20,18927 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 148,4 replyHeader:: 148,4294969407,0 request:: '/strimzi/topics/__consumer_offsets,F response:: #7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294969401,4294969401,1610095163323,1610095163323,0,0,0,0,233,0,4294969401} 2021-01-08 08:45:20,18969 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 102|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:45:20,19072 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 103|periodic kafka __consumer_offsets: Have private topic for topic __consumer_offsets in Kafka 2021-01-08 08:45:20,20025 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 101|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:45:20,20048 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:45:20,20075 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095580200, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:45:20,20122 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=172 2021-01-08 08:45:20,20147 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=172) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:45:20,20373 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=172): org.apache.kafka.common.requests.CreateTopicsResponse@2cc6bbdd 2021-01-08 08:45:20,20470 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095580204, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:45:20,20493 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095580204, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:45:20,20512 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=173 2021-01-08 08:45:20,20530 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=173) and timeout 30000 to node 1: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:45:20,21024 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@483c9a73 to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=174 2021-01-08 08:45:20,21047 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 102|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:45:20,21094 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=174) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:45:20,21105 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:45:20,21160 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095580211, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:45:20,21343 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=173): org.apache.kafka.common.requests.MetadataResponse@4c130886 2021-01-08 08:45:20,21659 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=174): org.apache.kafka.common.requests.DescribeConfigsResponse@70d4a6f2 2021-01-08 08:45:20,22073 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 103|periodic kafka __consumer_offsets: Concurrent modification in kube: new version 1213783 2021-01-08 08:45:20,22076 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=175 2021-01-08 08:45:20,22097 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:45:20,22118 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095580221, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:45:20,22265 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=175) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:45:20,22280 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 101|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:45:20,22298 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 101|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:45:20,22321 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 101|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:45:20,22334 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 101|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:45:20,22349 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 101|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:45:20,22360 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 101|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:45:20,22382 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:45:20,22404 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:45:20,22446 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 101|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:45:20,22467 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 101|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:45:20,22472 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=175): org.apache.kafka.common.requests.CreateTopicsResponse@69d5f366 2021-01-08 08:45:20,22526 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=176 2021-01-08 08:45:20,22566 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=176) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:45:20,22570 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095580225, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:45:20,22592 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095580225, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:45:20,22676 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=177 2021-01-08 08:45:20,22678 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:45:20.224932Z"} 2021-01-08 08:45:20,22696 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=177) and timeout 30000 to node 1: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:45:20,22711 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 101|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:45:20,22728 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 101|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:45:20,22822 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=176): org.apache.kafka.common.requests.CreateTopicsResponse@4596032d 2021-01-08 08:45:20,22927 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095580229, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:45:20,22949 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095580229, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:45:20,22997 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 1 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=177): org.apache.kafka.common.requests.MetadataResponse@11956421 2021-01-08 08:45:20,23117 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=178 2021-01-08 08:45:20,23136 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=178) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:45:20,23162 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@4423b3d7 to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=179 2021-01-08 08:45:20,23174 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=179) and timeout 30000 to node 0: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:45:20,23192 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@22b36f6 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=180 2021-01-08 08:45:20,23204 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=180) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:45:20,24349 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 0 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=179): org.apache.kafka.common.requests.DescribeConfigsResponse@7751d2f5 2021-01-08 08:45:20,24386 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=178): org.apache.kafka.common.requests.MetadataResponse@4d5f1452 2021-01-08 08:45:20,24401 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=180): org.apache.kafka.common.requests.DescribeConfigsResponse@14bbab25 2021-01-08 08:45:20,24841 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 103|periodic kafka __consumer_offsets|1213783: Reconciling topic __consumer_offsets, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:45:20,24864 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 103|periodic kafka __consumer_offsets|1213783: 3 way diff 2021-01-08 08:45:20,24886 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 103|periodic kafka __consumer_offsets|1213783: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:45:20,24898 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 103|periodic kafka __consumer_offsets|1213783: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:45:20,24910 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 103|periodic kafka __consumer_offsets|1213783: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:45:20,24919 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 103|periodic kafka __consumer_offsets|1213783: All three topics are identical 2021-01-08 08:45:20,24931 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a 2021-01-08 08:45:20,24943 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - __consumer_offsets reconcile success -> succeeded 2021-01-08 08:45:20,24962 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 103|periodic kafka __consumer_offsets|1213783: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:45:20,24974 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 103|periodic kafka __consumer_offsets|1213783: There is a KafkaTopic to set status on, rv=1213783, generation=1 2021-01-08 08:45:20,25082 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:45:20.249804Z"} 2021-01-08 08:45:20,25089 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 102|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:45:20,25148 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 102|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:45:20,25149 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 103|periodic kafka __consumer_offsets|1213783: Lock released 2021-01-08 08:45:20,25167 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 103|periodic kafka __consumer_offsets|1213783: Removing last waiter reconcile-from-kafka 2021-01-08 08:45:20,25169 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 102|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:45:20,25183 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 102|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:45:20,25195 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 102|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:45:20,25204 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 102|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:45:20,25215 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:45:20,25225 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:45:20,25238 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 102|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:45:20,25250 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 102|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:45:20,25318 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:45:20.252560Z"} 2021-01-08 08:45:20,25474 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 102|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:45:20,25491 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 102|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:45:20,26311 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 104|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:45:20,26341 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 105|periodic kube consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a: Concurrent modification in kube: new version 1213783 2021-01-08 08:45:20,26359 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 106|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:45:20,26378 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 107|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:45:26,86183 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:45:33,53475 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:45:40,20263 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:45:41,52852 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[], allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=181 2021-01-08 08:45:41,52880 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=181) and timeout 30000 to node 2: {topics=[],allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:45:41,53093 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=181): org.apache.kafka.common.requests.MetadataResponse@4f7cb3c2 2021-01-08 08:45:41,53138 DEBUG [kafka-admin-client-thread | adminclient-1] AdminMetadataManager:247 - [AdminClient clientId=adminclient-1] Updating cluster metadata to Cluster(id = 24eqSYNVTzaDk8st3RVFrQ, nodes = [my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null), my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null), my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null)], partitions = [], controller = my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null)) 2021-01-08 08:45:46,87566 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:45:53,54789 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:46:00,21511 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:46:06,88756 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:46:13,55937 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:46:20,23204 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:46:26,90379 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:46:33,57232 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:46:40,24326 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:46:46,91602 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms 2021-01-08 08:46:50,35393 INFO [vert.x-eventloop-thread-1] TopicOperator:1267 - Starting periodic reconciliation 2021-01-08 08:46:50,35412 DEBUG [vert.x-eventloop-thread-1] KafkaImpl:127 - Listing topics 2021-01-08 08:46:50,35430 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=listTopics, deadlineMs=1610095670354, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:46:50,35485 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=null, allowAutoTopicCreation=true, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=182 2021-01-08 08:46:50,35508 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=182) and timeout 30000 to node 2: {topics=null,allow_auto_topic_creation=true,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:46:50,35924 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=182): org.apache.kafka.common.requests.MetadataResponse@36c306a2 2021-01-08 08:46:50,36055 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1347 - Reconciling kafka topics [my-topic-reversed, my-topic, __consumer_offsets] 2021-01-08 08:46:50,36079 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 108|periodic kafka my-topic-reversed: Queuing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:46:50,36089 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 108|periodic kafka my-topic-reversed: Adding first waiter reconcile-from-kafka 2021-01-08 08:46:50,36118 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 109|periodic kafka my-topic: Queuing action reconcile-from-kafka on topic my-topic 2021-01-08 08:46:50,36126 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 109|periodic kafka my-topic: Adding first waiter reconcile-from-kafka 2021-01-08 08:46:50,36143 DEBUG [vert.x-eventloop-thread-0] TopicOperator:460 - 110|periodic kafka __consumer_offsets: Queuing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:46:50,36151 DEBUG [vert.x-eventloop-thread-0] TopicOperator:463 - 110|periodic kafka __consumer_offsets: Adding first waiter reconcile-from-kafka 2021-01-08 08:46:50,36170 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 108|periodic kafka my-topic-reversed: Lock acquired 2021-01-08 08:46:50,36178 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 108|periodic kafka my-topic-reversed: Executing action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:46:50,36209 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 109|periodic kafka my-topic: Lock acquired 2021-01-08 08:46:50,36217 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 109|periodic kafka my-topic: Executing action reconcile-from-kafka on topic my-topic 2021-01-08 08:46:50,36232 DEBUG [vert.x-eventloop-thread-0] TopicOperator:472 - 110|periodic kafka __consumer_offsets: Lock acquired 2021-01-08 08:46:50,36238 DEBUG [vert.x-eventloop-thread-0] TopicOperator:473 - 110|periodic kafka __consumer_offsets: Executing action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:46:50,36472 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 149,4 replyHeader:: 149,4294969407,0 request:: '/strimzi/topics/my-topic-reversed,F response:: #7b226d61702d6e616d65223a226d792d746f7069632d7265766572736564222c22746f7069632d6e616d65223a226d792d746f7069632d7265766572736564222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967573,4294967573,1610094347670,1610094347670,0,0,0,0,106,0,4294967573} 2021-01-08 08:46:50,36605 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 150,4 replyHeader:: 150,4294969407,0 request:: '/strimzi/topics/my-topic,F response:: #7b226d61702d6e616d65223a226d792d746f706963222c22746f7069632d6e616d65223a226d792d746f706963222c22706172746974696f6e73223a31322c227265706c69636173223a332c22636f6e666967223a7b7d7d,s{4294967574,4294967574,1610094347671,1610094347671,0,0,0,0,88,0,4294967574} 2021-01-08 08:46:50,36657 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 108|periodic kafka my-topic-reversed: Have private topic for topic my-topic-reversed in Kafka 2021-01-08 08:46:50,36735 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:923 - Reading reply sessionid:0x101a07e36350011, packet:: clientPath:null serverPath:null finished:false header:: 151,4 replyHeader:: 151,4294969407,0 request:: '/strimzi/topics/__consumer_offsets,F response:: #7b226d61702d6e616d65223a22636f6e73756d65722d6f6666736574732d2d2d38346537613637386430386634626432323638373265356364643465623532376661646331633661222c22746f7069632d6e616d65223a225f5f636f6e73756d65725f6f666673657473222c22706172746974696f6e73223a35302c227265706c69636173223a332c22636f6e666967223a7b22636f6d7072657373696f6e2e74797065223a2270726f6475636572222c22636c65616e75702e706f6c696379223a22636f6d70616374222c227365676d656e742e6279746573223a22313034383537363030227d7d,s{4294969401,4294969401,1610095163323,1610095163323,0,0,0,0,233,0,4294969401} 2021-01-08 08:46:50,36762 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 109|periodic kafka my-topic: Have private topic for topic my-topic in Kafka 2021-01-08 08:46:50,36860 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1368 - 110|periodic kafka __consumer_offsets: Have private topic for topic __consumer_offsets in Kafka 2021-01-08 08:46:50,37962 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 108|periodic kafka my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:46:50,37989 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic my-topic-reversed 2021-01-08 08:46:50,38014 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095670379, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:46:50,38071 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic-reversed', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=183 2021-01-08 08:46:50,38096 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=183) and timeout 30000 to node 2: {topics=[{name=my-topic-reversed,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:46:50,38376 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=183): org.apache.kafka.common.requests.CreateTopicsResponse@38b79ac 2021-01-08 08:46:50,38510 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095670384, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:46:50,38537 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095670385, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:46:50,38587 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic-reversed')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-0.my-cluster-kafka-brokers.myproject.svc:9091 (id: 0 rack: null). correlationId=184 2021-01-08 08:46:50,38608 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=184) and timeout 30000 to node 0: {topics=[{name=my-topic-reversed,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:46:50,38634 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@4aa0087b to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=185 2021-01-08 08:46:50,38649 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=185) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=my-topic-reversed,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:46:50,38711 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 109|periodic kafka my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:46:50,38730 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic my-topic 2021-01-08 08:46:50,38749 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095670387, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:46:50,38775 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='my-topic', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=186 2021-01-08 08:46:50,38794 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=186) and timeout 30000 to node 2: {topics=[{name=my-topic,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:46:50,38982 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=186): org.apache.kafka.common.requests.CreateTopicsResponse@60e8404e 2021-01-08 08:46:50,39135 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095670391, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:46:50,39159 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095670391, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:46:50,39222 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 0 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=184): org.apache.kafka.common.requests.MetadataResponse@4e30bce7 2021-01-08 08:46:50,39248 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=185): org.apache.kafka.common.requests.DescribeConfigsResponse@16aefa94 2021-01-08 08:46:50,39303 DEBUG [vert.x-eventloop-thread-0] K8sTopicWatcher:72 - 110|periodic kafka __consumer_offsets: Concurrent modification in kube: new version 1213783 2021-01-08 08:46:50,39368 DEBUG [vert.x-eventloop-thread-0] KafkaImpl:108 - Getting metadata for topic __consumer_offsets 2021-01-08 08:46:50,39390 DEBUG [vert.x-eventloop-thread-0] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=createTopics, deadlineMs=1610095670393, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:46:50,39392 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='my-topic')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=187 2021-01-08 08:46:50,39431 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=187) and timeout 30000 to node 2: {topics=[{name=my-topic,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:46:50,39443 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 108|periodic kafka my-topic-reversed|1208693: Reconciling topic my-topic-reversed, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:46:50,39457 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 108|periodic kafka my-topic-reversed|1208693: 3 way diff 2021-01-08 08:46:50,39474 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 108|periodic kafka my-topic-reversed|1208693: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:46:50,39486 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 108|periodic kafka my-topic-reversed|1208693: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:46:50,39498 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 108|periodic kafka my-topic-reversed|1208693: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:46:50,39507 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 108|periodic kafka my-topic-reversed|1208693: All three topics are identical 2021-01-08 08:46:50,39520 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic-reversed 2021-01-08 08:46:50,39530 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic-reversed reconcile success -> succeeded 2021-01-08 08:46:50,39542 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 108|periodic kafka my-topic-reversed|1208693: Executing handler for action reconcile-from-kafka on topic my-topic-reversed 2021-01-08 08:46:50,39552 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 108|periodic kafka my-topic-reversed|1208693: There is a KafkaTopic to set status on, rv=1208693, generation=1 2021-01-08 08:46:50,39649 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:46:50.395623Z"} 2021-01-08 08:46:50,39670 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 108|periodic kafka my-topic-reversed|1208693: Lock released 2021-01-08 08:46:50,39682 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 108|periodic kafka my-topic-reversed|1208693: Removing last waiter reconcile-from-kafka 2021-01-08 08:46:50,40024 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=187): org.apache.kafka.common.requests.MetadataResponse@5c07cbea 2021-01-08 08:46:50,40106 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@7e4f1a14 to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=188 2021-01-08 08:46:50,40120 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=188) and timeout 30000 to node 2: {resources=[{resource_type=2,resource_name=my-topic,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:46:50,40606 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 2 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=188): org.apache.kafka.common.requests.DescribeConfigsResponse@35d356be 2021-01-08 08:46:50,40708 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending CreateTopicsRequestData(topics=[CreatableTopic(name='__consumer_offsets', numPartitions=1, replicationFactor=1, assignments=[], configs=[])], timeoutMs=30000, validateOnly=true) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=189 2021-01-08 08:46:50,40730 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending CREATE_TOPICS request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=189) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,num_partitions=1,replication_factor=1,assignments=[],configs=[],_tagged_fields={}}],timeout_ms=30000,validate_only=true,_tagged_fields={}} 2021-01-08 08:46:50,40858 INFO [vert.x-eventloop-thread-0] TopicOperator:538 - 109|periodic kafka my-topic|1208692: Reconciling topic my-topic, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:46:50,40873 DEBUG [vert.x-eventloop-thread-0] TopicOperator:598 - 109|periodic kafka my-topic|1208692: 3 way diff 2021-01-08 08:46:50,40890 DEBUG [vert.x-eventloop-thread-0] TopicOperator:680 - 109|periodic kafka my-topic|1208692: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:46:50,40901 DEBUG [vert.x-eventloop-thread-0] TopicOperator:682 - 109|periodic kafka my-topic|1208692: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:46:50,40911 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received CREATE_TOPICS response from node 2 for request with header RequestHeader(apiKey=CREATE_TOPICS, apiVersion=6, clientId=adminclient-1, correlationId=189): org.apache.kafka.common.requests.CreateTopicsResponse@54c860ca 2021-01-08 08:46:50,40914 DEBUG [vert.x-eventloop-thread-0] TopicOperator:691 - 109|periodic kafka my-topic|1208692: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:46:50,40927 INFO [vert.x-eventloop-thread-0] TopicOperator:693 - 109|periodic kafka my-topic|1208692: All three topics are identical 2021-01-08 08:46:50,41050 INFO [vert.x-eventloop-thread-0] TopicOperator:1435 - Success reconciling KafkaTopic myproject/my-topic 2021-01-08 08:46:50,41070 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1371 - my-topic reconcile success -> succeeded 2021-01-08 08:46:50,41078 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeTopics, deadlineMs=1610095670410, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:46:50,41086 DEBUG [vert.x-eventloop-thread-0] TopicOperator:475 - 109|periodic kafka my-topic|1208692: Executing handler for action reconcile-from-kafka on topic my-topic 2021-01-08 08:46:50,41099 DEBUG [vert.x-eventloop-thread-0] TopicOperator:1000 - 109|periodic kafka my-topic|1208692: There is a KafkaTopic to set status on, rv=1208692, generation=1 2021-01-08 08:46:50,41113 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending MetadataRequestData(topics=[MetadataRequestTopic(name='__consumer_offsets')], allowAutoTopicCreation=false, includeClusterAuthorizedOperations=false, includeTopicAuthorizedOperations=false) to my-cluster-kafka-2.my-cluster-kafka-brokers.myproject.svc:9091 (id: 2 rack: null). correlationId=190 2021-01-08 08:46:50,41131 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending METADATA request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=190) and timeout 30000 to node 2: {topics=[{name=__consumer_offsets,_tagged_fields={}}],allow_auto_topic_creation=false,include_cluster_authorized_operations=false,include_topic_authorized_operations=false,_tagged_fields={}} 2021-01-08 08:46:50,41206 DEBUG [vert.x-eventloop-thread-0] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:46:50.411078Z"} 2021-01-08 08:46:50,41224 DEBUG [vert.x-eventloop-thread-0] TopicOperator:492 - 109|periodic kafka my-topic|1208692: Lock released 2021-01-08 08:46:50,41238 DEBUG [vert.x-eventloop-thread-0] TopicOperator:449 - 109|periodic kafka my-topic|1208692: Removing last waiter reconcile-from-kafka 2021-01-08 08:46:50,41098 DEBUG [vert.x-eventloop-thread-1] KafkaAdminClient:1358 - [AdminClient clientId=adminclient-1] Queueing Call(callName=describeConfigs, deadlineMs=1610095670410, tries=0, nextAllowedTryMs=0) with a timeout 60000 ms from now. 2021-01-08 08:46:50,41271 DEBUG [kafka-admin-client-thread | adminclient-1] KafkaAdminClient:1094 - [AdminClient clientId=adminclient-1] Sending org.apache.kafka.common.requests.DescribeConfigsRequest$Builder@36f2b922 to my-cluster-kafka-1.my-cluster-kafka-brokers.myproject.svc:9091 (id: 1 rack: null). correlationId=191 2021-01-08 08:46:50,41286 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:522 - [AdminClient clientId=adminclient-1] Sending DESCRIBE_CONFIGS request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=191) and timeout 30000 to node 1: {resources=[{resource_type=2,resource_name=__consumer_offsets,configuration_keys=null}],include_synonyms=false,include_documentation=false} 2021-01-08 08:46:50,41405 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received METADATA response from node 2 for request with header RequestHeader(apiKey=METADATA, apiVersion=9, clientId=adminclient-1, correlationId=190): org.apache.kafka.common.requests.MetadataResponse@30eb8a52 2021-01-08 08:46:50,42090 DEBUG [kafka-admin-client-thread | adminclient-1] NetworkClient:884 - [AdminClient clientId=adminclient-1] Received DESCRIBE_CONFIGS response from node 1 for request with header RequestHeader(apiKey=DESCRIBE_CONFIGS, apiVersion=3, clientId=adminclient-1, correlationId=191): org.apache.kafka.common.requests.DescribeConfigsResponse@1fa24c57 2021-01-08 08:46:50,42225 INFO [vert.x-eventloop-thread-1] TopicOperator:538 - 110|periodic kafka __consumer_offsets|1213783: Reconciling topic __consumer_offsets, k8sTopic:nonnull, kafkaTopic:nonnull, privateTopic:nonnull 2021-01-08 08:46:50,42242 DEBUG [vert.x-eventloop-thread-1] TopicOperator:598 - 110|periodic kafka __consumer_offsets|1213783: 3 way diff 2021-01-08 08:46:50,42262 DEBUG [vert.x-eventloop-thread-1] TopicOperator:680 - 110|periodic kafka __consumer_offsets|1213783: topicStore->kafkaTopic: TopicDiff{differences={}} 2021-01-08 08:46:50,42273 DEBUG [vert.x-eventloop-thread-1] TopicOperator:682 - 110|periodic kafka __consumer_offsets|1213783: topicStore->k8sTopic: TopicDiff{differences={}} 2021-01-08 08:46:50,42286 DEBUG [vert.x-eventloop-thread-1] TopicOperator:691 - 110|periodic kafka __consumer_offsets|1213783: Diffs do not conflict, merged diff: TopicDiff{differences={}} 2021-01-08 08:46:50,42297 INFO [vert.x-eventloop-thread-1] TopicOperator:693 - 110|periodic kafka __consumer_offsets|1213783: All three topics are identical 2021-01-08 08:46:50,42311 INFO [vert.x-eventloop-thread-1] TopicOperator:1435 - Success reconciling KafkaTopic myproject/consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a 2021-01-08 08:46:50,42321 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1371 - __consumer_offsets reconcile success -> succeeded 2021-01-08 08:46:50,42335 DEBUG [vert.x-eventloop-thread-1] TopicOperator:475 - 110|periodic kafka __consumer_offsets|1213783: Executing handler for action reconcile-from-kafka on topic __consumer_offsets 2021-01-08 08:46:50,42347 DEBUG [vert.x-eventloop-thread-1] TopicOperator:1000 - 110|periodic kafka __consumer_offsets|1213783: There is a KafkaTopic to set status on, rv=1213783, generation=1 2021-01-08 08:46:50,42452 DEBUG [vert.x-eventloop-thread-1] StatusDiff:43 - Ignoring Status diff {"op":"replace","path":"/conditions/0/lastTransitionTime","value":"2021-01-08T08:46:50.423530Z"} 2021-01-08 08:46:50,42533 DEBUG [vert.x-eventloop-thread-1] TopicOperator:492 - 110|periodic kafka __consumer_offsets|1213783: Lock released 2021-01-08 08:46:50,42547 DEBUG [vert.x-eventloop-thread-1] TopicOperator:449 - 110|periodic kafka __consumer_offsets|1213783: Removing last waiter reconcile-from-kafka 2021-01-08 08:46:50,43402 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 111|periodic kube consumer-offsets: Concurrent modification in kube: new version 1210026 2021-01-08 08:46:50,43434 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 112|periodic kube consumer-offsets---84e7a678d08f4bd226872e5cdd4eb527fadc1c6a: Concurrent modification in kube: new version 1213783 2021-01-08 08:46:50,43450 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 113|periodic kube my-topic: Concurrent modification in kube: new version 1208692 2021-01-08 08:46:50,43465 DEBUG [vert.x-eventloop-thread-1] K8sTopicWatcher:72 - 114|periodic kube my-topic-reversed: Concurrent modification in kube: new version 1208693 2021-01-08 08:46:57,03926 DEBUG [vert.x-worker-thread-0-SendThread(localhost:2181)] ClientCnxn:821 - Got ping response for sessionid: 0x101a07e36350011 after 0ms