PC=0x473721 m=0 sigcode=0 goroutine 0 [idle]: runtime.futex() /usr/lib/golang/src/runtime/sys_linux_amd64.s:552 +0x21 runtime.futexsleep(0x10000000000?, 0x0?, 0x3?) /usr/lib/golang/src/runtime/os_linux.go:66 +0x36 runtime.notesleep(0x177f4a8) /usr/lib/golang/src/runtime/lock_futex.go:159 +0x87 runtime.mPark(...) /usr/lib/golang/src/runtime/proc.go:1449 runtime.stopm() /usr/lib/golang/src/runtime/proc.go:2228 +0x8d runtime.findrunnable() /usr/lib/golang/src/runtime/proc.go:2804 +0x865 runtime.schedule() /usr/lib/golang/src/runtime/proc.go:3187 +0x239 runtime.goexit0(0xc0006feb60) /usr/lib/golang/src/runtime/proc.go:3513 +0x297 runtime.mcall() /usr/lib/golang/src/runtime/asm_amd64.s:425 +0x43 goroutine 1 [select, 15893 minutes]: main.startKafkaMessageConsumer({0x7fff9d77ad3b, 0x6}) /go/src/app/cmd/cloud-connector/kafka_message_consumer.go:139 +0xc38 main.NewRootCommand.func2(0xc00080cc80?, {0xe7abfc?, 0x2?, 0x2?}) /go/src/app/cmd/cloud-connector/main.go:39 +0x25 github.com/spf13/cobra.(*Command).execute(0xc00080cc80, {0xc0004c2300, 0x2, 0x2}) /opt/app-root/src/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:856 +0x663 github.com/spf13/cobra.(*Command).ExecuteC(0xc00080c000) /opt/app-root/src/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:960 +0x39c github.com/spf13/cobra.(*Command).Execute(...) /opt/app-root/src/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:897 main.main() /go/src/app/cmd/cloud-connector/main.go:95 +0x4a goroutine 14 [select, 15893 minutes]: github.com/segmentio/kafka-go.(*ConsumerGroup).nextGeneration(0xc0001539e0, {0xc0005af290, 0x8c}) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/consumergroup.go:859 +0x9e7 github.com/segmentio/kafka-go.(*ConsumerGroup).run(0xc0001539e0) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/consumergroup.go:721 +0x50 github.com/segmentio/kafka-go.NewConsumerGroup.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/consumergroup.go:659 +0x25 created by github.com/segmentio/kafka-go.NewConsumerGroup /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/consumergroup.go:658 +0x16a goroutine 13 [select, 15893 minutes]: database/sql.(*DB).connectionOpener(0xc00041d2b0, {0x11a8b00, 0xc00013f580}) /usr/lib/golang/src/database/sql/sql.go:1226 +0x8d created by database/sql.OpenDB /usr/lib/golang/src/database/sql/sql.go:794 +0x18d goroutine 363 [sync.Cond.Wait, 173 minutes]: sync.runtime_notifyListWait(0xc0007389d0, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xae879b?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc00068e5b8) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc00068e5a0) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 91 [select]: github.com/redhatinsights/platform-go-middlewares/logging/cloudwatch.(*Hook).putBatches(0xc000092840, 0xc00068e1e0) /opt/app-root/src/go/pkg/mod/github.com/redhatinsights/platform-go-middlewares@v0.20.0/logging/cloudwatch/hook.go:151 +0xcf created by github.com/redhatinsights/platform-go-middlewares/logging/cloudwatch.NewBatchingHook /opt/app-root/src/go/pkg/mod/github.com/redhatinsights/platform-go-middlewares@v0.20.0/logging/cloudwatch/hook.go:84 +0x1db goroutine 15 [select, 15893 minutes]: github.com/segmentio/kafka-go.(*ConsumerGroup).Next(0xc0001539e0, {0x11a8b00, 0xc00013fcc0}) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/consumergroup.go:700 +0xd7 github.com/segmentio/kafka-go.(*Reader).run(0xc00026f6c0, 0xc0001539e0) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/reader.go:320 +0x354 created by github.com/segmentio/kafka-go.NewReader /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/reader.go:742 +0x791 goroutine 197 [select]: github.com/segmentio/kafka-go.(*Generation).heartbeatLoop.func1({0x11a97e8, 0xc0000c0a80}) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/consumergroup.go:477 +0x1c5 github.com/segmentio/kafka-go.(*Generation).Start.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/consumergroup.go:396 +0x31 created by github.com/segmentio/kafka-go.(*Generation).Start /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/consumergroup.go:395 +0x178 goroutine 2854 [sync.Cond.Wait, 134 minutes]: sync.runtime_notifyListWait(0xc000f85cd0, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x678c9d?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc00097f818) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc00097f800) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 129 [select, 3 minutes]: github.com/segmentio/kafka-go.(*Reader).FetchMessage(0xc00026f6c0, {0x11a8b00, 0xc000033200}) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/reader.go:826 +0x2c8 main.consumeMqttMessagesFromKafka(0x0?, 0xc0000c9e60, {0x11a8b00, 0xc000033200}, 0x0?) /go/src/app/cmd/cloud-connector/kafka_message_consumer.go:224 +0xb0 created by main.startKafkaMessageConsumer /go/src/app/cmd/cloud-connector/kafka_message_consumer.go:126 +0xaa5 goroutine 149 [IO wait]: internal/poll.runtime_pollWait(0x7f570a7a2ef0, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc000a2c180?, 0x0?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Accept(0xc000a2c180) /usr/lib/golang/src/internal/poll/fd_unix.go:614 +0x22c net.(*netFD).accept(0xc000a2c180) /usr/lib/golang/src/net/fd_unix.go:172 +0x35 net.(*TCPListener).accept(0xc000a00060) /usr/lib/golang/src/net/tcpsock_posix.go:139 +0x28 net.(*TCPListener).Accept(0xc000a00060) /usr/lib/golang/src/net/tcpsock.go:288 +0x3d net/http.(*Server).Serve(0xc0002387e0, {0x11a7c88, 0xc000a00060}) /usr/lib/golang/src/net/http/server.go:3039 +0x385 net/http.(*Server).ListenAndServe(0xc0002387e0) /usr/lib/golang/src/net/http/server.go:2968 +0x7d github.com/RedHatInsights/cloud-connector/internal/platform/utils.StartHTTPServer.func1() /go/src/app/internal/platform/utils/utils.go:26 +0xef created by github.com/RedHatInsights/cloud-connector/internal/platform/utils.StartHTTPServer /go/src/app/internal/platform/utils/utils.go:24 +0x13a goroutine 151 [syscall, 15893 minutes]: os/signal.signal_recv() /usr/lib/golang/src/runtime/sigqueue.go:151 +0x2f os/signal.loop() /usr/lib/golang/src/os/signal/signal_unix.go:23 +0x19 created by os/signal.Notify.func1.1 /usr/lib/golang/src/os/signal/signal.go:151 +0x2a goroutine 3545 [sync.Cond.Wait, 3 minutes]: sync.runtime_notifyListWait(0xc00020c890, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xae879b?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000d6c438) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000d6c420) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 199 [select, 3 minutes]: github.com/segmentio/kafka-go.(*Reader).commitLoopImmediate(0xc00026f6c0, {0x11a97e8, 0xc0000c0a80}, 0xc000480f00?) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/reader.go:213 +0x14d github.com/segmentio/kafka-go.(*Reader).commitLoop(0xc000480fc0?, {0x11a97e8?, 0xc0000c0a80?}, 0x0?) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/reader.go:296 +0x18b github.com/segmentio/kafka-go.(*Reader).run.func3({0x11a97e8?, 0xc0000c0a80?}) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/reader.go:350 +0x34 github.com/segmentio/kafka-go.(*Generation).Start.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/consumergroup.go:396 +0x31 created by github.com/segmentio/kafka-go.(*Generation).Start /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/consumergroup.go:395 +0x178 goroutine 200 [select, 15893 minutes]: github.com/segmentio/kafka-go.(*Reader).run.func4({0x11a97e8?, 0xc0000c0a80?}) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/reader.go:354 +0x91 github.com/segmentio/kafka-go.(*Generation).Start.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/consumergroup.go:396 +0x31 created by github.com/segmentio/kafka-go.(*Generation).Start /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/consumergroup.go:395 +0x178 goroutine 198 [IO wait]: internal/poll.runtime_pollWait(0x7f570a7a2d10, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc0005a4e00?, 0xc00045f000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc0005a4e00, {0xc00045f000, 0x1000, 0x1000}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0005a4e00, {0xc00045f000?, 0xc000531820?, 0xc000531898?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc00058e1b0, {0xc00045f000?, 0xb8d080?, 0xc0005318e8?}) /usr/lib/golang/src/net/net.go:183 +0x45 bufio.(*Reader).fill(0xc0003e03f0) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc0003e03f0, 0x8) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d github.com/segmentio/kafka-go.(*Conn).peekResponseSizeAndID(0xc0003e04f0?) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/conn.go:1250 +0x28 github.com/segmentio/kafka-go.(*Conn).waitResponse(0xc0003e03c0, 0x1e0001?, 0x216a5) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/conn.go:1341 +0xa5 github.com/segmentio/kafka-go.(*Conn).ReadBatchWith(0xc0003e03c0, {0x36353d054c765?, 0x177e5a0?, 0xa0?, 0x363517c48ca07?}) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/conn.go:839 +0x68c github.com/segmentio/kafka-go.(*reader).read(0xc000531ed8, {0x11a8b00, 0xc00020d2c0}, 0x1e909b, 0xc0003e03c0) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/reader.go:1467 +0xf5 github.com/segmentio/kafka-go.(*reader).run(0xc000531ed8, {0x11a8b00, 0xc00020d2c0}, 0x1e909b) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/reader.go:1310 +0x5e5 github.com/segmentio/kafka-go.(*Reader).start.func1({0x11a8b00?, 0xc00020d2c0?}, {{0xc00014a720?, 0x0?}, 0x5c6076?}, 0x5c5e8a?, 0x1?) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/reader.go:1211 +0x1be created by github.com/segmentio/kafka-go.(*Reader).start /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/reader.go:1191 +0x145 goroutine 3311 [sync.Cond.Wait, 115 minutes]: sync.runtime_notifyListWait(0xc000a30610, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x678c9d?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000a4c678) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000a4c660) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 303 [select]: github.com/segmentio/kafka-go.(*connPool).discover(0xc000a269c0, {0x11a8b00, 0xc00056e780}, 0xc000a542a0) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/transport.go:620 +0x497 created by github.com/segmentio/kafka-go.(*Transport).grabPool /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/transport.go:254 +0x5ad goroutine 285 [sync.Cond.Wait, 173 minutes]: sync.runtime_notifyListWait(0xc000554ed0, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000a39e78) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000a39e60) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 284 [chan receive]: github.com/segmentio/kafka-go.(*conn).run(0xc000864870, 0xc000554d40, 0x0?) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/transport.go:1246 +0xaa created by github.com/segmentio/kafka-go.(*connGroup).connect /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/transport.go:1224 +0xd1b goroutine 1311 [sync.Cond.Wait, 136 minutes]: sync.runtime_notifyListWait(0xc000a30bd0, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xae879b?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000a38af8) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000a38ae0) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 2081 [sync.Cond.Wait, 134 minutes]: sync.runtime_notifyListWait(0xc00056f6d0, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xae879b?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0005e0f18) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0005e0f00) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 2243 [sync.Cond.Wait, 135 minutes]: sync.runtime_notifyListWait(0xc000a30150, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x29c0?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000e53758) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000e53740) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 2967 [sync.Cond.Wait, 134 minutes]: sync.runtime_notifyListWait(0xc000e0eb10, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x678c9d?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000b47b18) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000b47b00) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 3276 [sync.Cond.Wait, 10 minutes]: sync.runtime_notifyListWait(0xc000554790, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x17b0720?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000a4c1f8) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000a4c1e0) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 1113 [sync.Cond.Wait, 138 minutes]: sync.runtime_notifyListWait(0xc000982610, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x678c9d?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000fd3a58) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000fd3a40) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 1068 [sync.Cond.Wait, 137 minutes]: sync.runtime_notifyListWait(0xc000554290, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x1180?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0005e01f8) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0005e01e0) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 2874 [sync.Cond.Wait, 134 minutes]: sync.runtime_notifyListWait(0xc000a31450, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x678c9d?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc00068e8b8) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc00068e8a0) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 845517 [select]: github.com/eclipse/paho%2emqtt%2egolang.startIncomingComms.func1() /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/net.go:175 +0x18d created by github.com/eclipse/paho%2emqtt%2egolang.startIncomingComms /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/net.go:164 +0x1b5 goroutine 3368 [sync.Cond.Wait, 115 minutes]: sync.runtime_notifyListWait(0xc00080a590, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xae879b?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000092798) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000092780) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 3634 [sync.Cond.Wait, 174 minutes]: sync.runtime_notifyListWait(0xc000555190, 0xe1) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x2400?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000b46a98) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000b46a80) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 1774 [sync.Cond.Wait, 135 minutes]: sync.runtime_notifyListWait(0xc000032ed0, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xae879b?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc00097e078) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc00097e060) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 3004 [sync.Cond.Wait, 116 minutes]: sync.runtime_notifyListWait(0xc000982a50, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xae879b?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc00013d218) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc00013d200) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 3154 [sync.Cond.Wait, 128 minutes]: sync.runtime_notifyListWait(0xc000b3e390, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc000b85f50?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0009e3338) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0009e3320) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 3151 [sync.Cond.Wait, 133 minutes]: sync.runtime_notifyListWait(0xc00020db90, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc0009e2f78) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc0009e2f60) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 3021 [sync.Cond.Wait, 133 minutes]: sync.runtime_notifyListWait(0xc000a30910, 0xe2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x678c9d?) /usr/lib/golang/src/sync/cond.go:56 +0x8c github.com/segmentio/kafka-go.(*batchQueue).Get(0xc000e523d8) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:918 +0x9c github.com/segmentio/kafka-go.(*partitionWriter).writeBatches(0xc000e523c0) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:978 +0x3b github.com/segmentio/kafka-go.(*Writer).spawn.func1() /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:532 +0x64 created by github.com/segmentio/kafka-go.(*Writer).spawn /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/writer.go:530 +0x8d goroutine 845512 [select]: github.com/eclipse/paho%2emqtt%2egolang.keepalive(0xc000375b00, {0x7f56ea6141b8, 0xc00070eb10}) /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/ping.go:48 +0x1e9 created by github.com/eclipse/paho%2emqtt%2egolang.(*client).startCommsWorkers /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/client.go:561 +0x31b goroutine 845518 [select, 5897 minutes]: github.com/eclipse/paho%2emqtt%2egolang.startOutgoingComms.func1() /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/net.go:277 +0x1e5 created by github.com/eclipse/paho%2emqtt%2egolang.startOutgoingComms /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/net.go:264 +0x255 goroutine 1248596 [IO wait]: internal/poll.runtime_pollWait(0x7f570a7a22c0, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc0000c0680?, 0xc000ee9000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc0000c0680, {0xc000ee9000, 0x1000, 0x1000}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0000c0680, {0xc000ee9000?, 0x4e2686?, 0x7f570a7a23a8?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc00058e050, {0xc000ee9000?, 0x0?, 0x0?}) /usr/lib/golang/src/net/net.go:183 +0x45 net/http.(*connReader).Read(0xc00097ae40, {0xc000ee9000, 0x1000, 0x1000}) /usr/lib/golang/src/net/http/server.go:780 +0x16d bufio.(*Reader).fill(0xc000fd2120) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc000fd2120, 0x4) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d net/http.(*conn).serve(0xc000418820, {0x11a8ba8, 0xc0000e2390}) /usr/lib/golang/src/net/http/server.go:1992 +0x716 created by net/http.(*Server).Serve /usr/lib/golang/src/net/http/server.go:3071 +0x4db goroutine 845515 [select, 5897 minutes]: github.com/eclipse/paho%2emqtt%2egolang.(*client).startCommsWorkers.func1() /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/client.go:586 +0x179 created by github.com/eclipse/paho%2emqtt%2egolang.(*client).startCommsWorkers /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/client.go:583 +0x565 goroutine 845513 [chan receive, 5897 minutes]: github.com/eclipse/paho%2emqtt%2egolang.(*router).matchAndDispatch.func2() /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/router.go:174 +0xbc created by github.com/eclipse/paho%2emqtt%2egolang.(*router).matchAndDispatch /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/router.go:173 +0x26a goroutine 1248344 [chan receive, 3 minutes]: github.com/segmentio/kafka-go.(*conn).run(0xc0005ec370, 0xc00056e400, 0x0?) /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/transport.go:1246 +0xaa created by github.com/segmentio/kafka-go.(*connGroup).connect /opt/app-root/src/go/pkg/mod/github.com/segmentio/kafka-go@v0.4.27/transport.go:1224 +0xd1b goroutine 845516 [IO wait]: internal/poll.runtime_pollWait(0x7f570a7a23b0, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc000a2c880?, 0xc000b8db00?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc000a2c880, {0xc000b8db00, 0x1ab1, 0x1ab1}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc000a2c880, {0xc000b8db00?, 0xc000a2e440?, 0xc000b8db05?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc000e10030, {0xc000b8db00?, 0xc000a2ca80?, 0xc0005c6000?}) /usr/lib/golang/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc000929ae8, {0xc000b8db00?, 0x0?, 0x3?}) /usr/lib/golang/src/crypto/tls/conn.go:785 +0x3d bytes.(*Buffer).ReadFrom(0xc000cee5f8, {0x11a1380, 0xc000929ae8}) /usr/lib/golang/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc000cee380, {0x11a2da0?, 0xc000e10030}, 0x1ab1?) /usr/lib/golang/src/crypto/tls/conn.go:807 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc000cee380, 0x0) /usr/lib/golang/src/crypto/tls/conn.go:614 +0x116 crypto/tls.(*Conn).readRecord(...) /usr/lib/golang/src/crypto/tls/conn.go:582 crypto/tls.(*Conn).Read(0xc000cee380, {0xc00088a000, 0x1000, 0x0?}) /usr/lib/golang/src/crypto/tls/conn.go:1285 +0x16f bufio.(*Reader).fill(0xc000a4d020) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc000a4d020, 0x2) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d github.com/gorilla/websocket.(*Conn).read(0xc00025a2c0, 0xc78df4?) /opt/app-root/src/go/pkg/mod/github.com/gorilla/websocket@v1.5.0/conn.go:371 +0x2c github.com/gorilla/websocket.(*Conn).advanceFrame(0xc00025a2c0) /opt/app-root/src/go/pkg/mod/github.com/gorilla/websocket@v1.5.0/conn.go:809 +0x7b github.com/gorilla/websocket.(*Conn).NextReader(0xc00025a2c0) /opt/app-root/src/go/pkg/mod/github.com/gorilla/websocket@v1.5.0/conn.go:1009 +0xc5 github.com/eclipse/paho%2emqtt%2egolang.(*websocketConnector).Read(0xc00070eb10, {0xc000a83ece, 0x1, 0x1}) /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/websocket.go:115 +0xf5 io.ReadAtLeast({0x7f56ea6141d8, 0xc00070eb10}, {0xc000a83ece, 0x1, 0x1}, 0x1) /usr/lib/golang/src/io/io.go:331 +0x9a io.ReadFull(...) /usr/lib/golang/src/io/io.go:350 github.com/eclipse/paho.mqtt.golang/packets.ReadPacket({0x7f56ea6141d8, 0xc00070eb10}) /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/packets/packets.go:131 +0x5f github.com/eclipse/paho%2emqtt%2egolang.startIncoming.func1() /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/net.go:124 +0xea created by github.com/eclipse/paho%2emqtt%2egolang.startIncoming /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/net.go:122 +0x14d goroutine 845521 [semacquire, 5897 minutes]: sync.runtime_Semacquire(0xc0007aa7a8?) /usr/lib/golang/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc0007aa7d0?) /usr/lib/golang/src/sync/waitgroup.go:136 +0x52 github.com/eclipse/paho%2emqtt%2egolang.startComms.func3() /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/net.go:437 +0x2a created by github.com/eclipse/paho%2emqtt%2egolang.startComms /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/net.go:436 +0x2ea goroutine 845538 [select, 5897 minutes]: github.com/eclipse/paho%2emqtt%2egolang.(*client).startCommsWorkers.func2() /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/client.go:621 +0xfb created by github.com/eclipse/paho%2emqtt%2egolang.(*client).startCommsWorkers /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/client.go:616 +0x6db goroutine 845519 [chan receive, 5897 minutes]: github.com/eclipse/paho%2emqtt%2egolang.startComms.func1() /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/net.go:406 +0x71 created by github.com/eclipse/paho%2emqtt%2egolang.startComms /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/net.go:405 +0x21d goroutine 845520 [chan receive, 5897 minutes]: github.com/eclipse/paho%2emqtt%2egolang.startComms.func2() /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/net.go:429 +0x74 created by github.com/eclipse/paho%2emqtt%2egolang.startComms /opt/app-root/src/go/pkg/mod/github.com/eclipse/paho.mqtt.golang@v1.4.1/net.go:428 +0x28e rax 0xca rbx 0x0 rcx 0x473723 rdx 0x0 rdi 0x177f4a8 rsi 0x80 rbp 0x7fff9d77a210 rsp 0x7fff9d77a1c8 r8 0x0 r9 0x0 r10 0x0 r11 0x286 r12 0x1ca89 r13 0x1 r14 0x177ea40 r15 0x0 rip 0x473721 rflags 0x286 cs 0x33 fs 0x0 gs 0x0