2023/10/19 00:42:11 oauthproxy.go:857: authenticated "internal" via basic auth 2023/10/19 00:42:11 oauthproxy.go:857: authenticated "internal" via basic auth --------------------------------------------------------------------------------- KIALI RELOADED --------------------------------------------------------------------------------- unexpected fault address 0x0 fatal error: fault [signal SIGSEGV: segmentation violation code=0x80 addr=0x0 pc=0x46cedf] goroutine 8762 [running]: runtime.throw({0x26d55b7?, 0xc001317b90?}) /usr/lib/golang/src/runtime/panic.go:992 +0x71 fp=0xc00215e978 sp=0xc00215e948 pc=0x43c671 runtime.sigpanic() /usr/lib/golang/src/runtime/signal_unix.go:829 +0x2ec fp=0xc00215e9c8 sp=0xc00215e978 pc=0x452d4c aeshashbody() /usr/lib/golang/src/runtime/asm_amd64.s:1343 +0x39f fp=0xc00215e9d0 sp=0xc00215e9c8 pc=0x46cedf runtime.mapiternext(0xc0012d8340) /usr/lib/golang/src/runtime/map.go:934 +0x2cb fp=0xc00215ea40 sp=0xc00215e9d0 pc=0x415feb runtime.mapiterinit(0x409454?, 0x413805?, 0xa?) /usr/lib/golang/src/runtime/map.go:861 +0x228 fp=0xc00215ea60 sp=0xc00215ea40 pc=0x415cc8 reflect.mapiterinit(0x4155dd?, 0x2249a80?, 0xc00089e388?) /usr/lib/golang/src/runtime/map.go:1373 +0x19 fp=0xc00215ea88 sp=0xc00215ea60 pc=0x469519 github.com/modern-go/reflect2.(*UnsafeMapType).UnsafeIterate(...) /remote-source/prometheus/app/vendor/github.com/modern-go/reflect2/unsafe_map.go:112 github.com/json-iterator/go.(*sortKeysMapEncoder).Encode(0xc001317b90, 0xc00239fb58, 0xc0019d2d20) /remote-source/prometheus/app/vendor/github.com/json-iterator/go/reflect_map.go:291 +0x225 fp=0xc00215ebf8 sp=0xc00215ea88 pc=0xa4a265 github.com/json-iterator/go.(*onePtrEncoder).Encode(0xc001682ad0, 0xc000646360, 0xc0012d8340?) /remote-source/prometheus/app/vendor/github.com/json-iterator/go/reflect.go:219 +0x82 fp=0xc00215ec30 sp=0xc00215ebf8 pc=0xa42642 github.com/json-iterator/go.(*Stream).WriteVal(0xc0019d2d20, {0x229d6a0, 0xc000646360}) /remote-source/prometheus/app/vendor/github.com/json-iterator/go/reflect.go:98 +0x158 fp=0xc00215eca0 sp=0xc00215ec30 pc=0xa41958 github.com/json-iterator/go.(*dynamicEncoder).Encode(0x22cfa80?, 0xc00215ed01?, 0xc0019d2d38?) /remote-source/prometheus/app/vendor/github.com/json-iterator/go/reflect_dynamic.go:15 +0x39 fp=0xc00215ecd0 sp=0xc00215eca0 pc=0xa44019 github.com/json-iterator/go.(*structFieldEncoder).Encode(0xc0013651a0, 0x202b118?, 0xc0019d2d20) /remote-source/prometheus/app/vendor/github.com/json-iterator/go/reflect_struct_encoder.go:110 +0x56 fp=0xc00215ed48 sp=0xc00215ecd0 pc=0xa579b6 github.com/json-iterator/go.(*structEncoder).Encode(0xc0013652c0, 0x0?, 0xc0019d2d20) /remote-source/prometheus/app/vendor/github.com/json-iterator/go/reflect_struct_encoder.go:158 +0x765 fp=0xc00215ee30 sp=0xc00215ed48 pc=0xa583c5 github.com/json-iterator/go.(*OptionalEncoder).Encode(0xc00215eea8?, 0x4117a5?, 0x2309d80?) /remote-source/prometheus/app/vendor/github.com/json-iterator/go/reflect_optional.go:70 +0xa4 fp=0xc00215ee80 sp=0xc00215ee30 pc=0xa4f5c4 github.com/json-iterator/go.(*onePtrEncoder).Encode(0xc00165cba0, 0xc001890720, 0xc00215eef8?) /remote-source/prometheus/app/vendor/github.com/json-iterator/go/reflect.go:219 +0x82 fp=0xc00215eeb8 sp=0xc00215ee80 pc=0xa42642 github.com/json-iterator/go.(*Stream).WriteVal(0xc0019d2d20, {0x20e43c0, 0xc001890720}) /remote-source/prometheus/app/vendor/github.com/json-iterator/go/reflect.go:98 +0x158 fp=0xc00215ef28 sp=0xc00215eeb8 pc=0xa41958 github.com/json-iterator/go.(*frozenConfig).Marshal(0xc0000c7220, {0x20e43c0, 0xc001890720}) /remote-source/prometheus/app/vendor/github.com/json-iterator/go/config.go:299 +0xc9 fp=0xc00215efc0 sp=0xc00215ef28 pc=0xa38c09 github.com/prometheus/prometheus/web/api/v1.(*API).respond(0xc000428f00, {0x2e528f0, 0xc00125ea80}, {0x229d6a0?, 0xc000646360}, {0x0?, 0x7fc3581bdcd8?, 0x20?}) /remote-source/prometheus/app/web/api/v1/api.go:1516 +0x162 fp=0xc00215f0f8 sp=0xc00215efc0 pc=0x1f81402 github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1({0x2e528f0, 0xc00125ea80}, 0x7fc3581d7a40?) /remote-source/prometheus/app/web/api/v1/api.go:275 +0x20b fp=0xc00215f218 sp=0xc00215f0f8 pc=0x1f72ecb net/http.HandlerFunc.ServeHTTP(0x7fc3581d7a40?, {0x2e528f0?, 0xc00125ea80?}, 0xc0036989b0?) /usr/lib/golang/src/net/http/server.go:2084 +0x2f fp=0xc00215f240 sp=0xc00215f218 pc=0x73766f github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP({{0x2e44f00?, 0xc000dc7260?}}, {0x7fc3581d7a40?, 0xc0036989b0?}, 0x2e43ca0?) /remote-source/prometheus/app/util/httputil/compression.go:90 +0x69 fp=0xc00215f290 sp=0xc00215f240 pc=0x1f712a9 github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP-fm({0x7fc3581d7a40?, 0xc0036989b0?}, 0x1f56fed?) :1 +0x45 fp=0xc00215f2c8 sp=0xc00215f290 pc=0x1f84ae5 github.com/prometheus/prometheus/web.(*Handler).testReady.func1({0x7fc3581d7a40?, 0xc0036989b0?}, 0x7fc3581d7a00?) /remote-source/prometheus/app/web/web.go:508 +0x39 fp=0xc00215f310 sp=0xc00215f2c8 pc=0x1f9dc59 net/http.HandlerFunc.ServeHTTP(0x7fc3581d7a40?, {0x7fc3581d7a40?, 0xc0036989b0?}, 0x50?) /usr/lib/golang/src/net/http/server.go:2084 +0x2f fp=0xc00215f338 sp=0xc00215f310 pc=0x73766f github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1({0x7fc3581d7a40?, 0xc003698960?}, 0xc001ba3f00) /remote-source/prometheus/app/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:198 +0xa5 fp=0xc00215f3b0 sp=0xc00215f338 pc=0x1f593a5 net/http.HandlerFunc.ServeHTTP(0x2e43d00?, {0x7fc3581d7a40?, 0xc003698960?}, 0xc001317aa0?) /usr/lib/golang/src/net/http/server.go:2084 +0x2f fp=0xc00215f3d8 sp=0xc00215f3b0 pc=0x73766f github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2({0x7fc3581d7a40, 0xc003698960}, 0xc001ba3f00) /remote-source/prometheus/app/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76 +0xa2 fp=0xc00215f458 sp=0xc00215f3d8 pc=0x1f58ba2 net/http.HandlerFunc.ServeHTTP(0x2e60c90?, {0x7fc3581d7a40?, 0xc003698960?}, 0x0?) /usr/lib/golang/src/net/http/server.go:2084 +0x2f fp=0xc00215f480 sp=0xc00215f458 pc=0x73766f github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1({0x2e60c90?, 0xc003698910?}, 0xc001ba3f00) /remote-source/prometheus/app/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:101 +0x94 fp=0xc00215f4e0 sp=0xc00215f480 pc=0x1f59154 github.com/prometheus/prometheus/web.setPathWithPrefix.func1.1({0x2e60c90, 0xc003698910}, 0xc001ba3e00) /remote-source/prometheus/app/web/web.go:1116 +0x290 fp=0xc00215f560 sp=0xc00215f4e0 pc=0x1fa3d10 github.com/prometheus/common/route.(*Router).handle.func1({0x2e60c90, 0xc003698910}, 0xc001ba3d00, {0x0, 0x0, 0xc0014618f4?}) /remote-source/prometheus/app/vendor/github.com/prometheus/common/route/route.go:83 +0x2ae fp=0xc00215f610 sp=0xc00215f560 pc=0x1f6ef0e github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc00049c180, {0x2e60c90, 0xc003698910}, 0xc001ba3d00) /remote-source/prometheus/app/vendor/github.com/julienschmidt/httprouter/router.go:387 +0x82b fp=0xc00215f720 sp=0xc00215f610 pc=0x1f6bbeb github.com/prometheus/common/route.(*Router).ServeHTTP(0x8?, {0x2e60c90?, 0xc003698910?}, 0xc00215f7e0?) /remote-source/prometheus/app/vendor/github.com/prometheus/common/route/route.go:121 +0x26 fp=0xc00215f750 sp=0xc00215f720 pc=0x1f6f526 net/http.StripPrefix.func1({0x2e60c90, 0xc003698910}, 0xc001ba3c00) /usr/lib/golang/src/net/http/server.go:2127 +0x330 fp=0xc00215f7f0 sp=0xc00215f750 pc=0x737dd0 net/http.HandlerFunc.ServeHTTP(0xc00215f848?, {0x2e60c90?, 0xc003698910?}, 0x7fc38316ef18?) /usr/lib/golang/src/net/http/server.go:2084 +0x2f fp=0xc00215f818 sp=0xc00215f7f0 pc=0x73766f net/http.(*ServeMux).ServeHTTP(0x413b67?, {0x2e60c90, 0xc003698910}, 0xc001ba3c00) /usr/lib/golang/src/net/http/server.go:2462 +0x149 fp=0xc00215f868 sp=0xc00215f818 pc=0x7394c9 net/http.Handler.ServeHTTP-fm({0x2e60c90?, 0xc003698910?}, 0x2e707c0?) :1 +0x42 fp=0xc00215f898 sp=0xc00215f868 pc=0x75f362 github.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5({0x2e5f730?, 0xc0020aaee0}, 0xc001ba3b00) /remote-source/prometheus/app/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:154 +0x623 fp=0xc00215f9e8 sp=0xc00215f898 pc=0x1f25a03 net/http.HandlerFunc.ServeHTTP(0x0?, {0x2e5f730?, 0xc0020aaee0?}, 0xffffffffffffffff?) /usr/lib/golang/src/net/http/server.go:2084 +0x2f fp=0xc00215fa10 sp=0xc00215f9e8 pc=0x73766f github.com/prometheus/prometheus/web.withStackTracer.func1({0x2e5f730?, 0xc0020aaee0?}, 0xc001682940?) /remote-source/prometheus/app/web/web.go:98 +0x97 fp=0xc00215fa70 sp=0xc00215fa10 pc=0x1f99bd7 net/http.HandlerFunc.ServeHTTP(0x0?, {0x2e5f730?, 0xc0020aaee0?}, 0xc0015bec00?) /usr/lib/golang/src/net/http/server.go:2084 +0x2f fp=0xc00215fa98 sp=0xc00215fa70 pc=0x73766f net/http.serverHandler.ServeHTTP({0xc00248b530?}, {0x2e5f730, 0xc0020aaee0}, 0xc001ba3b00) /usr/lib/golang/src/net/http/server.go:2916 +0x43b fp=0xc00215fb58 sp=0xc00215fa98 pc=0x73b11b net/http.(*conn).serve(0xc0032180a0, {0x2e61040, 0xc000ddd320}) /usr/lib/golang/src/net/http/server.go:1966 +0x5d7 fp=0xc00215ffb8 sp=0xc00215fb58 pc=0x736117 net/http.(*Server).Serve.func3() /usr/lib/golang/src/net/http/server.go:3071 +0x2e fp=0xc00215ffe0 sp=0xc00215ffb8 pc=0x73ba6e runtime.goexit() /usr/lib/golang/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc00215ffe8 sp=0xc00215ffe0 pc=0x46fae1 created by net/http.(*Server).Serve /usr/lib/golang/src/net/http/server.go:3071 +0x4db goroutine 1 [chan receive, 61 minutes]: github.com/oklog/run.(*Group).Run(0xc001885b68) /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:43 +0x7c main.main() /remote-source/prometheus/app/cmd/prometheus/main.go:819 +0x8ae5 goroutine 5 [chan receive]: k8s.io/klog/v2.(*loggingT).flushDaemon(0xc00029e0c0?) /remote-source/prometheus/app/vendor/k8s.io/klog/v2/klog.go:1169 +0x6a created by k8s.io/klog/v2.init.0 /remote-source/prometheus/app/vendor/k8s.io/klog/v2/klog.go:417 +0xf6 goroutine 10 [select]: go.opencensus.io/stats/view.(*worker).start(0xc000294180) /remote-source/prometheus/app/vendor/go.opencensus.io/stats/view/worker.go:276 +0xad created by go.opencensus.io/stats/view.init.0 /remote-source/prometheus/app/vendor/go.opencensus.io/stats/view/worker.go:34 +0x8d goroutine 115 [select]: github.com/prometheus/prometheus/pkg/logging.(*Deduper).run(0xc000815ac0) /remote-source/prometheus/app/pkg/logging/dedupe.go:75 +0xe8 created by github.com/prometheus/prometheus/pkg/logging.Dedupe /remote-source/prometheus/app/pkg/logging/dedupe.go:61 +0x10a goroutine 104 [sync.Cond.Wait, 61 minutes]: sync.runtime_notifyListWait(0xc0006600c8, 0x0) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x14?) /usr/lib/golang/src/sync/cond.go:56 +0x8c k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0006600a0, 0xc000662a90) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:493 +0x1f3 k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000e8e240) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:183 +0x36 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x413805?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x9a6ba5?, {0x2e41fe0, 0xc000ddc030}, 0x1, 0xc000e08180) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000e8e2a8?, 0x3b9aca00, 0x0, 0x0?, 0x7fc35839c1e8?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*controller).Run(0xc000e8e240, 0xc000e08180) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:154 +0x2c5 k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc000897b80, 0xc000802000?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:410 +0x42c created by istio.io/istio/pkg/servicemesh/controller.(*serviceMeshMemberRollController).Start /remote-source/prometheus/app/vendor/istio.io/istio/pkg/servicemesh/controller/controller.go:83 +0x114 goroutine 84 [select]: github.com/prometheus/prometheus/discovery.(*Manager).sender(0xc000982fc0) /remote-source/prometheus/app/discovery/manager.go:254 +0xd7 created by github.com/prometheus/prometheus/discovery.(*Manager).Run /remote-source/prometheus/app/discovery/manager.go:161 +0x5a goroutine 122 [chan receive]: github.com/prometheus/prometheus/storage/remote.(*WriteStorage).run(0xc00014cb40) /remote-source/prometheus/app/storage/remote/write.go:96 +0xa8 created by github.com/prometheus/prometheus/storage/remote.NewWriteStorage /remote-source/prometheus/app/storage/remote/write.go:89 +0x445 goroutine 274 [syscall, 61 minutes]: os/signal.signal_recv() /usr/lib/golang/src/runtime/sigqueue.go:151 +0x2f os/signal.loop() /usr/lib/golang/src/os/signal/signal_unix.go:23 +0x19 created by os/signal.Notify.func1.1 /usr/lib/golang/src/os/signal/signal.go:151 +0x2a goroutine 275 [select, 61 minutes]: main.main.func6() /remote-source/prometheus/app/cmd/prometheus/main.go:582 +0xa8 github.com/oklog/run.(*Group).Run.func1({0xc00070a380?, 0xc0006620d0?}) /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:38 +0x2f created by github.com/oklog/run.(*Group).Run /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:37 +0x22a goroutine 276 [chan receive, 61 minutes]: github.com/prometheus/prometheus/discovery.(*Manager).Run(0xc000982fc0) /remote-source/prometheus/app/discovery/manager.go:162 +0x77 main.main.func8() /remote-source/prometheus/app/cmd/prometheus/main.go:602 +0x33 github.com/oklog/run.(*Group).Run.func1({0xc0000a0680?, 0xc0000a1000?}) /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:38 +0x2f created by github.com/oklog/run.(*Group).Run /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:37 +0x22a goroutine 277 [chan receive, 61 minutes]: github.com/prometheus/prometheus/discovery.(*Manager).Run(0xc000983080) /remote-source/prometheus/app/discovery/manager.go:162 +0x77 main.main.func10() /remote-source/prometheus/app/cmd/prometheus/main.go:616 +0x33 github.com/oklog/run.(*Group).Run.func1({0xc0000a1560?, 0xc0000a15c0?}) /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:38 +0x2f created by github.com/oklog/run.(*Group).Run /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:37 +0x22a goroutine 278 [select]: github.com/prometheus/prometheus/scrape.(*Manager).Run(0xc00046afc0, 0xc0000fa600) /remote-source/prometheus/app/scrape/manager.go:142 +0xba main.main.func12() /remote-source/prometheus/app/cmd/prometheus/main.go:636 +0x6e github.com/oklog/run.(*Group).Run.func1({0xc000968000?, 0xc0000a1600?}) /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:38 +0x2f created by github.com/oklog/run.(*Group).Run /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:37 +0x22a goroutine 279 [select, 61 minutes]: main.main.func14() /remote-source/prometheus/app/cmd/prometheus/main.go:661 +0x135 github.com/oklog/run.(*Group).Run.func1({0xc000481c20?, 0xc000524cc0?}) /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:38 +0x2f created by github.com/oklog/run.(*Group).Run /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:37 +0x22a goroutine 280 [chan receive, 61 minutes]: main.main.func16() /remote-source/prometheus/app/cmd/prometheus/main.go:707 +0x2b6 github.com/oklog/run.(*Group).Run.func1({0xc000481c80?, 0xc000524ce0?}) /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:38 +0x2f created by github.com/oklog/run.(*Group).Run /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:37 +0x22a goroutine 281 [chan receive, 61 minutes]: github.com/prometheus/prometheus/rules.(*Manager).Run(...) /remote-source/prometheus/app/rules/manager.go:912 main.main.func18() /remote-source/prometheus/app/cmd/prometheus/main.go:720 +0x4b github.com/oklog/run.(*Group).Run.func1({0xc000d8d4d0?, 0xc000524cf0?}) /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:38 +0x2f created by github.com/oklog/run.(*Group).Run /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:37 +0x22a goroutine 282 [chan receive, 61 minutes]: main.main.func20() /remote-source/prometheus/app/cmd/prometheus/main.go:772 +0xaac github.com/oklog/run.(*Group).Run.func1({0xc000984780?, 0xc000d99020?}) /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:38 +0x2f created by github.com/oklog/run.(*Group).Run /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:37 +0x22a goroutine 283 [select, 61 minutes]: github.com/prometheus/prometheus/web.(*Handler).Run(0xc0006b2a20, {0x2e60f98, 0xc000815bc0}) /remote-source/prometheus/app/web/web.go:572 +0xa65 main.main.func22() /remote-source/prometheus/app/cmd/prometheus/main.go:787 +0x25 github.com/oklog/run.(*Group).Run.func1({0xc0001437a0?, 0xc000524d00?}) /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:38 +0x2f created by github.com/oklog/run.(*Group).Run /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:37 +0x22a goroutine 284 [select, 61 minutes]: github.com/prometheus/prometheus/notifier.(*Manager).Run(0xc00044bb00, 0xc0000fa780) /remote-source/prometheus/app/notifier/notifier.go:308 +0xb7 main.main.func24() /remote-source/prometheus/app/cmd/prometheus/main.go:810 +0x65 github.com/oklog/run.(*Group).Run.func1({0xc000d99050?, 0xc000524d10?}) /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:38 +0x2f created by github.com/oklog/run.(*Group).Run /remote-source/prometheus/app/vendor/github.com/oklog/run/group.go:37 +0x22a goroutine 73 [select]: github.com/prometheus/prometheus/discovery.(*Manager).sender(0xc000983080) /remote-source/prometheus/app/discovery/manager.go:254 +0xd7 created by github.com/prometheus/prometheus/discovery.(*Manager).Run /remote-source/prometheus/app/discovery/manager.go:161 +0x5a goroutine 388 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fc258, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc00087fb00?, 0xc000095d78?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Accept(0xc00087fb00) /usr/lib/golang/src/internal/poll/fd_unix.go:614 +0x22c net.(*netFD).accept(0xc00087fb00) /usr/lib/golang/src/net/fd_unix.go:172 +0x35 net.(*TCPListener).accept(0xc000d8d518) /usr/lib/golang/src/net/tcpsock_posix.go:139 +0x28 net.(*TCPListener).Accept(0xc000d8d518) /usr/lib/golang/src/net/tcpsock.go:288 +0x3d golang.org/x/net/netutil.(*limitListener).Accept(0xc000d991a0) /remote-source/prometheus/app/vendor/golang.org/x/net/netutil/listen.go:63 +0x3f github.com/mwitkow/go-conntrack.(*connTrackListener).Accept(0xc000d8d560) /remote-source/prometheus/app/vendor/github.com/mwitkow/go-conntrack/listener_wrapper.go:100 +0x6c net/http.(*Server).Serve(0xc000de6000, {0x2e520e0, 0xc000d8d560}) /usr/lib/golang/src/net/http/server.go:3039 +0x385 github.com/prometheus/prometheus/web.(*Handler).Run.func2() /remote-source/prometheus/app/web/web.go:569 +0x2e created by github.com/prometheus/prometheus/web.(*Handler).Run /remote-source/prometheus/app/web/web.go:568 +0x9ea goroutine 364 [select, 61 minutes]: github.com/prometheus/prometheus/tsdb/wal.(*WAL).run(0xc000e8e120) /remote-source/prometheus/app/tsdb/wal/wal.go:332 +0xa5 created by github.com/prometheus/prometheus/tsdb/wal.NewSize /remote-source/prometheus/app/tsdb/wal/wal.go:301 +0x2ca goroutine 87 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*processorListener).run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:772 +0x57 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x0?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00117c738?, {0x2e41fe0, 0xc00068ec90}, 0x1, 0xc0005ba720) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00117c7b8?, 0x3b9aca00, 0x0, 0x7a?, 0xc00117c788?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*processorListener).run(0xc000e2a200?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:771 +0x6b k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 453 [select]: github.com/prometheus/prometheus/tsdb.(*DB).run(0xc0000fe000) /remote-source/prometheus/app/tsdb/db.go:734 +0x17a created by github.com/prometheus/prometheus/tsdb.open /remote-source/prometheus/app/tsdb/db.go:684 +0xa51 goroutine 458 [sync.Cond.Wait, 61 minutes]: sync.runtime_notifyListWait(0xc00078c028, 0x0) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x14?) /usr/lib/golang/src/sync/cond.go:56 +0x8c k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00078c000, 0xc00007e0c0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:493 +0x1f3 k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0005ca000) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:183 +0x36 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x413805?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x9a6ba5?, {0x2e41fe0, 0xc0005c8030}, 0x1, 0xc000e8a6c0) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005ca068?, 0x3b9aca00, 0x0, 0x0?, 0x7fc35839c1e8?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*controller).Run(0xc0005ca000, 0xc000e8a6c0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:154 +0x2c5 k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc000661360, 0xc00118f740?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:410 +0x42c created by istio.io/istio/pkg/servicemesh/controller.(*serviceMeshMemberRollController).Start /remote-source/prometheus/app/vendor/istio.io/istio/pkg/servicemesh/controller/controller.go:83 +0x114 goroutine 394 [select]: k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:373 +0x130 created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:367 +0x393 goroutine 9814 [select]: golang.org/x/net/http2.(*clientStream).writeRequest(0xc0027a8900, 0xc001ba2500) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1464 +0xa89 golang.org/x/net/http2.(*clientStream).doRequest(0xc0019d2de0?, 0xc001bee7a0?) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1326 +0x1e created by golang.org/x/net/http2.(*ClientConn).RoundTrip /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1232 +0x30a goroutine 562 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*controller).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:130 +0x28 created by k8s.io/client-go/tools/cache.(*controller).Run /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:129 +0xbe goroutine 392 [select]: k8s.io/client-go/tools/cache.(*processorListener).pop(0xc001337180) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:742 +0x119 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 461 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc000a46fc0, 0x0?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:628 +0x45 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 549 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0005911f0, 0x0?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:628 +0x45 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 75 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0005308c0, 0x0?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:628 +0x45 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 76 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*controller).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:130 +0x28 created by k8s.io/client-go/tools/cache.(*controller).Run /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:129 +0xbe goroutine 77 [select]: k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0005e4000, {0x0?, 0x0?, 0x42150e0?}, {0x2e4c8c0?, 0xc002350440}, 0xc0005efc18, 0xc0007176e0, 0xc000e8a6c0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:463 +0x1b0 k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0005e4000, 0xc000e8a6c0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:427 +0x691 k8s.io/client-go/tools/cache.(*Reflector).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:221 +0x26 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x4115b6?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000334200?, {0x2e41fc0, 0xc0005be050}, 0x1, 0xc000e8a6c0) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0005e4000, 0xc000e8a6c0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:220 +0x211 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 493 [select]: k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:373 +0x130 created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:367 +0x393 goroutine 551 [select]: k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc000e4e0e0, {0x0?, 0x0?, 0x42150e0?}, {0x2e4c8c0?, 0xc0025d0180}, 0xc00148dc18, 0xc00154c0c0, 0xc000e8b5c0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:463 +0x1b0 k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc000e4e0e0, 0xc000e8b5c0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:427 +0x691 k8s.io/client-go/tools/cache.(*Reflector).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:221 +0x26 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x100004115b6?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000e44300?, {0x2e41fc0, 0xc000aac5f0}, 0x1, 0xc000e8b5c0) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/client-go/tools/cache.(*Reflector).Run(0xc000e4e0e0, 0xc000e8b5c0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:220 +0x211 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 462 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*controller).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:130 +0x28 created by k8s.io/client-go/tools/cache.(*controller).Run /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:129 +0xbe goroutine 481 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0004e9ce0, 0x0?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:628 +0x45 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 513 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0005913b0, 0xc000e8a540?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:628 +0x45 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 463 [select]: k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0005e40e0, {0x0?, 0x0?, 0x42150e0?}, {0x2e4c8c0?, 0xc0038665c0}, 0xc000db5c18, 0xc0004813e0, 0xc000e08180) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:463 +0x1b0 k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0005e40e0, 0xc000e08180) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:427 +0x691 k8s.io/client-go/tools/cache.(*Reflector).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:221 +0x26 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x4115b6?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000a182c0?, {0x2e41fc0, 0xc0012b8000}, 0x1, 0xc000e08180) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0005e40e0, 0xc000e08180) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:220 +0x211 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 506 [select]: k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:373 +0x130 created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:367 +0x393 goroutine 579 [select]: k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc000de6380, {0x0?, 0x0?, 0x42150e0?}, {0x2e4c8c0?, 0xc00175a680}, 0xc00150bc18, 0xc001891aa0, 0xc000e8b740) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:463 +0x1b0 k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc000de6380, 0xc000e8b740) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:427 +0x691 k8s.io/client-go/tools/cache.(*Reflector).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:221 +0x26 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x4115b6?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0012b3740?, {0x2e41fc0, 0xc0005be9b0}, 0x1, 0xc000e8b740) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/client-go/tools/cache.(*Reflector).Run(0xc000de6380, 0xc000e8b740) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:220 +0x211 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 578 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*controller).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:130 +0x28 created by k8s.io/client-go/tools/cache.(*controller).Run /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:129 +0xbe goroutine 106 [sync.Cond.Wait, 61 minutes]: sync.runtime_notifyListWait(0xc00014dec8, 0x0) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x14?) /usr/lib/golang/src/sync/cond.go:56 +0x8c k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00014dea0, 0xc001288070) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:493 +0x1f3 k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0005f8000) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:183 +0x36 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x413805?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x9a6ba5?, {0x2e41fe0, 0xc0007031a0}, 0x1, 0xc000e08360) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005f8068?, 0x3b9aca00, 0x0, 0x0?, 0x7fc35839c1e8?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*controller).Run(0xc0005f8000, 0xc000e08360) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:154 +0x2c5 k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00078dd60, 0x0?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:410 +0x42c created by istio.io/istio/pkg/servicemesh/controller.(*serviceMeshMemberRollController).Start /remote-source/prometheus/app/vendor/istio.io/istio/pkg/servicemesh/controller/controller.go:83 +0x114 goroutine 511 [sync.Cond.Wait, 61 minutes]: sync.runtime_notifyListWait(0xc0012da3e8, 0x2) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc001406d00?) /usr/lib/golang/src/sync/cond.go:56 +0x8c k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0012da3c0, 0xc0011d2570) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:493 +0x1f3 k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0005f90e0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:183 +0x36 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x413805?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x9a6ba5?, {0x2e41fe0, 0xc0014aca80}, 0x1, 0xc000e8b740) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005f9148?, 0x3b9aca00, 0x0, 0x0?, 0x7fc35839c1e8?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*controller).Run(0xc0005f90e0, 0xc000e8b740) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:154 +0x2c5 k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0012da320, 0xc0012d92c0?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:410 +0x42c created by github.com/maistra/xns-informer/pkg/informers.(*multiNamespaceInformer).AddNamespace /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/informers/informer.go:164 +0x42b goroutine 495 [chan receive]: k8s.io/client-go/tools/cache.(*processorListener).run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:772 +0x57 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x0?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00117ff38?, {0x2e41fe0, 0xc001335350}, 0x1, 0xc000966f60) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0?, 0x3b9aca00, 0x0, 0x0?, 0xc00117ff88?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*processorListener).run(0xc0014a9080?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:771 +0x6b k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 499 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc000a47570, 0xc000e921a0?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:628 +0x45 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 500 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*controller).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:130 +0x28 created by k8s.io/client-go/tools/cache.(*controller).Run /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:129 +0xbe goroutine 550 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*controller).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:130 +0x28 2023/10/19 00:42:11 reverseproxy.go:505: http: proxy error: dial tcp [::1]:9090: connect: connection refused created by k8s.io/client-go/tools/cache.(*controller).Run /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:129 +0xbe goroutine 479 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc000778028, 0x7) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc002045380?) /usr/lib/golang/src/sync/cond.go:56 +0x8c k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc000778000, 0xc0007722c0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:493 +0x1f3 k8s.io/client-go/tools/cache.(*controller).processLoop(0xc00073a870) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:183 +0x36 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x413805?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x9a6ba5?, {0x2e41fe0, 0xc00076cff0}, 0x1, 0xc0005bb740) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00073a8d8?, 0x3b9aca00, 0x0, 0x80?, 0x7fc35839c1e8?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*controller).Run(0xc00073a870, 0xc0005bb740) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:154 +0x2c5 k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc000313f40, 0xc0002a4778?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:410 +0x42c created by github.com/maistra/xns-informer/pkg/informers.(*multiNamespaceInformer).AddNamespace /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/informers/informer.go:164 +0x42b goroutine 501 [select]: k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc000de60e0, {0x0?, 0x0?, 0x42150e0?}, {0x2e4c8c0?, 0xc0036f92c0}, 0xc001295c18, 0xc0006bea20, 0xc000e08360) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:463 +0x1b0 k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc000de60e0, 0xc000e08360) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:427 +0x691 k8s.io/client-go/tools/cache.(*Reflector).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:221 +0x26 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x4115b6?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000334340?, {0x2e41fc0, 0xc0005be190}, 0x1, 0xc000e08360) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/client-go/tools/cache.(*Reflector).Run(0xc000de60e0, 0xc000e08360) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:220 +0x211 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 477 [select]: k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:373 +0x130 created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:367 +0x393 goroutine 563 [select]: k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc00072e540, {0x0?, 0x0?, 0x42150e0?}, {0x2e4c8c0?, 0xc00223a340}, 0xc00150fc18, 0xc00029f860, 0xc0005bb740) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:463 +0x1b0 k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00072e540, 0xc0005bb740) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:427 +0x691 k8s.io/client-go/tools/cache.(*Reflector).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:221 +0x26 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x4115b6?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00075cf00?, {0x2e41fc0, 0xc000dd2eb0}, 0x1, 0xc0005bb740) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00072e540, 0xc0005bb740) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:220 +0x211 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 108 [semacquire, 61 minutes]: sync.runtime_Semacquire(0xc0012bc340?) /usr/lib/golang/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc0001cf340?) /usr/lib/golang/src/sync/waitgroup.go:136 +0x52 github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0001cf340, {0x2e60f98, 0xc00005f640}, 0xc000e08480) /remote-source/prometheus/app/discovery/kubernetes/kubernetes.go:472 +0xfa9 created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider /remote-source/prometheus/app/discovery/manager.go:221 +0x3af goroutine 109 [select]: github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000982fc0, {0x2e60f98, 0xc00005f640}, 0xc0000d9900, 0xc000e08480) /remote-source/prometheus/app/discovery/manager.go:227 +0xd9 created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider /remote-source/prometheus/app/discovery/manager.go:222 +0x458 goroutine 523 [select]: k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:373 +0x130 created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:367 +0x393 goroutine 112 [semacquire, 61 minutes]: sync.runtime_Semacquire(0xc0004ae060?) /usr/lib/golang/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc001195340?) /usr/lib/golang/src/sync/waitgroup.go:136 +0x52 github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc001195340, {0x2e60f98, 0xc00005fa40}, 0xc000e08540) /remote-source/prometheus/app/discovery/kubernetes/kubernetes.go:472 +0xfa9 created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider /remote-source/prometheus/app/discovery/manager.go:221 +0x3af goroutine 113 [select]: github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000982fc0, {0x2e60f98, 0xc00005fa40}, 0xc00092a5a0, 0xc000e08540) /remote-source/prometheus/app/discovery/manager.go:227 +0xd9 created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider /remote-source/prometheus/app/discovery/manager.go:222 +0x458 goroutine 514 [semacquire, 61 minutes]: sync.runtime_Semacquire(0xc000142580?) /usr/lib/golang/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc001307760?) /usr/lib/golang/src/sync/waitgroup.go:136 +0x52 github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc001307760, {0x2e60f98, 0xc00005fac0}, 0xc000e085a0) /remote-source/prometheus/app/discovery/kubernetes/kubernetes.go:472 +0xfa9 created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider /remote-source/prometheus/app/discovery/manager.go:221 +0x3af goroutine 515 [select]: github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000982fc0, {0x2e60f98, 0xc00005fac0}, 0xc00047b450, 0xc000e085a0) /remote-source/prometheus/app/discovery/manager.go:227 +0xd9 created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider /remote-source/prometheus/app/discovery/manager.go:222 +0x458 goroutine 391 [chan receive]: k8s.io/client-go/tools/cache.(*processorListener).run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:772 +0x57 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x0?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000e3a738?, {0x2e41fe0, 0xc000de8030}, 0x1, 0xc000118660) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0?, 0x3b9aca00, 0x0, 0x0?, 0xc000e3a788?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*processorListener).run(0xc001337180?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:771 +0x6b k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 487 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*processorListener).run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:772 +0x57 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0005b76c0?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000d9ef38?, {0x2e41fe0, 0xc00068ec00}, 0x1, 0xc0005ba540) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000d9efb0?, 0x3b9aca00, 0x0, 0xd0?, 0x74ddd1?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*processorListener).run(0xc000294980?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:771 +0x6b k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 488 [select, 61 minutes]: k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000294980) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:742 +0x119 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 519 [chan receive]: k8s.io/client-go/tools/cache.(*processorListener).run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:772 +0x57 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x0?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc001442738?, {0x2e41fe0, 0xc000d8f2f0}, 0x1, 0xc000e08840) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0?, 0x3b9aca00, 0x0, 0x0?, 0xc001442788?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*processorListener).run(0xc0014a8f80?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:771 +0x6b k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 516 [select]: github.com/prometheus/prometheus/scrape.(*Manager).reloader(0xc00046afc0) /remote-source/prometheus/app/scrape/manager.go:166 +0x13c created by github.com/prometheus/prometheus/scrape.(*Manager).Run /remote-source/prometheus/app/scrape/manager.go:140 +0x65 goroutine 509 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc000e540c8, 0x8) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc001406060?) /usr/lib/golang/src/sync/cond.go:56 +0x8c k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc000e540a0, 0xc000111f00) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:493 +0x1f3 k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000e286c0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:183 +0x36 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x413805?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x9a6ba5?, {0x2e41fe0, 0xc000d95860}, 0x1, 0xc000e8b5c0) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000e28728?, 0x3b9aca00, 0x0, 0x80?, 0x7fc35839c1e8?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*controller).Run(0xc000e286c0, 0xc000e8b5c0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:154 +0x2c5 k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0012da1e0, 0xc0002a9f78?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:410 +0x42c created by github.com/maistra/xns-informer/pkg/informers.(*multiNamespaceInformer).AddNamespace /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/informers/informer.go:164 +0x42b goroutine 531 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*processorListener).run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:772 +0x57 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x0?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00143df38?, {0x2e41fe0, 0xc000d8f230}, 0x1, 0xc000e08660) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0?, 0x3b9aca00, 0x0, 0x0?, 0x0?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*processorListener).run(0xc000422200?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:771 +0x6b k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 532 [select, 61 minutes]: k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000422200) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:742 +0x119 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 595 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc000530540, 0x0?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:628 +0x45 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 510 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0001f16a8, 0x7) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc001366000?) /usr/lib/golang/src/sync/cond.go:56 +0x8c k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0001f1680, 0xc0006963e0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:493 +0x1f3 k8s.io/client-go/tools/cache.(*controller).processLoop(0xc00041c120) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:183 +0x36 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x413805?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x9a6ba5?, {0x2e41fe0, 0xc000d989c0}, 0x1, 0xc000e8b6e0) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00041c188?, 0x3b9aca00, 0x0, 0x80?, 0x7fc35839c1e8?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*controller).Run(0xc00041c120, 0xc000e8b6e0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:154 +0x2c5 k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0012da280, 0xc0002aaf78?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:410 +0x42c created by github.com/maistra/xns-informer/pkg/informers.(*multiNamespaceInformer).AddNamespace /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/informers/informer.go:164 +0x42b goroutine 489 [chan receive]: k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc000717080) /remote-source/prometheus/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xa7 created by k8s.io/client-go/util/workqueue.newQueue /remote-source/prometheus/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x19b goroutine 490 [chan receive, 61 minutes]: github.com/maistra/xns-informer/pkg/informers.(*multiNamespaceInformer).Run(0xc000294a00, 0x0?) /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/informers/informer.go:210 +0x58 created by github.com/maistra/xns-informer/pkg/generated/kube.(*sharedInformerFactory).Start /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/generated/kube/factory.go:134 +0x1be goroutine 491 [chan receive, 61 minutes]: github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run(0xc0012f6440, {0x2e60f98, 0xc00005f640}, 0xc000e08480) /remote-source/prometheus/app/discovery/kubernetes/pod.go:103 +0x1a9 github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func7({0x2e3f140?, 0xc0012f6440?}) /remote-source/prometheus/app/discovery/kubernetes/kubernetes.go:466 +0x75 created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run /remote-source/prometheus/app/discovery/kubernetes/kubernetes.go:464 +0x112a goroutine 497 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc000660488, 0x8) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0018ba460?) /usr/lib/golang/src/sync/cond.go:56 +0x8c k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc000660460, 0xc0013384a0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:493 +0x1f3 k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000e8f680) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:183 +0x36 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x413805?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x9a6ba5?, {0x2e41fe0, 0xc001335560}, 0x1, 0xc000967080) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000e8f6e8?, 0x3b9aca00, 0x0, 0x0?, 0x7fc35839c1e8?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*controller).Run(0xc000e8f680, 0xc000967080) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:154 +0x2c5 k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc000660320, 0xc00143dfb8?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:410 +0x42c created by github.com/maistra/xns-informer/pkg/informers.(*multiNamespaceInformer).AddNamespace /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/informers/informer.go:164 +0x42b goroutine 496 [select]: k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0014a9080) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:742 +0x119 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 533 [chan receive]: k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc00029e8a0) /remote-source/prometheus/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xa7 created by k8s.io/client-go/util/workqueue.newQueue /remote-source/prometheus/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x19b goroutine 88 [select, 61 minutes]: k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000e2a200) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:742 +0x119 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 534 [chan receive, 61 minutes]: github.com/maistra/xns-informer/pkg/informers.(*multiNamespaceInformer).Run(0xc000422280, 0x0?) /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/informers/informer.go:210 +0x58 created by github.com/maistra/xns-informer/pkg/generated/kube.(*sharedInformerFactory).Start /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/generated/kube/factory.go:134 +0x1be goroutine 535 [chan receive, 61 minutes]: github.com/prometheus/prometheus/discovery/kubernetes.(*Service).Run(0xc000eb4540, {0x2e60f98, 0xc00005fac0}, 0xc000e085a0) /remote-source/prometheus/app/discovery/kubernetes/service.go:96 +0x1aa github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func7({0x2e3f160?, 0xc000eb4540?}) /remote-source/prometheus/app/discovery/kubernetes/kubernetes.go:466 +0x75 created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run /remote-source/prometheus/app/discovery/kubernetes/kubernetes.go:464 +0x112a goroutine 492 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0012f6410, 0x8) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x219b900?) /usr/lib/golang/src/sync/cond.go:56 +0x8c k8s.io/client-go/util/workqueue.(*Type).Get(0xc000717080) /remote-source/prometheus/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0x9e github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).process(0xc0012f6440, {0x2e60f98, 0xc00005f640}, 0xc000087f98?) /remote-source/prometheus/app/discovery/kubernetes/pod.go:107 +0x6e github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run.func1() /remote-source/prometheus/app/discovery/kubernetes/pod.go:98 +0x53 created by github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run /remote-source/prometheus/app/discovery/kubernetes/pod.go:97 +0x18a goroutine 536 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc000eb4450, 0x8) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x219b900?) /usr/lib/golang/src/sync/cond.go:56 +0x8c k8s.io/client-go/util/workqueue.(*Type).Get(0xc00029e8a0) /remote-source/prometheus/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0x9e github.com/prometheus/prometheus/discovery/kubernetes.(*Service).process(0xc000eb4540, {0x2e60f98, 0xc00005fac0}, 0xc00143ff98?) /remote-source/prometheus/app/discovery/kubernetes/service.go:100 +0x6e github.com/prometheus/prometheus/discovery/kubernetes.(*Service).Run.func1() /remote-source/prometheus/app/discovery/kubernetes/service.go:91 +0x53 created by github.com/prometheus/prometheus/discovery/kubernetes.(*Service).Run /remote-source/prometheus/app/discovery/kubernetes/service.go:90 +0x18d goroutine 89 [chan receive]: k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc000449440) /remote-source/prometheus/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xa7 created by k8s.io/client-go/util/workqueue.newQueue /remote-source/prometheus/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x19b goroutine 90 [chan receive, 61 minutes]: github.com/maistra/xns-informer/pkg/informers.(*multiNamespaceInformer).Run(0xc000e2a280, 0x0?) /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/informers/informer.go:210 +0x58 created by github.com/maistra/xns-informer/pkg/generated/kube.(*sharedInformerFactory).Start /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/generated/kube/factory.go:134 +0x1be goroutine 91 [chan receive, 61 minutes]: github.com/maistra/xns-informer/pkg/informers.(*multiNamespaceInformer).Run(0xc000e2a300, 0x0?) /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/informers/informer.go:210 +0x58 created by github.com/maistra/xns-informer/pkg/generated/kube.(*sharedInformerFactory).Start /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/generated/kube/factory.go:134 +0x1be goroutine 92 [chan receive, 61 minutes]: github.com/maistra/xns-informer/pkg/informers.(*multiNamespaceInformer).Run(0xc000e2a380, 0x0?) /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/informers/informer.go:210 +0x58 created by github.com/maistra/xns-informer/pkg/generated/kube.(*sharedInformerFactory).Start /remote-source/prometheus/app/vendor/github.com/maistra/xns-informer/pkg/generated/kube/factory.go:134 +0x1be goroutine 93 [chan receive, 61 minutes]: github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc000e2a480, {0x2e60f98, 0xc00005fa40}, 0xc000e08540) /remote-source/prometheus/app/discovery/kubernetes/endpoints.go:149 +0x230 github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func7({0x2e3f0e0?, 0xc000e2a480?}) /remote-source/prometheus/app/discovery/kubernetes/kubernetes.go:466 +0x75 created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run /remote-source/prometheus/app/discovery/kubernetes/kubernetes.go:464 +0x112a goroutine 94 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc000e34110, 0x9) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x219b900?) /usr/lib/golang/src/sync/cond.go:56 +0x8c k8s.io/client-go/util/workqueue.(*Type).Get(0xc000449440) /remote-source/prometheus/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0x9e github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc000e2a480, {0x2e60f98, 0xc00005fa40}, 0xc000e3ff98?) /remote-source/prometheus/app/discovery/kubernetes/endpoints.go:153 +0x6e github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1() /remote-source/prometheus/app/discovery/kubernetes/endpoints.go:144 +0x53 created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run /remote-source/prometheus/app/discovery/kubernetes/endpoints.go:143 +0x213 goroutine 473 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fc168, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc0005e6180?, 0xc0015c4000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc0005e6180, {0xc0015c4000, 0x8be7, 0x8be7}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0005e6180, {0xc0015c4000?, 0xc0005d0220?, 0xc0015c4005?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc000e80000, {0xc0015c4000?, 0xc000db7810?, 0x435f70?}) /usr/lib/golang/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc0018bf728, {0xc0015c4000?, 0x0?, 0x0?}) /usr/lib/golang/src/crypto/tls/conn.go:785 +0x3d bytes.(*Buffer).ReadFrom(0xc000228cf8, {0x2e3cf40, 0xc0018bf728}) /usr/lib/golang/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc000228a80, {0x2e42580?, 0xc000e80000}, 0x8be7?) /usr/lib/golang/src/crypto/tls/conn.go:807 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc000228a80, 0x0) /usr/lib/golang/src/crypto/tls/conn.go:614 +0x116 crypto/tls.(*Conn).readRecord(...) /usr/lib/golang/src/crypto/tls/conn.go:582 crypto/tls.(*Conn).Read(0xc000228a80, {0xc0007a9000, 0x1000, 0xc0015ba000?}) /usr/lib/golang/src/crypto/tls/conn.go:1285 +0x16f bufio.(*Reader).Read(0xc00004e960, {0xc00072e2e0, 0x9, 0xc000db7d28?}) /usr/lib/golang/src/bufio/bufio.go:236 +0x1b4 io.ReadAtLeast({0x2e3cd40, 0xc00004e960}, {0xc00072e2e0, 0x9, 0x9}, 0x9) /usr/lib/golang/src/io/io.go:331 +0x9a io.ReadFull(...) /usr/lib/golang/src/io/io.go:350 golang.org/x/net/http2.readFrameHeader({0xc00072e2e0?, 0x9?, 0xc000000000?}, {0x2e3cd40?, 0xc00004e960?}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc00072e2a0) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/frame.go:498 +0x95 golang.org/x/net/http2.(*clientConnReadLoop).run(0xc000db7f98) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:2275 +0x130 golang.org/x/net/http2.(*ClientConn).readLoop(0xc00047f980) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:2170 +0x6f created by golang.org/x/net/http2.(*Transport).newClientConn /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:821 +0xc1c goroutine 538 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0005912d0, 0x0?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:628 +0x45 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 539 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*controller).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:130 +0x28 created by k8s.io/client-go/tools/cache.(*controller).Run /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:129 +0xbe goroutine 520 [select]: k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0014a8f80) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:742 +0x119 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 521 [chan receive]: k8s.io/client-go/tools/cache.(*processorListener).run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:772 +0x57 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x10000000011?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc001442738?, {0x2e41fe0, 0xc000de8000}, 0x1, 0xc0001185a0) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0?, 0x3b9aca00, 0x0, 0x0?, 0xc001442788?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*processorListener).run(0xc00076e880?) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:771 +0x6b k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 522 [select]: k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00076e880) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:742 +0x119 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 596 [chan receive, 61 minutes]: k8s.io/client-go/tools/cache.(*controller).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:130 +0x28 created by k8s.io/client-go/tools/cache.(*controller).Run /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/controller.go:129 +0xbe goroutine 543 [select]: k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:373 +0x130 created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:367 +0x393 goroutine 597 [select]: k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0005e41c0, {0x0?, 0x0?, 0x42150e0?}, {0x2e4c8c0?, 0xc000e35080}, 0xc000db9c18, 0xc000717920, 0xc000967080) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:463 +0x1b0 k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0005e41c0, 0xc000967080) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:427 +0x691 k8s.io/client-go/tools/cache.(*Reflector).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:221 +0x26 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x4115b6?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000a19fc0?, {0x2e41fc0, 0xc0012b8550}, 0x1, 0xc000967080) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0005e41c0, 0xc000967080) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:220 +0x211 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 555 [select]: k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:373 +0x130 created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:367 +0x393 goroutine 9089 [select, 5 minutes]: golang.org/x/net/http2.(*clientStream).writeRequest(0xc000788480, 0xc00045e400) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1464 +0xa89 golang.org/x/net/http2.(*clientStream).doRequest(0x2e61af8?, 0xc001ea4570?) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1326 +0x1e created by golang.org/x/net/http2.(*ClientConn).RoundTrip /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1232 +0x30a goroutine 540 [select]: k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc001592000, {0x0?, 0x0?, 0x42150e0?}, {0x2e4c8c0?, 0xc0011a6080}, 0xc00128fc18, 0xc00118ea80, 0xc000e8b6e0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:463 +0x1b0 k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc001592000, 0xc000e8b6e0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:427 +0x691 k8s.io/client-go/tools/cache.(*Reflector).Run.func1() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:221 +0x26 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x4115b6?) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x3e k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00158e100?, {0x2e41fc0, 0xc000116500}, 0x1, 0xc000e8b6e0) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0xb6 k8s.io/client-go/tools/cache.(*Reflector).Run(0xc001592000, 0xc000e8b6e0) /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:220 +0x211 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x85 goroutine 610 [select]: k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:373 +0x130 created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch /remote-source/prometheus/app/vendor/k8s.io/client-go/tools/cache/reflector.go:367 +0x393 goroutine 8719 [select, 9 minutes]: golang.org/x/net/http2.(*clientStream).writeRequest(0xc000789200, 0xc0020b4c00) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1464 +0xa89 golang.org/x/net/http2.(*clientStream).doRequest(0xc00118f5c0?, 0xc001c4d7a0?) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1326 +0x1e created by golang.org/x/net/http2.(*ClientConn).RoundTrip /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1232 +0x30a goroutine 8842 [select, 7 minutes]: golang.org/x/net/http2.(*clientStream).writeRequest(0xc00151c900, 0xc00046d000) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1464 +0xa89 golang.org/x/net/http2.(*clientStream).doRequest(0xc000420cc0?, 0xc002233fa0?) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1326 +0x1e created by golang.org/x/net/http2.(*ClientConn).RoundTrip /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1232 +0x30a goroutine 605 [select]: net/http.(*persistConn).writeLoop(0xc00139e000) /usr/lib/golang/src/net/http/transport.go:2392 +0xf5 created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1751 +0x1791 goroutine 570 [select]: net/http.(*persistConn).writeLoop(0xc001353d40) /usr/lib/golang/src/net/http/transport.go:2392 +0xf5 created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1751 +0x1791 goroutine 629 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fbda8, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc001760000?, 0xc001dc5000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc001760000, {0xc001dc5000, 0x1000, 0x1000}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc001760000, {0xc001dc5000?, 0x4da5c6?, 0x7fc35a2fbe90?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc000e80008, {0xc001dc5000?, 0x0?, 0x0?}) /usr/lib/golang/src/net/net.go:183 +0x45 net/http.(*connReader).Read(0xc0018f40f0, {0xc001dc5000, 0x1000, 0x1000}) /usr/lib/golang/src/net/http/server.go:780 +0x16d bufio.(*Reader).fill(0xc0014fa000) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc0014fa000, 0x4) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d net/http.(*conn).serve(0xc001e88000, {0x2e61040, 0xc000ddd320}) /usr/lib/golang/src/net/http/server.go:1992 +0x716 created by net/http.(*Server).Serve /usr/lib/golang/src/net/http/server.go:3071 +0x4db goroutine 525 [select]: github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc000038a50, 0x37e11d600, 0x72ff26?, 0xc000d95110?) /remote-source/prometheus/app/scrape/scrape.go:1041 +0x419 created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync /remote-source/prometheus/app/scrape/scrape.go:510 +0x965 goroutine 545 [select]: github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc001582160, 0x37e11d600, 0xaa7f66?, 0xc0012d9580?) /remote-source/prometheus/app/scrape/scrape.go:1041 +0x419 created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync /remote-source/prometheus/app/scrape/scrape.go:510 +0x965 goroutine 642 [select]: github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc001582210, 0x37e11d600, 0x0?, 0x0?) /remote-source/prometheus/app/scrape/scrape.go:1041 +0x419 created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync /remote-source/prometheus/app/scrape/scrape.go:510 +0x965 goroutine 643 [select]: github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0015822c0, 0x37e11d600, 0x0?, 0x0?) /remote-source/prometheus/app/scrape/scrape.go:1041 +0x419 created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync /remote-source/prometheus/app/scrape/scrape.go:510 +0x965 goroutine 644 [select]: github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc001582370, 0x37e11d600, 0x0?, 0x0?) /remote-source/prometheus/app/scrape/scrape.go:1041 +0x419 created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync /remote-source/prometheus/app/scrape/scrape.go:510 +0x965 goroutine 645 [select]: github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0015824d0, 0x37e11d600, 0x0?, 0x0?) /remote-source/prometheus/app/scrape/scrape.go:1041 +0x419 created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync /remote-source/prometheus/app/scrape/scrape.go:510 +0x965 goroutine 567 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fbf88, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc001a32a00?, 0xc001b9e000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc001a32a00, {0xc001b9e000, 0x1000, 0x1000}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc001a32a00, {0xc001b9e000?, 0x43f307?, 0xc00173bc30?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc000df4068, {0xc001b9e000?, 0x12?, 0xc001b60b60?}) /usr/lib/golang/src/net/net.go:183 +0x45 net/http.(*persistConn).Read(0xc001353c20, {0xc001b9e000?, 0xc00180a300?, 0xc00173bd30?}) /usr/lib/golang/src/net/http/transport.go:1929 +0x4e bufio.(*Reader).fill(0xc00046bce0) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc00046bce0, 0x1) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d net/http.(*persistConn).readLoop(0xc001353c20) /usr/lib/golang/src/net/http/transport.go:2093 +0x1ac created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1750 +0x173e goroutine 568 [select]: net/http.(*persistConn).writeLoop(0xc001353c20) /usr/lib/golang/src/net/http/transport.go:2392 +0xf5 created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1751 +0x1791 goroutine 569 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fc078, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc001a32b80?, 0xc001ba0000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc001a32b80, {0xc001ba0000, 0x1000, 0x1000}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc001a32b80, {0xc001ba0000?, 0x43f307?, 0xc001670c30?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc000df4070, {0xc001ba0000?, 0x12?, 0xc001b60d00?}) /usr/lib/golang/src/net/net.go:183 +0x45 net/http.(*persistConn).Read(0xc001353d40, {0xc001ba0000?, 0xc001c9c480?, 0xc001670d30?}) /usr/lib/golang/src/net/http/transport.go:1929 +0x4e bufio.(*Reader).fill(0xc0004d44e0) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc0004d44e0, 0x1) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d net/http.(*persistConn).readLoop(0xc001353d40) /usr/lib/golang/src/net/http/transport.go:2093 +0x1ac created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1750 +0x173e goroutine 606 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fbbc8, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc00132da00?, 0xc0013a8000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc00132da00, {0xc0013a8000, 0x1000, 0x1000}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc00132da00, {0xc0013a8000?, 0x43f307?, 0xc000df0c30?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc00167cfd0, {0xc0013a8000?, 0x12?, 0xc001b60820?}) /usr/lib/golang/src/net/net.go:183 +0x45 net/http.(*persistConn).Read(0xc0019aa6c0, {0xc0013a8000?, 0xc00180a6c0?, 0xc000df0d30?}) /usr/lib/golang/src/net/http/transport.go:1929 +0x4e bufio.(*Reader).fill(0xc0004d4e40) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc0004d4e40, 0x1) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d net/http.(*persistConn).readLoop(0xc0019aa6c0) /usr/lib/golang/src/net/http/transport.go:2093 +0x1ac created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1750 +0x173e goroutine 660 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fbe98, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc001a32200?, 0xc000cfe000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc001a32200, {0xc000cfe000, 0x1000, 0x1000}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc001a32200, {0xc000cfe000?, 0x43f307?, 0xc00173ec30?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc0013860c8, {0xc000cfe000?, 0xe?, 0xc00144dd40?}) /usr/lib/golang/src/net/net.go:183 +0x45 net/http.(*persistConn).Read(0xc000649e60, {0xc000cfe000?, 0xc0005ba4e0?, 0xc00173ed30?}) /usr/lib/golang/src/net/http/transport.go:1929 +0x4e bufio.(*Reader).fill(0xc00118eba0) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc00118eba0, 0x1) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d net/http.(*persistConn).readLoop(0xc000649e60) /usr/lib/golang/src/net/http/transport.go:2093 +0x1ac created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1750 +0x173e goroutine 661 [select]: net/http.(*persistConn).writeLoop(0xc000649e60) /usr/lib/golang/src/net/http/transport.go:2392 +0xf5 created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1751 +0x1791 goroutine 607 [select]: net/http.(*persistConn).writeLoop(0xc0019aa6c0) /usr/lib/golang/src/net/http/transport.go:2392 +0xf5 created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1751 +0x1791 goroutine 604 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fbcb8, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc002080200?, 0xc0013a0000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc002080200, {0xc0013a0000, 0x1000, 0x1000}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc002080200, {0xc0013a0000?, 0x43f307?, 0xc001671c30?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc00167cfc8, {0xc0013a0000?, 0x11?, 0xc001b60680?}) /usr/lib/golang/src/net/net.go:183 +0x45 net/http.(*persistConn).Read(0xc00139e000, {0xc0013a0000?, 0xc00180a5a0?, 0xc001671d30?}) /usr/lib/golang/src/net/http/transport.go:1929 +0x4e bufio.(*Reader).fill(0xc0004d4d20) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc0004d4d20, 0x1) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d net/http.(*persistConn).readLoop(0xc00139e000) /usr/lib/golang/src/net/http/transport.go:2093 +0x1ac created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1750 +0x173e goroutine 675 [select]: net/http.(*persistConn).writeLoop(0xc00136a120) /usr/lib/golang/src/net/http/transport.go:2392 +0xf5 created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1751 +0x1791 goroutine 674 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fbad8, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc0018f6380?, 0xc00227c000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc0018f6380, {0xc00227c000, 0x1000, 0x1000}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0018f6380, {0xc00227c000?, 0x43f307?, 0xc00166cc30?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc000135af8, {0xc00227c000?, 0x12?, 0xc001b609c0?}) /usr/lib/golang/src/net/net.go:183 +0x45 net/http.(*persistConn).Read(0xc00136a120, {0xc00227c000?, 0xc0009676e0?, 0xc00166cd30?}) /usr/lib/golang/src/net/http/transport.go:1929 +0x4e bufio.(*Reader).fill(0xc0019d3920) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc0019d3920, 0x1) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d net/http.(*persistConn).readLoop(0xc00136a120) /usr/lib/golang/src/net/http/transport.go:2093 +0x1ac created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1750 +0x173e goroutine 1965 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fb808, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc001228600?, 0xc001dae000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc001228600, {0xc001dae000, 0x1000, 0x1000}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc001228600, {0xc001dae000?, 0x4da5c6?, 0x7fc35a2fb8f0?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc000483c58, {0xc001dae000?, 0x0?, 0x0?}) /usr/lib/golang/src/net/net.go:183 +0x45 net/http.(*connReader).Read(0xc001a478f0, {0xc001dae000, 0x1000, 0x1000}) /usr/lib/golang/src/net/http/server.go:780 +0x16d bufio.(*Reader).fill(0xc000dd8a20) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc000dd8a20, 0x4) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d net/http.(*conn).serve(0xc002675ea0, {0x2e61040, 0xc000ddd320}) /usr/lib/golang/src/net/http/server.go:1992 +0x716 created by net/http.(*Server).Serve /usr/lib/golang/src/net/http/server.go:3071 +0x4db goroutine 8945 [select, 7 minutes]: golang.org/x/net/http2.(*clientStream).writeRequest(0xc000689500, 0xc00147da00) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1464 +0xa89 golang.org/x/net/http2.(*clientStream).doRequest(0x2e61040?, 0xc000ddd320?) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1326 +0x1e created by golang.org/x/net/http2.(*ClientConn).RoundTrip /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1232 +0x30a goroutine 9555 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fb8f8, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc0018fc180?, 0xc000762000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc0018fc180, {0xc000762000, 0x1000, 0x1000}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0018fc180, {0xc000762000?, 0x4da5c6?, 0x7fc35a2fb9e0?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc000e816a8, {0xc000762000?, 0x0?, 0x0?}) /usr/lib/golang/src/net/net.go:183 +0x45 net/http.(*connReader).Read(0xc0036b2b40, {0xc000762000, 0x1000, 0x1000}) /usr/lib/golang/src/net/http/server.go:780 +0x16d bufio.(*Reader).fill(0xc000717740) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc000717740, 0x4) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d net/http.(*conn).serve(0xc0028e0000, {0x2e61040, 0xc000ddd320}) /usr/lib/golang/src/net/http/server.go:1992 +0x716 created by net/http.(*Server).Serve /usr/lib/golang/src/net/http/server.go:3071 +0x4db goroutine 2020 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fb718, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc001230500?, 0xc002983000?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc001230500, {0xc002983000, 0x1000, 0x1000}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc001230500, {0xc002983000?, 0x43f307?, 0xc001446c30?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc0013861d0, {0xc002983000?, 0x11?, 0xc002637520?}) /usr/lib/golang/src/net/net.go:183 +0x45 net/http.(*persistConn).Read(0xc002776240, {0xc002983000?, 0xc001e76480?, 0xc001446d30?}) /usr/lib/golang/src/net/http/transport.go:1929 +0x4e bufio.(*Reader).fill(0xc000716de0) /usr/lib/golang/src/bufio/bufio.go:106 +0x103 bufio.(*Reader).Peek(0xc000716de0, 0x1) /usr/lib/golang/src/bufio/bufio.go:144 +0x5d net/http.(*persistConn).readLoop(0xc002776240) /usr/lib/golang/src/net/http/transport.go:2093 +0x1ac created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1750 +0x173e goroutine 9319 [select, 5 minutes]: golang.org/x/net/http2.(*clientStream).writeRequest(0xc001429b00, 0xc00045fe00) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1464 +0xa89 golang.org/x/net/http2.(*clientStream).doRequest(0xc001890cc0?, 0xc001f9ffa0?) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1326 +0x1e created by golang.org/x/net/http2.(*ClientConn).RoundTrip /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1232 +0x30a goroutine 8843 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc00151c948, 0x7) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x400?) /usr/lib/golang/src/sync/cond.go:56 +0x8c golang.org/x/net/http2.(*pipe).Read(0xc00151c930, {0xc000e4a001, 0x5ff, 0x5ff}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/pipe.go:76 +0xeb golang.org/x/net/http2.transportResponseBody.Read({0x0?}, {0xc000e4a001?, 0x2?, 0x26de0ff?}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:2558 +0x75 encoding/json.(*Decoder).refill(0xc000770280) /usr/lib/golang/src/encoding/json/stream.go:165 +0x17f encoding/json.(*Decoder).readValue(0xc000770280) /usr/lib/golang/src/encoding/json/stream.go:140 +0xbb encoding/json.(*Decoder).Decode(0xc000770280, {0x22cba80, 0xc000474af8}) /usr/lib/golang/src/encoding/json/stream.go:63 +0x78 k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0027116b0, {0xc0022ac800, 0x400, 0x400}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:152 +0x19c k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc00220c230, 0xc0003eb200?, {0x2e4c6e0, 0xc001378b80}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc001a90340) /remote-source/prometheus/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x4f k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00175a680) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x114 created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0x10a goroutine 9791 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0022c8048, 0x1) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc00312db60?) /usr/lib/golang/src/sync/cond.go:56 +0x8c golang.org/x/net/http2.(*pipe).Read(0xc0022c8030, {0xc000e4ac01, 0x5ff, 0x5ff}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/pipe.go:76 +0xeb golang.org/x/net/http2.transportResponseBody.Read({0x0?}, {0xc000e4ac01?, 0x2?, 0x26de0ff?}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:2558 +0x75 encoding/json.(*Decoder).refill(0xc00134aa00) /usr/lib/golang/src/encoding/json/stream.go:165 +0x17f encoding/json.(*Decoder).readValue(0xc00134aa00) /usr/lib/golang/src/encoding/json/stream.go:140 +0xbb encoding/json.(*Decoder).Decode(0xc00134aa00, {0x22cba80, 0xc000685098}) /usr/lib/golang/src/encoding/json/stream.go:63 +0x78 k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc001759290, {0xc0025ba800, 0x400, 0x400}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:152 +0x19c k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc001d1e3c0, 0xc0003eb440?, {0x2e4c6e0, 0xc00275d140}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000df8780) /remote-source/prometheus/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x4f k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc000e35080) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x114 created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0x10a goroutine 9815 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0027a8948, 0x1) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc00148bb60?) /usr/lib/golang/src/sync/cond.go:56 +0x8c golang.org/x/net/http2.(*pipe).Read(0xc0027a8930, {0xc000742c01, 0x5ff, 0x5ff}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/pipe.go:76 +0xeb golang.org/x/net/http2.transportResponseBody.Read({0x0?}, {0xc000742c01?, 0x2?, 0x26de0ff?}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:2558 +0x75 encoding/json.(*Decoder).refill(0xc0027b8640) /usr/lib/golang/src/encoding/json/stream.go:165 +0x17f encoding/json.(*Decoder).readValue(0xc0027b8640) /usr/lib/golang/src/encoding/json/stream.go:140 +0xbb encoding/json.(*Decoder).Decode(0xc0027b8640, {0x22cba80, 0xc0018bf740}) /usr/lib/golang/src/encoding/json/stream.go:63 +0x78 k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc001193d10, {0xc000462000, 0x400, 0x400}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:152 +0x19c k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc001d1e6e0, 0xc000dd05b0?, {0x2e4c6e0, 0xc001db4280}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc002222300) /remote-source/prometheus/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x4f k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0025d0180) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x114 created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0x10a goroutine 10027 [IO wait]: internal/poll.runtime_pollWait(0x7fc35a2fb628, 0x72) /usr/lib/golang/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc001286400?, 0xc00248b541?, 0x0) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc001286400, {0xc00248b541, 0x1, 0x1}) /usr/lib/golang/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc001286400, {0xc00248b541?, 0xc0013be9c0?, 0x4?}) /usr/lib/golang/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc00239f638, {0xc00248b541?, 0x42463c0?, 0xc002f3af68?}) /usr/lib/golang/src/net/net.go:183 +0x45 net/http.(*connReader).backgroundRead(0xc00248b530) /usr/lib/golang/src/net/http/server.go:672 +0x3f created by net/http.(*connReader).startBackgroundRead /usr/lib/golang/src/net/http/server.go:668 +0xca goroutine 2021 [select]: net/http.(*persistConn).writeLoop(0xc002776240) /usr/lib/golang/src/net/http/transport.go:2392 +0xf5 created by net/http.(*Transport).dialConn /usr/lib/golang/src/net/http/transport.go:1751 +0x1791 goroutine 8720 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc000789248, 0x8) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x400?) /usr/lib/golang/src/sync/cond.go:56 +0x8c golang.org/x/net/http2.(*pipe).Read(0xc000789230, {0xc000743201, 0x5ff, 0x5ff}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/pipe.go:76 +0xeb golang.org/x/net/http2.transportResponseBody.Read({0x0?}, {0xc000743201?, 0x2?, 0x26de0ff?}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:2558 +0x75 encoding/json.(*Decoder).refill(0xc00014e8c0) /usr/lib/golang/src/encoding/json/stream.go:165 +0x17f encoding/json.(*Decoder).readValue(0xc00014e8c0) /usr/lib/golang/src/encoding/json/stream.go:140 +0xbb encoding/json.(*Decoder).Decode(0xc00014e8c0, {0x22cba80, 0xc000684e40}) /usr/lib/golang/src/encoding/json/stream.go:63 +0x78 k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc001245890, {0xc002759c00, 0x400, 0x400}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:152 +0x19c k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc001a5a0f0, 0xc0003eb320?, {0x2e4c6e0, 0xc00275ce40}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000e59180) /remote-source/prometheus/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x4f k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00223a340) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x114 created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0x10a goroutine 9320 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc001429b48, 0x4) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x400?) /usr/lib/golang/src/sync/cond.go:56 +0x8c golang.org/x/net/http2.(*pipe).Read(0xc001429b30, {0xc000742601, 0x5ff, 0x5ff}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/pipe.go:76 +0xeb golang.org/x/net/http2.transportResponseBody.Read({0x0?}, {0xc000742601?, 0x2?, 0x26de0ff?}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:2558 +0x75 encoding/json.(*Decoder).refill(0xc00014ec80) /usr/lib/golang/src/encoding/json/stream.go:165 +0x17f encoding/json.(*Decoder).readValue(0xc00014ec80) /usr/lib/golang/src/encoding/json/stream.go:140 +0xbb encoding/json.(*Decoder).Decode(0xc00014ec80, {0x22cba80, 0xc0018cffb0}) /usr/lib/golang/src/encoding/json/stream.go:63 +0x78 k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0015332f0, {0xc001900800, 0x400, 0x400}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:152 +0x19c k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc002262780, 0xc001f90ee0?, {0x2e4c6e0, 0xc001efc840}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0022aafa0) /remote-source/prometheus/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x4f k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0036f92c0) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x114 created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0x10a goroutine 9727 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0027a8648, 0x1) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc002869b60?) /usr/lib/golang/src/sync/cond.go:56 +0x8c golang.org/x/net/http2.(*pipe).Read(0xc0027a8630, {0xc00177a601, 0x5ff, 0x5ff}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/pipe.go:76 +0xeb golang.org/x/net/http2.transportResponseBody.Read({0x0?}, {0xc00177a601?, 0x2?, 0x26de0ff?}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:2558 +0x75 encoding/json.(*Decoder).refill(0xc000209e00) /usr/lib/golang/src/encoding/json/stream.go:165 +0x17f encoding/json.(*Decoder).readValue(0xc000209e00) /usr/lib/golang/src/encoding/json/stream.go:140 +0xbb encoding/json.(*Decoder).Decode(0xc000209e00, {0x22cba80, 0xc0019d9218}) /usr/lib/golang/src/encoding/json/stream.go:63 +0x78 k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc00234ccf0, {0xc002256800, 0x400, 0x400}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:152 +0x19c k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc00207ce10, 0xc001f90690?, {0x2e4c6e0, 0xc00378da00}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc002678b20) /remote-source/prometheus/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x4f k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc002350440) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x114 created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0x10a goroutine 9154 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0007884c8, 0x5) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x400?) /usr/lib/golang/src/sync/cond.go:56 +0x8c golang.org/x/net/http2.(*pipe).Read(0xc0007884b0, {0xc000e4a601, 0x5ff, 0x5ff}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/pipe.go:76 +0xeb golang.org/x/net/http2.transportResponseBody.Read({0x0?}, {0xc000e4a601?, 0x2?, 0x26de0ff?}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:2558 +0x75 encoding/json.(*Decoder).refill(0xc000770140) /usr/lib/golang/src/encoding/json/stream.go:165 +0x17f encoding/json.(*Decoder).readValue(0xc000770140) /usr/lib/golang/src/encoding/json/stream.go:140 +0xbb encoding/json.(*Decoder).Decode(0xc000770140, {0x22cba80, 0xc000474918}) /usr/lib/golang/src/encoding/json/stream.go:63 +0x78 k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc00216c390, {0xc0024cc800, 0x400, 0x400}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:152 +0x19c k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc000dbec30, 0xc0003eb0e0?, {0x2e4c6e0, 0xc001a97740}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0007c62e0) /remote-source/prometheus/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x4f k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0011a6080) 2023/10/19 00:42:11 oauthproxy.go:857: authenticated "internal" via basic auth 2023/10/19 00:42:11 reverseproxy.go:505: http: proxy error: dial tcp [::1]:9090: connect: connection refused /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x114 created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0x10a goroutine 1962 [select]: github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc001d8a0b0, 0x37e11d600, 0x2e61c10?, 0xc0005e1c40?) /remote-source/prometheus/app/scrape/scrape.go:1041 +0x419 created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync /remote-source/prometheus/app/scrape/scrape.go:510 +0x965 goroutine 9726 [select, 3 minutes]: golang.org/x/net/http2.(*clientStream).writeRequest(0xc0027a8600, 0xc00225f600) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1464 +0xa89 golang.org/x/net/http2.(*clientStream).doRequest(0x2e61040?, 0xc000ddd320?) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1326 +0x1e created by golang.org/x/net/http2.(*ClientConn).RoundTrip /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1232 +0x30a goroutine 9790 [select]: golang.org/x/net/http2.(*clientStream).writeRequest(0xc0022c8000, 0xc00225eb00) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1464 +0xa89 golang.org/x/net/http2.(*clientStream).doRequest(0xaa7f66?, 0xc002350440?) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1326 +0x1e created by golang.org/x/net/http2.(*ClientConn).RoundTrip /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:1232 +0x30a goroutine 9010 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc000689548, 0x6) /usr/lib/golang/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x400?) /usr/lib/golang/src/sync/cond.go:56 +0x8c golang.org/x/net/http2.(*pipe).Read(0xc000689530, {0xc00177a001, 0x5ff, 0x5ff}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/pipe.go:76 +0xeb golang.org/x/net/http2.transportResponseBody.Read({0x0?}, {0xc00177a001?, 0x2?, 0x26de0ff?}) /remote-source/prometheus/app/vendor/golang.org/x/net/http2/transport.go:2558 +0x75 encoding/json.(*Decoder).refill(0xc00014eb40) /usr/lib/golang/src/encoding/json/stream.go:165 +0x17f encoding/json.(*Decoder).readValue(0xc00014eb40) /usr/lib/golang/src/encoding/json/stream.go:140 +0xbb encoding/json.(*Decoder).Decode(0xc00014eb40, {0x22cba80, 0xc000685038}) /usr/lib/golang/src/encoding/json/stream.go:63 +0x78 k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc001a3dce0, {0xc002917000, 0x400, 0x400}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:152 +0x19c k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc001c00190, 0xc00279ea80?, {0x2e4c6e0, 0xc00275d100}) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00188e7e0) /remote-source/prometheus/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x4f k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0038665c0) /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x114 created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher /remote-source/prometheus/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0x10a 2023/10/19 00:42:11 oauthproxy.go:857: authenticated "internal" via basic auth 2023/10/19 00:42:11 reverseproxy.go:505: http: proxy error: dial tcp [::1]:9090: connect: connection refused 2023/10/19 00:42:11 oauthproxy.go:857: authenticated "internal" via basic auth 2023/10/19 00:42:11 oauthproxy.go:857: authenticated "internal" via basic auth 2023/10/19 00:42:11 reverseproxy.go:505: http: proxy error: dial tcp [::1]:9090: connect: connection refused 2023/10/19 00:42:11 reverseproxy.go:505: http: proxy error: dial tcp [::1]:9090: connect: connection refused --------------------------------------------------------------------------------- KIALI BACK UP ---------------------------------------------------------------------------------