Customer is running 2.12.1-394

it seems that at a certain point the metrics endpoint is not responding with:

W0730 00:00:35.586215       1 logging.go:59] [core] [Channel #1 SubChannel #4] grpc: addrConn.createTransport failed to connect to {Addr: "keda-operator.xxxxx.svc.cluster.local:9666", ServerName: "keda-operator.xxxxxx.svc.cluster.local:9666", }. Err: connection error: desc = "transport: Error while dialing: dial tcp 172.30.x.y:9666: connect: connection refused"

inmediately after as the request is returning nil vaiue, it's crashing with this stack trace:

E0729 21:44:16.362477       1 timeout.go:142] post-timeout activity - time-elapsed: 85.660546ms, GET "/apis/external.metrics.k8s.io/v1beta1/namespaces/<namespace>/<metrics endpoint>" result: runtime error: invalid memory address or nil pointer dereference
goroutine 21782 [running]:
k8s.io/apiserver/pkg/server/filters.(*timeoutHandler).ServeHTTP.func1.1()
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/server/filters/timeout.go:110 +0x9c
panic({0x2684640?, 0x484a8f0?})
	/usr/lib/golang/src/runtime/panic.go:914 +0x21f
sigs.k8s.io/custom-metrics-apiserver/pkg/registry/external_metrics.(*REST).List(0xc0009a2480, {0x2fe3038, 0xc0014d7170}, 0x0?)
	/remote-source/keda/app/vendor/sigs.k8s.io/custom-metrics-apiserver/pkg/registry/external_metrics/reststorage.go:92 +0x112
k8s.io/apiserver/pkg/endpoints/handlers.ListResource.func1({0x2fd3f20?, 0xc00123ef40}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/handlers/get.go:278 +0xe9c
sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/installer.(*EMHandlers).registerResourceHandlers.restfulListResource.func1(0xc00123ef20, 0xc0004dd3b0)
	/remote-source/keda/app/vendor/sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/installer/installer.go:274 +0x6a
sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/installer.(*EMHandlers).registerResourceHandlers.InstrumentRouteFunc.func2(0xc00123ef20, 0xc0004dd3b0)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go:571 +0x1ce
github.com/emicklei/go-restful/v3.(*Container).dispatch(0xc001568ea0, {0x2fd3f20, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/github.com/emicklei/go-restful/v3/container.go:299 +0x5f0
github.com/emicklei/go-restful/v3.(*Container).Dispatch(...)
	/remote-source/keda/app/vendor/github.com/emicklei/go-restful/v3/container.go:204
k8s.io/apiserver/pkg/server.director.ServeHTTP({{0x2b20adf?, 0x29b30a0?}, 0xc001568ea0?, 0xc0006d4930?}, {0x2fd3f20, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/server/handler.go:146 +0x56c
k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.TrackCompleted.trackCompleted.func21({0x2fd3f20?, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:110 +0x177
net/http.HandlerFunc.ServeHTTP(0x2fe3038?, {0x2fd3f20?, 0xc00123e140?}, 0x4?)
	/usr/lib/golang/src/net/http/server.go:2136 +0x29
k8s.io/apiserver/pkg/endpoints/filters.withAuthorization.func1({0x2fd3f20, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/filters/authorization.go:78 +0x639
net/http.HandlerFunc.ServeHTTP(0xc1a2211d106f5039?, {0x2fd3f20?, 0xc00123e140?}, 0xc0009090b8?)
	/usr/lib/golang/src/net/http/server.go:2136 +0x29
k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1({0x2fd3f20, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:84 +0x192
net/http.HandlerFunc.ServeHTTP(0x4864600?, {0x2fd3f20?, 0xc00123e140?}, 0x4?)
	/usr/lib/golang/src/net/http/server.go:2136 +0x29
k8s.io/apiserver/pkg/server/filters.WithMaxInFlightLimit.func1({0x2fd3f20, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/server/filters/maxinflight.go:196 +0x262
net/http.HandlerFunc.ServeHTTP(0x1909f6f?, {0x2fd3f20?, 0xc00123e140?}, 0x2fab540?)
	/usr/lib/golang/src/net/http/server.go:2136 +0x29
k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.TrackCompleted.trackCompleted.func23({0x2fd3f20?, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:110 +0x177
net/http.HandlerFunc.ServeHTTP(0x1?, {0x2fd3f20?, 0xc00123e140?}, 0x1?)
	/usr/lib/golang/src/net/http/server.go:2136 +0x29
k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.WithImpersonation.func4({0x2fd3f20, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go:50 +0x1c3
net/http.HandlerFunc.ServeHTTP(0x829de0?, {0x2fd3f20?, 0xc00123e140?}, 0x0?)
	/usr/lib/golang/src/net/http/server.go:2136 +0x29
k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1({0x2fd3f20, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:84 +0x192
net/http.HandlerFunc.ServeHTTP(0x1909f6f?, {0x2fd3f20?, 0xc00123e140?}, 0x2fab540?)
	/usr/lib/golang/src/net/http/server.go:2136 +0x29
k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.TrackCompleted.trackCompleted.func24({0x2fd3f20?, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:110 +0x177
net/http.HandlerFunc.ServeHTTP(0x829de0?, {0x2fd3f20?, 0xc00123e140?}, 0x1?)
	/usr/lib/golang/src/net/http/server.go:2136 +0x29
k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1({0x2fd3f20, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:84 +0x192
net/http.HandlerFunc.ServeHTTP(0x1909f6f?, {0x2fd3f20?, 0xc00123e140?}, 0x2fab540?)
	/usr/lib/golang/src/net/http/server.go:2136 +0x29
k8s.io/apiserver/pkg/server.DefaultBuildHandlerChain.TrackCompleted.trackCompleted.func26({0x2fd3f20?, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:110 +0x177
net/http.HandlerFunc.ServeHTTP(0x2fe3038?, {0x2fd3f20?, 0xc00123e140?}, 0x2fa5230?)
	/usr/lib/golang/src/net/http/server.go:2136 +0x29
k8s.io/apiserver/pkg/endpoints/filters.withAuthentication.func1({0x2fd3f20, 0xc00123e140}, 0xc000e0cc00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/filters/authentication.go:105 +0x6c5
net/http.HandlerFunc.ServeHTTP(0x2fe3038?, {0x2fd3f20?, 0xc00123e140?}, 0x2fab540?)
	/usr/lib/golang/src/net/http/server.go:2136 +0x29
k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1({0x2fd3f20, 0xc00123e140}, 0xc000e0ca00)
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:94 +0x37a
net/http.HandlerFunc.ServeHTTP(0xc000fb4f68?, {0x2fd3f20?, 0xc00123e140?}, 0x9e80cc?)
	/usr/lib/golang/src/net/http/server.go:2136 +0x29
k8s.io/apiserver/pkg/server/filters.(*timeoutHandler).ServeHTTP.func1()
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/server/filters/timeout.go:115 +0x62
created by k8s.io/apiserver/pkg/server/filters.(*timeoutHandler).ServeHTTP in goroutine 21781
	/remote-source/keda/app/vendor/k8s.io/apiserver/pkg/server/filters/timeout.go:101 +0x1b2



    

       
   CMA 2.12.1-394 / OCP 4.14