vipikuma@kvy:~/rhwa$ oc get clusterversion oc get nodes NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.21.0-0.nightly-2026-02-18-135253 True False 7m28s Cluster version is 4.21.0-0.nightly-2026-02-18-135253 NAME STATUS ROLES AGE VERSION ip-10-0-10-253.us-west-1.compute.internal Ready control-plane,master 34m v1.34.2 ip-10-0-114-234.us-west-1.compute.internal Ready control-plane,master 34m v1.34.2 ip-10-0-55-237.us-west-1.compute.internal Ready worker 18m v1.34.2 ip-10-0-67-241.us-west-1.compute.internal Ready worker 26m v1.34.2 ip-10-0-86-220.us-west-1.compute.internal Ready control-plane,master 34m v1.34.2 ip-10-0-89-20.us-west-1.compute.internal Ready worker 26m v1.34.2 vipikuma@kvy:~/rhwa$ oc get csv NAME DISPLAY VERSION REPLACES PHASE node-healthcheck-operator.v0.11.0 Node Health Check Operator 0.11.0 node-healthcheck-operator.v0.10.1 Succeeded self-node-remediation.v0.12.0 Self Node Remediation Operator 0.12.0 self-node-remediation.v0.11.0 Succeeded vipikuma@kvy:~/rhwa$ oc get pods NAME READY STATUS RESTARTS AGE node-healthcheck-controller-manager-5c6b78985f-2jmcg 2/2 Running 0 3m9s node-healthcheck-controller-manager-5c6b78985f-f6k8w 2/2 Running 0 3m9s node-healthcheck-node-remediation-console-plugin-79d48b86dp89qf 1/1 Running 0 3m9s self-node-remediation-controller-manager-54495f59b-gfqfj 2/2 Running 0 3m7s self-node-remediation-controller-manager-54495f59b-wk5zj 2/2 Running 1 (2m49s ago) 3m7s self-node-remediation-ds-jdkp5 1/1 Running 0 2m33s self-node-remediation-ds-kg789 1/1 Running 0 2m33s self-node-remediation-ds-kqg7l 1/1 Running 0 2m34s self-node-remediation-ds-pxscz 1/1 Running 0 2m33s self-node-remediation-ds-qbgd7 1/1 Running 0 2m33s self-node-remediation-ds-xghpv 1/1 Running 0 2m33s vipikuma@kvy:~/rhwa$ PODS=$(oc get pods -o name -n openshift-workload-availability | grep self-node-remediation-controller-manager) echo $PODS for p in $PODS; do echo "== $p" oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName done pod/self-node-remediation-controller-manager-54495f59b-gfqfj pod/self-node-remediation-controller-manager-54495f59b-wk5zj == pod/self-node-remediation-controller-manager-54495f59b-gfqfj "ip-10-0-67-241.us-west-1.compute.internal" == pod/self-node-remediation-controller-manager-54495f59b-wk5zj "ip-10-0-55-237.us-west-1.compute.internal" vipikuma@kvy:~/rhwa$ oc debug node/ip-10-0-89-20.us-west-1.compute.internal -- chroot /host bash -c "date" Temporary namespace openshift-debug-mz8s2 is created for debugging node... Starting pod/ip-10-0-89-20us-west-1computeinternal-debug-66l5j ... To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`. Sun Feb 22 19:17:08 UTC 2026 Removing debug pod ... Temporary namespace openshift-debug-mz8s2 was removed. vipikuma@kvy:~/rhwa$ oc get node ip-10-0-89-20.us-west-1.compute.internal -o yaml | yq .spec.taints null vipikuma@kvy:~/rhwa$ oc debug node/ip-10-0-89-20.us-west-1.compute.internal -- chroot /host bash -c "systemctl stop kubelet" Temporary namespace openshift-debug-tdw9q is created for debugging node... Starting pod/ip-10-0-89-20us-west-1computeinternal-debug-zvjzw ... To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`. vipikuma@kvy:~/rhwa$ oc get nodes NAME STATUS ROLES AGE VERSION ip-10-0-10-253.us-west-1.compute.internal Ready control-plane,master 45m v1.34.2 ip-10-0-114-234.us-west-1.compute.internal Ready control-plane,master 45m v1.34.2 ip-10-0-55-237.us-west-1.compute.internal Ready worker 30m v1.34.2 ip-10-0-67-241.us-west-1.compute.internal Ready worker 37m v1.34.2 ip-10-0-86-220.us-west-1.compute.internal Ready control-plane,master 45m v1.34.2 ip-10-0-89-20.us-west-1.compute.internal NotReady worker 37m v1.34.2 vipikuma@kvy:~/rhwa$ cat test.yaml apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: name: nhc-worker-test spec: minHealthy: 1 healthyDelay: 5s stormCooldownDuration: 60s remediationTemplate: apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate name: selfnoderemediationtemplate-sample namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/worker operator: Exists unhealthyConditions: - duration: 30s status: 'False' type: Ready - duration: 30s status: Unknown type: Ready --- apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate metadata: namespace: openshift-workload-availability name: selfnoderemediationtemplate-sample spec: template: spec: remediationStrategy: Automatic vipikuma@kvy:~/rhwa$ oc apply -f test.yaml nodehealthcheck.remediation.medik8s.io/nhc-worker-test created selfnoderemediationtemplate.self-node-remediation.medik8s.io/selfnoderemediationtemplate-sample created vipikuma@kvy:~/rhwa$ oc get nhc NAME AGE nhc-worker-test 13s vipikuma@kvy:~/rhwa$ oc get snr NAME AGE ip-10-0-89-20.us-west-1.compute.internal-7m8dl 25s vipikuma@kvy:~/rhwa$ oc get node ip-10-0-89-20.us-west-1.compute.internal -o yaml | yq .spec.taints - effect: NoSchedule key: node.kubernetes.io/unreachable timeAdded: "2026-02-22T19:19:08Z" - effect: NoExecute key: node.kubernetes.io/unreachable timeAdded: "2026-02-22T19:19:08Z" - effect: NoSchedule key: remediation.medik8s.io/self-node-remediation timeAdded: "2026-02-22T19:20:19Z" vipikuma@kvy:~/rhwa$ oc get nodes NAME STATUS ROLES AGE VERSION ip-10-0-10-253.us-west-1.compute.internal Ready control-plane,master 47m v1.34.2 ip-10-0-114-234.us-west-1.compute.internal Ready control-plane,master 47m v1.34.2 ip-10-0-55-237.us-west-1.compute.internal Ready worker 32m v1.34.2 ip-10-0-67-241.us-west-1.compute.internal Ready worker 39m v1.34.2 ip-10-0-86-220.us-west-1.compute.internal Ready control-plane,master 47m v1.34.2 ip-10-0-89-20.us-west-1.compute.internal Ready worker 39m v1.34.2 vipikuma@kvy:~/rhwa$ oc get snr NAME AGE ip-10-0-89-20.us-west-1.compute.internal-7m8dl 110s vipikuma@kvy:~/rhwa$ oc get nodes NAME STATUS ROLES AGE VERSION ip-10-0-10-253.us-west-1.compute.internal Ready control-plane,master 48m v1.34.2 ip-10-0-114-234.us-west-1.compute.internal Ready control-plane,master 48m v1.34.2 ip-10-0-55-237.us-west-1.compute.internal Ready worker 33m v1.34.2 ip-10-0-67-241.us-west-1.compute.internal Ready worker 41m v1.34.2 ip-10-0-86-220.us-west-1.compute.internal Ready control-plane,master 48m v1.34.2 ip-10-0-89-20.us-west-1.compute.internal NotReady worker 41m v1.34.2 vipikuma@kvy:~/rhwa$ oc get snr NAME AGE ip-10-0-89-20.us-west-1.compute.internal-784gf 3s vipikuma@kvy:~/rhwa$ oc logs pod/self-node-remediation-controller-manager-54495f59b-gfqfj Defaulted container "manager" out of: manager, kube-rbac-proxy 2026-02-22T19:12:57.307800757Z INFO setup Go Version: go1.25.3 (Red Hat 1.25.3-1.el9_7) X:strictfipsruntime 2026-02-22T19:12:57.307887872Z INFO setup Go OS/Arch: linux/amd64 2026-02-22T19:12:57.307891472Z INFO setup Operator Version: 3477dbe 2026-02-22T19:12:57.307893862Z INFO setup Git Commit: 3477dbe48d1c5f3e04213c5917ccb1013278d75f 2026-02-22T19:12:57.307896073Z INFO setup Build Date: 2026-01-28T09:37:18+00:00 2026-02-22T19:12:57.307898283Z INFO setup HTTP/2 for metrics and webhook server disabled 2026-02-22T19:12:57.315008342Z INFO utils-taints out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 34} 2026-02-22T19:12:57.315033083Z INFO utils-taints out of service taint strategy {"isGA": true, "k8sMajorVersion": 1, "k8sMinorVersion": 34} 2026-02-22T19:12:57.315039514Z INFO setup Starting as a manager that installs the daemonset 2026-02-22T19:12:57.315062685Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2026-02-22T19:12:57.315130869Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2026-02-22T19:12:57.315171001Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2026-02-22T19:12:57.315206783Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2026-02-22T19:12:57.315255786Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2026-02-22T19:12:57.315297328Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2026-02-22T19:12:57.31533409Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2026-02-22T19:12:57.315364772Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2026-02-22T19:12:57.315438686Z INFO setup starting manager 2026-02-22T19:12:57.315628747Z INFO controller-runtime.metrics Starting metrics server 2026-02-22T19:12:57.315702911Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-02-22T19:12:57.315730602Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": "127.0.0.1:8080", "secure": false} 2026-02-22T19:12:57.315750224Z INFO controller-runtime.webhook Starting webhook server I0222 19:12:57.315795 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/547f6cb6.medik8s.io... 2026-02-22T19:12:57.316018229Z INFO controller-runtime.certwatcher Updated current TLS certificate 2026-02-22T19:12:57.316145746Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-02-22T19:12:57.316206469Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} 2026-02-22T19:13:03.262153308Z DEBUG controller-runtime.certwatcher certificate event {"event": "CHMOD \"/apiserver.local.config/certificates/apiserver.crt\""} 2026-02-22T19:13:03.262483787Z DEBUG controller-runtime.certwatcher certificate event {"event": "CHMOD \"/apiserver.local.config/certificates/apiserver.key\""} I0222 19:13:21.973076 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/547f6cb6.medik8s.io 2026-02-22T19:13:21.973152038Z DEBUG events self-node-remediation-controller-manager-54495f59b-gfqfj_16be45ba-58d3-4403-9465-074a2e07aaf5 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"547f6cb6.medik8s.io","uid":"f85558b9-1cd2-4320-b786-878fb2d5c394","apiVersion":"coordination.k8s.io/v1","resourceVersion":"36951"}, "reason": "LeaderElection"} 2026-02-22T19:13:21.97353698Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1.DaemonSet"} 2026-02-22T19:13:21.97371396Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1alpha1.SelfNodeRemediationConfig"} 2026-02-22T19:13:21.973781894Z INFO Starting EventSource {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "source": "kind source: *v1alpha1.SelfNodeRemediation"} 2026-02-22T19:13:22.082587159Z INFO Starting Controller {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig"} 2026-02-22T19:13:22.082616381Z INFO Starting workers {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "worker count": 1} 2026-02-22T19:13:22.08260085Z INFO Starting Controller {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation"} 2026-02-22T19:13:22.082637012Z INFO Starting workers {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "worker count": 1} 2026-02-22T19:13:22.082743758Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2026-02-22T19:13:22.183081407Z INFO controllers.SelfNodeRemediationConfig Creating new certs 2026-02-22T19:13:22.943051411Z INFO controllers.SelfNodeRemediationConfig Storing certs in new secret 2026-02-22T19:13:22.9510345Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2026-02-22T19:13:22.951827385Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2026-02-22T19:13:22.951854516Z INFO controllers.SelfNodeRemediationConfig snr didn't find old daemonset to be deleted 2026/02/22 19:13:22 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2026/02/22 19:13:22 does not exist, creating (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2026/02/22 19:13:22 successfully created (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2026-02-22T19:13:22.969384823Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2026-02-22T19:13:22.969409015Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2026-02-22T19:13:22.969415115Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2026-02-22T19:13:22.969912543Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2026/02/22 19:13:22 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2026/02/22 19:13:22 update was successful 2026-02-22T19:13:31.979034887Z INFO selfnoderemediationtemplate-resource default {"name": "self-node-remediation-automatic-strategy-template"} 2026-02-22T19:20:04.43184485Z INFO selfnoderemediationtemplate-resource default {"name": "selfnoderemediationtemplate-sample"} 2026-02-22T19:20:04.437873318Z INFO selfnoderemediationtemplate-resource validate create {"name": "selfnoderemediationtemplate-sample"} 2026-02-22T19:20:19.084126011Z INFO selfnoderemediation-resource validate create {"name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl"} 2026-02-22T19:20:19.191252857Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:20:19.191286179Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:20:19.191281359Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","uid":"57b3e2b2-f7cf-409d-8ede-d0e54fd0f999","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"39439"}, "reason": "RemediationStarted"} 2026-02-22T19:20:19.295420597Z INFO selfnoderemediation-resource validate update {"name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl"} 2026-02-22T19:20:19.300531994Z INFO controllers.SelfNodeRemediation finalizer added {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:20:19.300664541Z DEBUG events [remediation] Remediation process - successful adding finalizer {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","uid":"57b3e2b2-f7cf-409d-8ede-d0e54fd0f999","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"39444"}, "reason": "AddFinalizer"} 2026-02-22T19:20:19.308251746Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:20:19.30831858Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:20:19.320693334Z INFO controllers.SelfNodeRemediation NoSchedule taint added {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2026-02-22T19:19:08Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-22T19:19:08Z"},{"key":"remediation.medik8s.io/self-node-remediation","effect":"NoSchedule","timeAdded":"2026-02-22T19:20:19Z"}]} 2026-02-22T19:20:19.320850122Z DEBUG events [remediation] Remediation process - NoSchedule taint added to the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"ip-10-0-89-20.us-west-1.compute.internal","uid":"0499146a-d0d4-46a1-8365-c41e080e1a7a","apiVersion":"v1","resourceVersion":"39447"}, "reason": "AddNoScheduleTaint"} 2026-02-22T19:20:19.321016472Z INFO rebootDurationCalculator No SafeTimeToAssumeNodeRebootedSeconds specified, using calculated minimum safe reboot time {"calculated minimum time in seconds": 120} 2026-02-22T19:20:19.321039833Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "node name": "ip-10-0-89-20.us-west-1.compute.internal", "time": "2026-02-22 19:22:19.321037573 +0000 UTC m=+562.026858841"} 2026-02-22T19:20:19.32115223Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","uid":"57b3e2b2-f7cf-409d-8ede-d0e54fd0f999","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"39444"}, "reason": "UpdateTimeAssumedRebooted"} 2026-02-22T19:20:19.329606144Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:20:19.329667807Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:20:19.330026407Z INFO rebootDurationCalculator No SafeTimeToAssumeNodeRebootedSeconds specified, using calculated minimum safe reboot time {"calculated minimum time in seconds": 120} 2026-02-22T19:20:19.330067979Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "node name": "ip-10-0-89-20.us-west-1.compute.internal", "time": "2026-02-22 19:22:19.330066579 +0000 UTC m=+562.035887867"} 2026-02-22T19:20:19.330191866Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","uid":"57b3e2b2-f7cf-409d-8ede-d0e54fd0f999","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"39445"}, "reason": "UpdateTimeAssumedRebooted"} 2026-02-22T19:20:19.338125881Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:20:19.338140682Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "node name": "ip-10-0-89-20.us-west-1.compute.internal", "time left": "2m0.661860248s"} 2026-02-22T19:20:20.330131876Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:20:20.330169618Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "node name": "ip-10-0-89-20.us-west-1.compute.internal", "time left": "1m59.669831542s"} 2026-02-22T19:21:19.366961253Z INFO selfnoderemediation-resource validate update {"name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl"} 2026-02-22T19:21:19.372414289Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:21:19.37243118Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "node name": "ip-10-0-89-20.us-west-1.compute.internal", "time left": "1m0.62756977s"} 2026-02-22T19:21:25.041058788Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:21:25.041078109Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "node name": "ip-10-0-89-20.us-west-1.compute.internal", "time left": "54.958922631s"} 2026-02-22T19:22:20.006845032Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:20.006884264Z INFO controllers.SelfNodeRemediation TimeAssumedRebooted is old. The unhealthy node assumed to been rebooted {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "node name": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:22:20.015729802Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:20.027558237Z INFO controllers.SelfNodeRemediation out-of-service taint added {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "new taints": [{"key":"remediation.medik8s.io/self-node-remediation","effect":"NoSchedule","timeAdded":"2026-02-22T19:20:19Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2026-02-22T19:22:09Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-22T19:22:09Z"},{"key":"node.kubernetes.io/out-of-service","value":"nodeshutdown","effect":"NoExecute","timeAdded":"2026-02-22T19:22:20Z"}]} 2026-02-22T19:22:20.027774129Z DEBUG events [remediation] Remediation process - add out-of-service taint to unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"ip-10-0-89-20.us-west-1.compute.internal","uid":"0499146a-d0d4-46a1-8365-c41e080e1a7a","apiVersion":"v1","resourceVersion":"40152"}, "reason": "AddOutOfService"} 2026-02-22T19:22:20.132915319Z INFO controllers.SelfNodeRemediation waiting for deleting volumeAttachement {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "name": "csi-4865f1cbad1ba56997254b569c6d2601cd8c3fe94be38516969728d917a31afe"} 2026-02-22T19:22:25.143723481Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:25.148249056Z INFO controllers.SelfNodeRemediation waiting for deleting volumeAttachement {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "name": "csi-4865f1cbad1ba56997254b569c6d2601cd8c3fe94be38516969728d917a31afe"} 2026-02-22T19:22:30.156064933Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:30.175155872Z INFO controllers.SelfNodeRemediation out-of-service taint removed {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "new taints": [{"key":"remediation.medik8s.io/self-node-remediation","effect":"NoSchedule","timeAdded":"2026-02-22T19:20:19Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2026-02-22T19:22:09Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-22T19:22:09Z"}]} 2026-02-22T19:22:30.175323592Z DEBUG events [remediation] Remediation process - remove out-of-service taint from node {"type": "Normal", "object": {"kind":"Node","name":"ip-10-0-89-20.us-west-1.compute.internal","uid":"0499146a-d0d4-46a1-8365-c41e080e1a7a","apiVersion":"v1","resourceVersion":"40639"}, "reason": "RemoveOutOfService"} 2026-02-22T19:22:30.175406876Z DEBUG events [remediation] Remediation process - finished deleting unhealthy node resources {"type": "Normal", "object": {"kind":"Node","name":"ip-10-0-89-20.us-west-1.compute.internal","uid":"0499146a-d0d4-46a1-8365-c41e080e1a7a","apiVersion":"v1","resourceVersion":"40639"}, "reason": "DeleteResources"} 2026-02-22T19:22:30.185712999Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:30.185792324Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:30.198741236Z INFO controllers.SelfNodeRemediation NoSchedule taint removed {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2026-02-22T19:22:09Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-22T19:22:09Z"}]} 2026-02-22T19:22:30.199033752Z DEBUG events [remediation] Remediation process - remove NoSchedule taint from healthy remediated node {"type": "Normal", "object": {"kind":"Node","name":"ip-10-0-89-20.us-west-1.compute.internal","uid":"0499146a-d0d4-46a1-8365-c41e080e1a7a","apiVersion":"v1","resourceVersion":"40643"}, "reason": "RemoveNoScheduleTaint"} 2026-02-22T19:22:30.204340042Z INFO selfnoderemediation-resource validate update {"name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl"} 2026-02-22T19:22:30.232762869Z INFO controllers.SelfNodeRemediation finalizer removed {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:30.233209604Z DEBUG events [remediation] Remediation process - remove finalizer from snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","uid":"57b3e2b2-f7cf-409d-8ede-d0e54fd0f999","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40641"}, "reason": "RemoveFinalizer"} 2026-02-22T19:22:30.23330524Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","uid":"57b3e2b2-f7cf-409d-8ede-d0e54fd0f999","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40641"}, "reason": "RemediationFinished"} 2026-02-22T19:22:30.242347211Z ERROR controllers.SelfNodeRemediation failed to update snr status {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "error": "selfnoderemediations.self-node-remediation.medik8s.io \"ip-10-0-89-20.us-west-1.compute.internal-7m8dl\" not found"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).updateSnrStatus /app/self-node-remediation/controllers/selfnoderemediation_controller.go:671 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager.func1 /app/self-node-remediation/controllers/selfnoderemediation_controller.go:209 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:313 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:30.242454327Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "reconcileID": "514be405-0182-402b-bb48-e566e6856ea8", "error": "selfnoderemediations.self-node-remediation.medik8s.io \"ip-10-0-89-20.us-west-1.compute.internal-7m8dl\" not found", "errorCauses": [{"error": "selfnoderemediations.self-node-remediation.medik8s.io \"ip-10-0-89-20.us-west-1.compute.internal-7m8dl\" not found"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:30.242536112Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:30.254658117Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.047808768Z INFO selfnoderemediation-resource validate create {"name": "ip-10-0-89-20.us-west-1.compute.internal-784gf"} 2026-02-22T19:22:40.052748807Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.052763298Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.052881955Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40740"}, "reason": "RemediationStarted"} 2026-02-22T19:22:40.053357461Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.066544227Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "4f2a89d2-eb89-422e-81de-d348bdd16087", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.066702826Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.066717807Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.06694969Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.066935539Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40742"}, "reason": "RemediationStarted"} 2026-02-22T19:22:40.075577327Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "ff715659-135d-4b89-9b5e-1f7669c5c9bc", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.07579369Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.075838382Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.075851243Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40744"}, "reason": "RemediationStarted"} 2026-02-22T19:22:40.076406294Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.083890498Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "ee879607-8116-4ca3-a4d3-c5701d2296b1", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.084072658Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.08410889Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.084354854Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40744"}, "reason": "RemediationStarted"} 2026-02-22T19:22:40.084633719Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.08995156Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "640cd247-7601-4779-9282-160f3a014e63", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.090085598Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.090094468Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.090174003Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40744"}, "reason": "RemediationStarted"} 2026-02-22T19:22:40.090496511Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.097271634Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "b4771d01-17a1-40ee-b801-417b22bc3da8", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.130442839Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.130471421Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.130481591Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40744"}, "reason": "RemediationStarted"} 2026-02-22T19:22:40.130681023Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.13663984Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "b1213233-51bc-4876-bf65-85f98f496526", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.297633641Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.297652062Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.297930868Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.298056425Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40744"}, "reason": "RemediationStarted"} 2026-02-22T19:22:40.304853069Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "ba855d5a-1904-4743-9818-4ddfc7de83aa", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.62610083Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.626121761Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:40.626193505Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40744"}, "reason": "RemediationStarted"} 2026-02-22T19:22:40.62644867Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:40.632525633Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "eaa3d05c-8584-4a0d-bd4b-ec7a8f451461", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:41.273247335Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:41.273270157Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:41.273350691Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40744"}, "reason": "RemediationStarted"} 2026-02-22T19:22:41.273660069Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:41.279930883Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "8882112f-7656-409e-a2eb-ace50faf37ee", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:42.560764862Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:42.560789594Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:42.560818936Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40744"}, "reason": "RemediationStarted"} 2026-02-22T19:22:42.56107189Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:42.567720146Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "ee07e965-b4bf-457f-bec1-6528507218c5", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:45.12818325Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:45.128202851Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:45.128266665Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40744"}, "reason": "RemediationStarted"} 2026-02-22T19:22:45.128691059Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:45.134484625Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "3e5e41e2-3f34-4745-aa4a-54275d7f8dd3", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:50.254760105Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:50.254785266Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:22:50.255082893Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:22:50.255576841Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40744"}, "reason": "RemediationStarted"} 2026-02-22T19:22:50.264570227Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "95b2b81c-fef3-4456-8715-75f93448acd4", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:23:00.504756053Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:23:00.504780584Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:23:00.504852578Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40744"}, "reason": "RemediationStarted"} 2026-02-22T19:23:00.50506691Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:23:00.512031034Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "915f4945-c14d-4b61-90e3-bd244304f379", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:23:20.992943141Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:23:20.992964782Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}} 2026-02-22T19:23:20.993028395Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","uid":"fd90dbe9-01e8-47ee-a81a-130670a9da4b","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"40744"}, "reason": "RemediationStarted"} 2026-02-22T19:23:20.993483141Z ERROR controllers.SelfNodeRemediation failed to get self node remediation agent pod resource {"pod": "manager", "selfnoderemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "error": "failed to find self node remediation pod matching the given node"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).isNodeRebootCapable /app/self-node-remediation/controllers/selfnoderemediation_controller.go:611 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot /app/self-node-remediation/controllers/selfnoderemediation_controller.go:479 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase /app/self-node-remediation/controllers/selfnoderemediation_controller.go:474 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval /app/self-node-remediation/controllers/selfnoderemediation_controller.go:458 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint /app/self-node-remediation/controllers/selfnoderemediation_controller.go:415 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:306 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:151 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-22T19:23:20.999532826Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"ip-10-0-89-20.us-west-1.compute.internal-784gf","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "reconcileID": "e4b0e64f-cb09-4f2b-a2de-a0c89a6376a2", "error": "Node is not capable to reboot itself", "errorVerbose": "Node is not capable to reboot itself\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).prepareReboot\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:481\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleFencingStartedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:474\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:458\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:415\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:306\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:151\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 vipikuma@kvy:~/rhwa$ vipikuma@kvy:~/rhwa$ oc logs pod/node-healthcheck-controller-manager-5c6b78985f-f6k8w 2026-02-22T19:13:00.417454089Z INFO setup Go Version: go1.25.3 (Red Hat 1.25.3-1.el9_7) X:strictfipsruntime 2026-02-22T19:13:00.417599016Z INFO setup Go OS/Arch: linux/amd64 2026-02-22T19:13:00.417605236Z INFO setup Operator Version: acf7a94 2026-02-22T19:13:00.417608976Z INFO setup Git Commit: acf7a94540e8f527122bbd1dc102a46ca985cb82 2026-02-22T19:13:00.417612586Z INFO setup Build Date: 2026-02-11T14:06:15+00:00 2026-02-22T19:13:00.417616166Z INFO setup HTTP/2 for metrics and webhook server disabled 2026-02-22T19:13:00.44846301Z INFO setup supported control plane topology {"topology": "HighlyAvailable"} 2026-02-22T19:13:00.448564765Z INFO setup Cluster capabilities {"IsOnOpenshift": true, "HasMachineAPI": true} 2026-02-22T19:13:00.448868869Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "remediation.medik8s.io/v1alpha1, Kind=NodeHealthCheck", "path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2026-02-22T19:13:00.448996434Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2026-02-22T19:13:00.449100189Z INFO setup starting manager 2026-02-22T19:13:00.449173793Z INFO controller-runtime.metrics Starting metrics server 2026-02-22T19:13:00.449214284Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-02-22T19:13:00.449303859Z INFO controller-runtime.webhook Starting webhook server 2026-02-22T19:13:00.449325569Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": "127.0.0.1:8080", "secure": false} 2026-02-22T19:13:00.449621193Z INFO controller-runtime.certwatcher Updated current TLS certificate 2026-02-22T19:13:00.449728228Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-02-22T19:13:00.449803081Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} I0222 19:13:00.549857 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/e1f13584.medik8s.io... I0222 19:13:00.561776 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/e1f13584.medik8s.io 2026-02-22T19:13:00.56184161Z DEBUG events node-healthcheck-controller-manager-5c6b78985f-f6k8w_669f42d4-7747-4262-9630-4564950d0b09 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"e1f13584.medik8s.io","uid":"bebdb776-792e-48cc-953e-d30938fa9665","apiVersion":"coordination.k8s.io/v1","resourceVersion":"36411"}, "reason": "LeaderElection"} 2026-02-22T19:13:00.562101312Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "channel source: 0xc000224070"} 2026-02-22T19:13:00.562231258Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1alpha1.NodeHealthCheck"} 2026-02-22T19:13:00.562233648Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1.Node"} 2026-02-22T19:13:00.562341103Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "channel source: 0xc0002242a0"} 2026-02-22T19:13:00.562399355Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.MachineHealthCheck"} 2026-02-22T19:13:00.562406246Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.Machine"} 2026-02-22T19:13:00.562419606Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1.Node"} I0222 19:13:00.564687 1 shared_informer.go:350] "Waiting for caches to sync" controller="feature gate accessor" 2026-02-22T19:13:00.662577505Z INFO MHCChecker found termination handler MHC, will ignore Nodes with Terminating condition 2026-02-22T19:13:00.662684409Z INFO MHCChecker MHC Checker status changed, notifying NHC controller 2026-02-22T19:13:00.662739952Z INFO adding all NHCs to reconcile queue for handling MHC event I0222 19:13:00.665830 1 shared_informer.go:357] "Caches are synced" controller="feature gate accessor" I0222 19:13:00.665897 1 simple_featuregate_reader.go:171] Starting feature-gate-detector 2026-02-22T19:13:00.666097945Z INFO FeatureGateAccessor FeatureGates initialized I0222 19:13:00.666179 1 recorder_logging.go:49] &Event{ObjectMeta:{dummy.1896a76ad30c776c.ec1e899a dummy 0 0001-01-01 00:00:00 +0000 UTC map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Pod,Namespace:dummy,Name:dummy,UID:,APIVersion:v1,ResourceVersion:,FieldPath:,},Reason:FeatureGatesInitialized,Message:FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "NewOLM", "NewOLMWebhookProviderOpenshiftServiceCA", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "HyperShiftOnlyDynamicResourceAllocation", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "OpenShiftPodSecurityAdmission", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}},Source:EventSource{Component:,Host:,},FirstTimestamp:2026-02-22 19:13:00.666050412 +0000 UTC m=+0.273727988,LastTimestamp:2026-02-22 19:13:00.666050412 +0000 UTC m=+0.273727988,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:,ReportingInstance:,} 2026-02-22T19:13:00.763654834Z INFO Starting Controller {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck"} 2026-02-22T19:13:00.763739578Z INFO Starting workers {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "worker count": 1} 2026-02-22T19:13:00.763663285Z INFO Starting Controller {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck"} 2026-02-22T19:13:00.763863774Z INFO Starting workers {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "worker count": 1} 2026-02-22T19:13:00.988317599Z INFO console-plugin successfully created / updated console plugin resources 2026-02-22T19:20:03.607758076Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:20:03.630387681Z INFO controllers.NodeHealthCheck disabling NHC {"NodeHealthCheck name": "nhc-worker-test", "reason": "RemediationTemplateNotFound", "message": "Remediation template not found: \"failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \\\"selfnoderemediationtemplate-sample\\\" not found\""} 2026-02-22T19:20:03.630961718Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":0,"healthyNodes":0,"conditions":[{"type":"Disabled","status":"True","lastTransitionTime":"2026-02-22T19:20:03Z","reason":"RemediationTemplateNotFound","message":"Remediation template not found: \"failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \\\"selfnoderemediationtemplate-sample\\\" not found\""}],"phase":"Disabled","reason":"NHC is disabled: RemediationTemplateNotFound: Remediation template not found: \"failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \\\"selfnoderemediationtemplate-sample\\\" not found\""}, "patch": "{\"status\":{\"conditions\":[{\"lastTransitionTime\":\"2026-02-22T19:20:03Z\",\"message\":\"Remediation template not found: \\\"failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \\\\\\\"selfnoderemediationtemplate-sample\\\\\\\" not found\\\"\",\"reason\":\"RemediationTemplateNotFound\",\"status\":\"True\",\"type\":\"Disabled\"}],\"healthyNodes\":0,\"observedNodes\":0,\"phase\":\"Disabled\",\"reason\":\"NHC is disabled: RemediationTemplateNotFound: Remediation template not found: \\\"failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \\\\\\\"selfnoderemediationtemplate-sample\\\\\\\" not found\\\"\"}}"} 2026-02-22T19:20:03.63143865Z DEBUG events [remediation] Disabling NHC. Reason: RemediationTemplateNotFound, Message: Remediation template not found: "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" not found" {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"39325"}, "reason": "Disabled"} 2026-02-22T19:20:03.845733697Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "15s"} 2026-02-22T19:20:18.847031028Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:20:18.854060162Z INFO controllers.NodeHealthCheck enabling NHC, valid config, no conflicting MHC configured in the cluster {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:20:18.854141186Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2026-02-22T19:20:18.854160617Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation template CRs {"kind": "SelfNodeRemediationTemplate"} 2026-02-22T19:20:18.854171127Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2026-02-22T19:20:18.854177848Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation CRs {"kind": "SelfNodeRemediation"} 2026-02-22T19:20:18.854269422Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "ip-10-0-89-20.us-west-1.compute.internal", "condition type": "Ready", "condition status": "Unknown"} 2026-02-22T19:20:18.854466181Z DEBUG events [remediation] No issues found, NodeHealthCheck is enabled. {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"39336"}, "reason": "Enabled"} 2026-02-22T19:20:18.854542495Z DEBUG events [remediation] Node matches unhealthy condition. Node "ip-10-0-89-20.us-west-1.compute.internal", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"39336"}, "reason": "DetectedUnhealthy"} 2026-02-22T19:20:18.861751017Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:20:18Z" level=info msg="invalidating lease" time="2026-02-22T19:20:18Z" level=info msg="getting lease" 2026-02-22T19:20:19.063380918Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:20:19Z" level=info msg="invalidating lease" time="2026-02-22T19:20:19Z" level=info msg="getting lease" 2026-02-22T19:20:19.067281348Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:20:19.073328577Z INFO controllers.NodeHealthCheck.resource manager Attempting to obtain Node Lease {"Node name": "ip-10-0-89-20.us-west-1.compute.internal"} time="2026-02-22T19:20:19Z" level=info msg="request lease" time="2026-02-22T19:20:19Z" level=info msg="getting lease" time="2026-02-22T19:20:19Z" level=info msg="create lease" 2026-02-22T19:20:19.078655063Z INFO controllers.NodeHealthCheck.resource manager Creating a remediation CR {"CR name": "", "CR kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} 2026-02-22T19:20:19.090616765Z DEBUG events [remediation] Created remediation object for node ip-10-0-89-20.us-west-1.compute.internal {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"39336"}, "reason": "RemediationCreated"} 2026-02-22T19:20:19.102570927Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"ip-10-0-89-20.us-west-1.compute.internal","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","uid":"57b3e2b2-f7cf-409d-8ede-d0e54fd0f999","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2026-02-22T19:20:19Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-22T19:20:18Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2026-02-22T19:20:03Z"}, "patch": "{\"status\":{\"conditions\":[{\"lastTransitionTime\":\"2026-02-22T19:20:18Z\",\"message\":\"No issues found, NodeHealthCheck is enabled.\",\"reason\":\"NodeHealthCheckEnabled\",\"status\":\"False\",\"type\":\"Disabled\"}],\"healthyNodes\":2,\"observedNodes\":3,\"phase\":\"Remediating\",\"reason\":\"NHC is remediating 1 nodes\",\"unhealthyNodes\":[{\"name\":\"ip-10-0-89-20.us-west-1.compute.internal\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"ip-10-0-89-20.us-west-1.compute.internal-7m8dl\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"57b3e2b2-f7cf-409d-8ede-d0e54fd0f999\"},\"started\":\"2026-02-22T19:20:19Z\",\"templateName\":\"selfnoderemediationtemplate-sample\"}]}]}}"} 2026-02-22T19:20:19.300693676Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:20:19.300772699Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:20:19.300817841Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:20:19.300858333Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:20:19.307814404Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:20:19.307839866Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:20:19.307851396Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:20:19.307859707Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:20:19.31399606Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2026-02-22T19:20:19.314094814Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:20:19.32721879Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "ip-10-0-89-20.us-west-1.compute.internal", "condition type": "Ready", "condition status": "Unknown"} 2026-02-22T19:20:19.338201208Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:20:19.338319193Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:20:19.338381256Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:20:19.338425938Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:20:19.438249077Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-22T19:20:19Z"} 2026-02-22T19:20:19.438293129Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:20:19Z" level=info msg="invalidating lease" time="2026-02-22T19:20:19Z" level=info msg="getting lease" 2026-02-22T19:20:19.539355546Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:20:19Z" level=info msg="invalidating lease" time="2026-02-22T19:20:19Z" level=info msg="getting lease" 2026-02-22T19:20:19.54313418Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:20:19.54984898Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2026-02-22T19:20:19Z" level=info msg="getting lease" 2026-02-22T19:20:19.549912483Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-worker-test", "lease name": "node-ip-10-0-89-20.us-west-1.compute.internal", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2026-02-22T19:20:19.553202095Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2026-02-22T19:20:19.553270228Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:20:19.559789819Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "ip-10-0-89-20.us-west-1.compute.internal", "condition type": "Ready", "condition status": "Unknown"} 2026-02-22T19:20:19.567278415Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-22T19:20:19Z"} 2026-02-22T19:20:19.567297676Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:20:19Z" level=info msg="invalidating lease" time="2026-02-22T19:20:19Z" level=info msg="getting lease" 2026-02-22T19:20:19.570600488Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:20:19Z" level=info msg="invalidating lease" time="2026-02-22T19:20:19Z" level=info msg="getting lease" 2026-02-22T19:20:19.573817237Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:20:19.579900228Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2026-02-22T19:20:19Z" level=info msg="getting lease" 2026-02-22T19:20:19.579974131Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-worker-test", "lease name": "node-ip-10-0-89-20.us-west-1.compute.internal", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2026-02-22T19:20:19.58384114Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2026-02-22T19:21:19.202424285Z INFO adding NHC to reconcile queue for handling node {"node": "ip-10-0-89-20.us-west-1.compute.internal", "NHC": "nhc-worker-test"} 2026-02-22T19:21:19.202704458Z INFO adding NHC to reconcile queue for handling node {"node": "ip-10-0-89-20.us-west-1.compute.internal", "NHC": "nhc-worker-test"} 2026-02-22T19:21:19.202823603Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:21:19.25432968Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-22T19:20:19Z"} 2026-02-22T19:21:19.254362822Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:21:19Z" level=info msg="invalidating lease" time="2026-02-22T19:21:19Z" level=info msg="getting lease" 2026-02-22T19:21:19.355119302Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:21:19Z" level=info msg="invalidating lease" time="2026-02-22T19:21:19Z" level=info msg="getting lease" 2026-02-22T19:21:19.358930508Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:21:19.362152776Z INFO controllers.NodeHealthCheck.resource manager setting a delay for node getting healthy {"node name": "ip-10-0-89-20.us-west-1.compute.internal", "delay in seconds": 5} 2026-02-22T19:21:19.372029752Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"ip-10-0-89-20.us-west-1.compute.internal","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"ip-10-0-89-20.us-west-1.compute.internal-7m8dl","uid":"57b3e2b2-f7cf-409d-8ede-d0e54fd0f999","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2026-02-22T19:20:19Z","templateName":"selfnoderemediationtemplate-sample"}],"conditionsHealthyTimestamp":"2026-02-22T19:21:19Z","healthyDelayed":true}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-22T19:20:18Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2026-02-22T19:20:19Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"conditionsHealthyTimestamp\":\"2026-02-22T19:21:19Z\",\"healthyDelayed\":true,\"name\":\"ip-10-0-89-20.us-west-1.compute.internal\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"ip-10-0-89-20.us-west-1.compute.internal-7m8dl\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"57b3e2b2-f7cf-409d-8ede-d0e54fd0f999\"},\"started\":\"2026-02-22T19:20:19Z\",\"templateName\":\"selfnoderemediationtemplate-sample\"}]}]}}"} 2026-02-22T19:21:19.372043813Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:21:19.372274484Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:21:19.372292164Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:21:19.372305005Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:21:19.580941633Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "6s"} 2026-02-22T19:21:19.58108546Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:21:19.616377639Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-22T19:20:19Z"} 2026-02-22T19:21:19.616419821Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:21:19Z" level=info msg="invalidating lease" time="2026-02-22T19:21:19Z" level=info msg="getting lease" 2026-02-22T19:21:19.717495735Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:21:19.720941094Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "ip-10-0-89-20.us-west-1.compute.internal", "remaining time in seconds": 4.279060686} 2026-02-22T19:21:19.720964005Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:21:19Z" level=info msg="invalidating lease" time="2026-02-22T19:21:19Z" level=info msg="getting lease" 2026-02-22T19:21:19.72454202Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "5.279060686s"} 2026-02-22T19:21:25.00436403Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:21:25.019328911Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-22T19:20:19Z"} 2026-02-22T19:21:25.019351402Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:21:25Z" level=info msg="invalidating lease" time="2026-02-22T19:21:25Z" level=info msg="getting lease" 2026-02-22T19:21:25.022422304Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:21:25Z" level=info msg="invalidating lease" time="2026-02-22T19:21:25Z" level=info msg="getting lease" 2026-02-22T19:21:25.025506576Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:21:25.028799398Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:21:25.040503828Z INFO controllers.NodeHealthCheck.resource manager deleted remediation CR {"name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl"} 2026-02-22T19:21:25.04076196Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "11s"} 2026-02-22T19:21:25.040745459Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:21:25.040784951Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:21:25.040795792Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:21:25.040802652Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:21:25.040807652Z DEBUG events [remediation] Deleted remediation CR of kind SelfNodeRemediation with name ip-10-0-89-20.us-west-1.compute.internal-7m8dl {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"39745"}, "reason": "RemediationRemoved"} 2026-02-22T19:21:25.040836793Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:21:25.120686128Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:21:25.123915737Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:21:25.127183668Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:21:25Z" level=info msg="invalidating lease" time="2026-02-22T19:21:25Z" level=info msg="getting lease" 2026-02-22T19:21:25.13069948Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:21:25Z" level=info msg="invalidating lease" time="2026-02-22T19:21:25Z" level=info msg="getting lease" 2026-02-22T19:21:25.13416837Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "17s"} 2026-02-22T19:21:36.041510919Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:21:36.05622943Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:21:36Z" level=info msg="invalidating lease" time="2026-02-22T19:21:36Z" level=info msg="getting lease" 2026-02-22T19:21:36.059591115Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:21:36Z" level=info msg="invalidating lease" time="2026-02-22T19:21:36Z" level=info msg="getting lease" 2026-02-22T19:21:36.063098317Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:21:36.066300845Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:21:36.069888341Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "17s"} 2026-02-22T19:21:53.070609244Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:21:53.085165096Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:21:53.088433627Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:21:53.092141378Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:21:53Z" level=info msg="invalidating lease" time="2026-02-22T19:21:53Z" level=info msg="getting lease" 2026-02-22T19:21:53.095298864Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:21:53Z" level=info msg="invalidating lease" time="2026-02-22T19:21:53Z" level=info msg="getting lease" 2026-02-22T19:21:53.09978466Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "17s"} 2026-02-22T19:22:09.623300831Z INFO adding NHC to reconcile queue for handling node {"node": "ip-10-0-89-20.us-west-1.compute.internal", "NHC": "nhc-worker-test"} 2026-02-22T19:22:09.623519412Z INFO adding NHC to reconcile queue for handling node {"node": "ip-10-0-89-20.us-west-1.compute.internal", "NHC": "nhc-worker-test"} 2026-02-22T19:22:09.623626966Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:22:09.653646573Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "ip-10-0-89-20.us-west-1.compute.internal", "condition type": "Ready", "condition status": "Unknown", "duration left": "29.346355757s"} 2026-02-22T19:22:09.667043182Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:22:09Z" level=info msg="invalidating lease" time="2026-02-22T19:22:09Z" level=info msg="getting lease" 2026-02-22T19:22:09.672152898Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:22:09Z" level=info msg="invalidating lease" time="2026-02-22T19:22:09Z" level=info msg="getting lease" 2026-02-22T19:22:09.67521038Z INFO controllers.NodeHealthCheck Ignoring node, because it was unhealthy, and is likely to be unhealthy again. {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:22:09.675458541Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "30.346355757s"} 2026-02-22T19:22:10.101003278Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:22:10.108797018Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "ip-10-0-89-20.us-west-1.compute.internal", "condition type": "Ready", "condition status": "Unknown", "duration left": "28.891204792s"} 2026-02-22T19:22:10.116158738Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:22:10Z" level=info msg="invalidating lease" time="2026-02-22T19:22:10Z" level=info msg="getting lease" 2026-02-22T19:22:10.119388298Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:22:10Z" level=info msg="invalidating lease" time="2026-02-22T19:22:10Z" level=info msg="getting lease" 2026-02-22T19:22:10.122591606Z INFO controllers.NodeHealthCheck Ignoring node, because it was unhealthy, and is likely to be unhealthy again. {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:22:10.122807346Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "29.891204792s"} 2026-02-22T19:22:20.015750386Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:22:20.015794859Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:22:20.015816509Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:22:20.01582694Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:22:20.015874682Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:22:20.023257923Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "ip-10-0-89-20.us-west-1.compute.internal", "condition type": "Ready", "condition status": "Unknown", "duration left": "18.976743667s"} 2026-02-22T19:22:20.062847503Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:22:20Z" level=info msg="invalidating lease" time="2026-02-22T19:22:20Z" level=info msg="getting lease" 2026-02-22T19:22:20.162885226Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:22:20Z" level=info msg="invalidating lease" time="2026-02-22T19:22:20Z" level=info msg="getting lease" 2026-02-22T19:22:20.263577509Z INFO controllers.NodeHealthCheck Ignoring node, because it was unhealthy, and is likely to be unhealthy again. {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:22:20.26381719Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "19.976743667s"} 2026-02-22T19:22:30.185346058Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:22:30.185421242Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:22:30.185439103Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:22:30.185447553Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:22:30.185494125Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:22:30.192116641Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "ip-10-0-89-20.us-west-1.compute.internal", "condition type": "Ready", "condition status": "Unknown", "duration left": "8.807884729s"} 2026-02-22T19:22:30.202446029Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:22:30Z" level=info msg="invalidating lease" time="2026-02-22T19:22:30Z" level=info msg="getting lease" 2026-02-22T19:22:30.219616702Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} 2026-02-22T19:22:30.219753548Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:22:30.21978785Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-7m8dl", "Remediation CR Kind": "SelfNodeRemediation"} time="2026-02-22T19:22:30Z" level=info msg="invalidating lease" time="2026-02-22T19:22:30Z" level=info msg="getting lease" 2026-02-22T19:22:30.224399003Z INFO controllers.NodeHealthCheck Ignoring node, because it was unhealthy, and is likely to be unhealthy again. {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:22:30.224615393Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "9.807884729s"} 2026-02-22T19:22:30.224682456Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:22:30.233012201Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "ip-10-0-89-20.us-west-1.compute.internal", "condition type": "Ready", "condition status": "Unknown", "duration left": "8.766989279s"} 2026-02-22T19:22:30.302261351Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:22:30Z" level=info msg="invalidating lease" time="2026-02-22T19:22:30Z" level=info msg="getting lease" 2026-02-22T19:22:30.305430688Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:22:30Z" level=info msg="invalidating lease" time="2026-02-22T19:22:30Z" level=info msg="getting lease" 2026-02-22T19:22:30.308435587Z INFO controllers.NodeHealthCheck Ignoring node, because it was unhealthy, and is likely to be unhealthy again. {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:22:30.308668207Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "9.766989279s"} 2026-02-22T19:22:40.014186779Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:22:40.021333269Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "ip-10-0-89-20.us-west-1.compute.internal", "condition type": "Ready", "condition status": "Unknown"} 2026-02-22T19:22:40.028451298Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:22:40Z" level=info msg="invalidating lease" time="2026-02-22T19:22:40Z" level=info msg="getting lease" 2026-02-22T19:22:40.031558011Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:22:40Z" level=info msg="invalidating lease" time="2026-02-22T19:22:40Z" level=info msg="getting lease" 2026-02-22T19:22:40.034574631Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:22:40.040824929Z INFO controllers.NodeHealthCheck.resource manager Attempting to obtain Node Lease {"Node name": "ip-10-0-89-20.us-west-1.compute.internal"} time="2026-02-22T19:22:40Z" level=info msg="request lease" time="2026-02-22T19:22:40Z" level=info msg="getting lease" time="2026-02-22T19:22:40Z" level=info msg="renew lease owned by NodeHealthCheck-nhc-worker-test setAcquireTime=false" 2026-02-22T19:22:40.045654332Z INFO controllers.NodeHealthCheck.resource manager Creating a remediation CR {"CR name": "", "CR kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} 2026-02-22T19:22:40.052339701Z DEBUG events [remediation] Created remediation object for node ip-10-0-89-20.us-west-1.compute.internal {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"39745"}, "reason": "RemediationCreated"} 2026-02-22T19:22:40.056024622Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2026-02-22T19:22:40.061570428Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:22:40.061595029Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:22:40.06161415Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:22:40.06162258Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:22:40.061660502Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:22:40.068268027Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "ip-10-0-89-20.us-west-1.compute.internal", "condition type": "Ready", "condition status": "Unknown"} 2026-02-22T19:22:40.075742042Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:22:40.075769094Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:22:40.075782254Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"3fdcae60-37bc-4e6b-8fcb-89e0b917aa8e","controller":false}} 2026-02-22T19:22:40.075790735Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-22T19:22:40.129616421Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-22T19:22:40Z"} 2026-02-22T19:22:40.129651403Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:22:40Z" level=info msg="invalidating lease" time="2026-02-22T19:22:40Z" level=info msg="getting lease" 2026-02-22T19:22:40.133306832Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:22:40Z" level=info msg="invalidating lease" time="2026-02-22T19:22:40Z" level=info msg="getting lease" 2026-02-22T19:22:40.138440209Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:22:40.230663579Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2026-02-22T19:22:40Z" level=info msg="getting lease" 2026-02-22T19:22:40.230860218Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-worker-test", "lease name": "node-ip-10-0-89-20.us-west-1.compute.internal", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2026-02-22T19:22:40.234865413Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2026-02-22T19:22:40.234936576Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-22T19:22:40.241532481Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "ip-10-0-89-20.us-west-1.compute.internal", "condition type": "Ready", "condition status": "Unknown"} 2026-02-22T19:22:40.248161407Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-22T19:22:40Z"} 2026-02-22T19:22:40.248186599Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-55-237.us-west-1.compute.internal"} time="2026-02-22T19:22:40Z" level=info msg="invalidating lease" time="2026-02-22T19:22:40Z" level=info msg="getting lease" 2026-02-22T19:22:40.251625057Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-67-241.us-west-1.compute.internal"} time="2026-02-22T19:22:40Z" level=info msg="invalidating lease" time="2026-02-22T19:22:40Z" level=info msg="getting lease" 2026-02-22T19:22:40.254865637Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "ip-10-0-89-20.us-west-1.compute.internal"} 2026-02-22T19:22:40.261581577Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "ip-10-0-89-20.us-west-1.compute.internal-784gf", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2026-02-22T19:22:40Z" level=info msg="getting lease" 2026-02-22T19:22:40.261631269Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-worker-test", "lease name": "node-ip-10-0-89-20.us-west-1.compute.internal", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2026-02-22T19:22:40.265499478Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"}