[root@cert-rhosp-02 ~]# oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.20.0-0.nightly-2025-11-06-033058 True False 16h Cluster version is 4.20.0-0.nightly-2025-11-06-033058 [root@cert-rhosp-02 ~]# oc get csv NAME DISPLAY VERSION REPLACES PHASE machine-deletion-remediation.v0.5.0 Machine Deletion Remediation operator 0.5.0 machine-deletion-remediation.v0.4.1 Succeeded node-healthcheck-operator.v0.10.1 Node Health Check Operator 0.10.1 node-healthcheck-operator.v0.10.0 Succeeded node-maintenance-operator.v5.5.0 Node Maintenance Operator 5.5.0 node-maintenance-operator.v5.4.1 Succeeded self-node-remediation.v0.11.0 Self Node Remediation Operator 0.11.0 self-node-remediation.v0.10.1 Succeeded [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep node-healthcheck-controller-manager) [root@cert-rhosp-02 ~]# echo $PODS pod/node-healthcheck-controller-manager-5f7fd5c797-gkwp4 pod/node-healthcheck-controller-manager-5f7fd5c797-zkcjs [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/node-healthcheck-controller-manager-5f7fd5c797-gkwp4 "master-0-0" == pod/node-healthcheck-controller-manager-5f7fd5c797-zkcjs "master-0-2" [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep self-node-remediation-controller-manager) [root@cert-rhosp-02 ~]# echo $PODS pod/self-node-remediation-controller-manager-6db4b9dfd7-fr4jd pod/self-node-remediation-controller-manager-6db4b9dfd7-mxn4b [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/self-node-remediation-controller-manager-6db4b9dfd7-fr4jd "worker-0-1" == pod/self-node-remediation-controller-manager-6db4b9dfd7-mxn4b "worker-0-0" [root@cert-rhosp-02 ~]# oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 5d22h v1.33.5 worker-0-1 Ready worker 5d22h v1.33.5 worker-0-2 Ready worker 5d22h v1.33.5 [root@cert-rhosp-02 ~]# oc debug node/worker-0-2 -- chroot /host bash -c "systemctl stop kubelet" Temporary namespace openshift-debug-sq7wb is created for debugging node... Starting pod/worker-0-2-debug-wl7rx ... To use host binaries, run `chroot /host` [root@cert-rhosp-02 ~]# oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 5d22h v1.33.5 worker-0-1 Ready worker 5d22h v1.33.5 worker-0-2 NotReady worker 5d22h v1.33.5 [root@cert-rhosp-02 ~]# cat test.yaml --- apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate metadata: namespace: openshift-workload-availability name: selfnoderemediationtemplate-sample spec: template: spec: remediationStrategy: OutOfServiceTaint --- apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: name: nhc-worker-test spec: maxUnhealthy: 2 healthyDelay: 90s remediationTemplate: apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate name: selfnoderemediationtemplate-sample namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/worker operator: Exists unhealthyConditions: - duration: 30s status: 'False' type: Ready - duration: 30s status: Unknown type: Ready [root@cert-rhosp-02 ~]# oc apply -f test.yaml selfnoderemediationtemplate.self-node-remediation.medik8s.io/selfnoderemediationtemplate-sample created nodehealthcheck.remediation.medik8s.io/nhc-worker-test created [root@cert-rhosp-02 ~]# oc get SelfNodeRemediationConfig -o yaml apiVersion: v1 items: - apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationConfig metadata: creationTimestamp: "2025-11-07T10:15:51Z" generation: 2 name: self-node-remediation-config namespace: openshift-workload-availability resourceVersion: "3167087" uid: 598e4b09-ba9d-46d9-98df-3b58f27641a1 spec: apiCheckInterval: 15s apiServerTimeout: 5s hostPort: 30001 isSoftwareRebootEnabled: true maxApiErrorThreshold: 3 minPeersForRemediation: 1 peerApiServerTimeout: 5s peerDialTimeout: 5s peerRequestTimeout: 7s peerUpdateInterval: 15m safeTimeToAssumeNodeRebootedSeconds: 60 watchdogFilePath: /dev/watchdog kind: List metadata: resourceVersion: "" [root@cert-rhosp-02 ~]# cat test.yaml apiVersion: v1 kind: Pod metadata: name: stuck-terminating-pod namespace: default finalizers: - kubernetes.io/pvc-protection # This blocks pod deletion - test.example.com/block-deletion # Custom finalizer spec: nodeName: worker-0-2 # CHANGE THIS to your test node name terminationGracePeriodSeconds: 300 # Also adds delay containers: - name: nginx image: nginx:latest resources: requests: memory: "64Mi" cpu: "250m" limits: memory: "128Mi" cpu: "500m" [root@cert-rhosp-02 ~]# oc apply -f test.yaml pod/stuck-terminating-pod created SNR Logs: 2025-11-12T08:38:47.882002403Z INFO setup Go Version: go1.23.4 (Red Hat 1.23.4-1.el9) X:strictfipsruntime 2025-11-12T08:38:47.882083498Z INFO setup Go OS/Arch: linux/amd64 2025-11-12T08:38:47.882086953Z INFO setup Operator Version: 0bfa3d8 2025-11-12T08:38:47.882089792Z INFO setup Git Commit: 0bfa3d8cc4f3191c831d36b702aed4b32b7ef625 2025-11-12T08:38:47.882092086Z INFO setup Build Date: 2025-09-29T06:25:34+00:00 2025-11-12T08:38:47.882093646Z INFO setup HTTP/2 for metrics and webhook server disabled 2025-11-12T08:38:47.882570901Z INFO controller-runtime.metrics Metrics server is starting to listen {"addr": "127.0.0.1:8080"} 2025-11-12T08:38:47.89258413Z INFO utils-taints out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 33} 2025-11-12T08:38:47.892620747Z INFO utils-taints out of service taint strategy {"isGA": true, "k8sMajorVersion": 1, "k8sMinorVersion": 33} 2025-11-12T08:38:47.892629367Z INFO setup Starting as a manager that installs the daemonset 2025-11-12T08:38:47.892643441Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig"} 2025-11-12T08:38:47.892698045Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2025-11-12T08:38:47.892805161Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2025-11-12T08:38:47.892870381Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-11-12T08:38:47.892929674Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-11-12T08:38:47.892964925Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-11-12T08:38:47.89301189Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-11-12T08:38:47.893064254Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation"} 2025-11-12T08:38:47.893102716Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2025-11-12T08:38:47.893156153Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2025-11-12T08:38:47.893245861Z INFO setup starting manager 2025-11-12T08:38:47.893484992Z INFO controller-runtime.webhook.webhooks Starting webhook server 2025-11-12T08:38:47.893542054Z INFO Starting server {"kind": "health probe", "addr": "[::]:8081"} 2025-11-12T08:38:47.89360165Z INFO starting server {"path": "/metrics", "kind": "metrics", "addr": "127.0.0.1:8080"} I1112 08:38:47.893712 1 leaderelection.go:245] attempting to acquire leader lease openshift-workload-availability/547f6cb6.medik8s.io... 2025-11-12T08:38:47.893781369Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-11-12T08:38:47.89392581Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-11-12T08:38:47.894013048Z INFO controller-runtime.certwatcher Starting certificate watcher 2025-11-12T08:39:11.739089697Z INFO selfnoderemediationtemplate-resource default {"name": "self-node-remediation-automatic-strategy-template"} 2025-11-12T08:39:11.7425061Z INFO selfnoderemediationtemplate-resource validate create {"name": "self-node-remediation-automatic-strategy-template"} 2025-11-12T16:40:22.413703828Z INFO selfnoderemediationtemplate-resource default {"name": "selfnoderemediationtemplate-sample"} 2025-11-12T16:40:22.416616652Z INFO selfnoderemediationtemplate-resource validate create {"name": "selfnoderemediationtemplate-sample"} 2025-11-12T16:40:22.766875468Z INFO selfnoderemediation-resource validate update {"name": "worker-0-2-srls6"} 2025-11-12T16:43:40.062397854Z INFO selfnoderemediation-resource validate update {"name": "worker-0-2-srls6"} 2025-11-12T17:15:38.241876258Z INFO selfnoderemediationconfig-resource validate create {"name": "self-node-remediation-config"} 2025-11-12T17:15:48.240721159Z INFO selfnoderemediationtemplate-resource default {"name": "self-node-remediation-automatic-strategy-template"} 2025-11-12T17:15:48.243564033Z INFO selfnoderemediationtemplate-resource validate create {"name": "self-node-remediation-automatic-strategy-template"} I1112 17:20:52.155195 1 leaderelection.go:255] successfully acquired lease openshift-workload-availability/547f6cb6.medik8s.io 2025-11-12T17:20:52.155252232Z DEBUG events self-node-remediation-controller-manager-6db4b9dfd7-fr4jd_5e9ac71d-11da-461b-a8b9-a4ca47f94775 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"547f6cb6.medik8s.io","uid":"e7f9cc92-58aa-4ec2-89f2-4045febcdce5","apiVersion":"coordination.k8s.io/v1","resourceVersion":"3091256"}, "reason": "LeaderElection"} 2025-11-12T17:20:52.155411964Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1alpha1.SelfNodeRemediationConfig"} 2025-11-12T17:20:52.155454491Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1.DaemonSet"} 2025-11-12T17:20:52.155486068Z INFO Starting Controller {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig"} 2025-11-12T17:20:52.155574577Z INFO Starting EventSource {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "source": "kind source: *v1alpha1.SelfNodeRemediation"} 2025-11-12T17:20:52.155607616Z INFO Starting Controller {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation"} 2025-11-12T17:20:52.161094242Z INFO selfnoderemediationconfig-resource validate create {"name": "self-node-remediation-config"} 2025-11-12T17:20:52.258376429Z INFO Starting workers {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "worker count": 1} 2025-11-12T17:20:52.25845498Z INFO Starting workers {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "worker count": 1} 2025-11-12T17:20:52.258574312Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-11-12T17:20:52.360376945Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-11-12T17:20:52.360412713Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-11-12T17:20:52.361396034Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/11/12 17:20:52 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/11/12 17:20:52 update was successful 2025-11-12T17:20:52.369587583Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-11-12T17:20:52.369614367Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-11-12T17:20:52.369617414Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-11-12T17:20:52.37022612Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/11/12 17:20:52 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/11/12 17:20:52 update was successful 2025-11-12T17:21:02.159391744Z INFO selfnoderemediationtemplate-resource default {"name": "self-node-remediation-automatic-strategy-template"} 2025-11-12T17:21:02.162146019Z INFO selfnoderemediationtemplate-resource validate create {"name": "self-node-remediation-automatic-strategy-template"} 2025-11-12T17:21:11.450062846Z INFO selfnoderemediation-resource validate create {"name": "worker-0-2-l62gs"} 2025-11-12T17:21:11.554098382Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}} 2025-11-12T17:21:11.554151426Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-l62gs","uid":"7148ad77-43f7-489a-859d-59929bfa6989","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3091393"}, "reason": "RemediationStarted"} 2025-11-12T17:21:11.659413865Z INFO selfnoderemediation-resource validate update {"name": "worker-0-2-l62gs"} 2025-11-12T17:21:11.662202886Z INFO controllers.SelfNodeRemediation finalizer added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}} 2025-11-12T17:21:11.662316493Z DEBUG events [remediation] Remediation process - successful adding finalizer {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-l62gs","uid":"7148ad77-43f7-489a-859d-59929bfa6989","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3091397"}, "reason": "AddFinalizer"} 2025-11-12T17:21:11.667225982Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}} 2025-11-12T17:21:11.679027157Z INFO controllers.SelfNodeRemediation NoExecute taint added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-12T17:20:40Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-12T17:20:46Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-12T17:21:11Z"}]} 2025-11-12T17:21:11.679087344Z INFO controllers.SelfNodeRemediation Marking node as unschedulable {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T17:21:11.679194577Z DEBUG events [remediation] Remediation process - NoExecute taint added to the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3091400"}, "reason": "AddNoExecute"} 2025-11-12T17:21:11.690926402Z DEBUG events [remediation] Remediation process - unhealthy node marked as unschedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3091405"}, "reason": "MarkUnschedulable"} 2025-11-12T17:21:11.703674391Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}} 2025-11-12T17:21:11.704394753Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T17:21:12.70431946Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}} 2025-11-12T17:21:12.704745431Z INFO rebootDurationCalculator No SafeTimeToAssumeNodeRebootedSeconds specified, using calculated minimum safe reboot time {"calculated minimum time in seconds": 120} 2025-11-12T17:21:12.704875745Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}, "node name": "worker-0-2", "time": "2025-11-12 17:23:12.704873201 +0000 UTC m=+31464.843194435"} 2025-11-12T17:21:12.70510456Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-l62gs","uid":"7148ad77-43f7-489a-859d-59929bfa6989","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3091416"}, "reason": "UpdateTimeAssumedRebooted"} 2025-11-12T17:21:12.710100773Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}, "node name": "worker-0-2", "time left": "2m0.289900737s"} 2025-11-12T17:23:13.004810404Z INFO controllers.SelfNodeRemediation TimeAssumedRebooted is old. The unhealthy node assumed to been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T17:23:13.01880605Z INFO controllers.SelfNodeRemediation out-of-service taint added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-12T17:20:40Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-12T17:20:46Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-12T17:21:11Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-11-12T17:21:11Z"},{"key":"node.kubernetes.io/out-of-service","value":"nodeshutdown","effect":"NoExecute","timeAdded":"2025-11-12T17:23:13Z"}]} 2025-11-12T17:23:13.019102604Z DEBUG events [remediation] Remediation process - add out-of-service taint to unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3092317"}, "reason": "AddOutOfService"} 2025-11-12T17:23:13.025150098Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}, "pod name": "dns-default-76lbj", "phase": "Running"} 2025-11-12T17:23:18.153689747Z INFO controllers.SelfNodeRemediation out-of-service taint removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-12T17:20:40Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-12T17:20:46Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-12T17:21:11Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-11-12T17:21:11Z"}]} 2025-11-12T17:23:18.153844746Z DEBUG events [remediation] Remediation process - remove out-of-service taint from node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3092386"}, "reason": "RemoveOutOfService"} 2025-11-12T17:23:18.15390235Z DEBUG events [remediation] Remediation process - finished deleting unhealthy node resources {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3092386"}, "reason": "DeleteResources"} 2025-11-12T20:10:48.201833329Z INFO selfnoderemediation-resource validate update {"name": "worker-0-2-l62gs"} 2025-11-12T20:12:19.021913833Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}} 2025-11-12T20:12:19.028785889Z DEBUG events [remediation] Remediation process - mark healthy remediated node as schedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3157550"}, "reason": "MarkNodeSchedulable"} 2025-11-12T20:12:20.041004808Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}} 2025-11-12T20:12:20.04882019Z INFO controllers.SelfNodeRemediation NoExecute taint removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}, "new taints": null} 2025-11-12T20:12:20.049134146Z DEBUG events [remediation] Remediation process - remove NoExecute taint from healthy remediated node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3157561"}, "reason": "RemoveNoExecuteTaint"} 2025-11-12T20:12:20.064872385Z INFO controllers.SelfNodeRemediation finalizer removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}} 2025-11-12T20:12:20.065079986Z DEBUG events [remediation] Remediation process - remove finalizer from snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-l62gs","uid":"7148ad77-43f7-489a-859d-59929bfa6989","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3157548"}, "reason": "RemoveFinalizer"} 2025-11-12T20:12:20.065106426Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-l62gs","uid":"7148ad77-43f7-489a-859d-59929bfa6989","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3157548"}, "reason": "RemediationFinished"} 2025-11-12T20:12:20.072647229Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}} 2025-11-12T20:12:21.07301212Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-l62gs","namespace":"openshift-workload-availability"}} 2025-11-12T20:36:03.890182454Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-11-12T20:36:03.890240706Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-11-12T20:36:03.890244123Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-11-12T20:36:03.891313373Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/11/12 20:36:03 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/11/12 20:36:03 update was successful 2025-11-12T20:41:59.453281015Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}} 2025-11-12T20:41:59.453318128Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-mcvqh","uid":"aaa73d63-45e9-4fa9-b4fa-f69fa50c0271","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3169461"}, "reason": "RemediationStarted"} 2025-11-12T20:41:59.459919027Z INFO controllers.SelfNodeRemediation finalizer added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}} 2025-11-12T20:41:59.460065678Z DEBUG events [remediation] Remediation process - successful adding finalizer {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-mcvqh","uid":"aaa73d63-45e9-4fa9-b4fa-f69fa50c0271","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3169464"}, "reason": "AddFinalizer"} 2025-11-12T20:41:59.464031319Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}} 2025-11-12T20:41:59.47160717Z INFO controllers.SelfNodeRemediation NoExecute taint added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-12T20:41:28Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-12T20:41:33Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-12T20:41:59Z"}]} 2025-11-12T20:41:59.47165489Z INFO controllers.SelfNodeRemediation Marking node as unschedulable {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T20:41:59.472294146Z DEBUG events [remediation] Remediation process - NoExecute taint added to the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3169469"}, "reason": "AddNoExecute"} 2025-11-12T20:41:59.480971891Z DEBUG events [remediation] Remediation process - unhealthy node marked as unschedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3169471"}, "reason": "MarkUnschedulable"} 2025-11-12T20:41:59.489879608Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}} 2025-11-12T20:41:59.490232117Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T20:42:00.49089779Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}} 2025-11-12T20:42:00.491416584Z INFO rebootDurationCalculator Warning: Ignoring specified SafeTimeToAssumeNodeRebootedSeconds because it's lower than the calculated minimum safe reboot time {"specified time in seconds": 2, "calculated minimum time in seconds": 120} 2025-11-12T20:42:00.4914823Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "node name": "worker-0-2", "time": "2025-11-12 20:44:00.491481044 +0000 UTC m=+43512.629802251"} 2025-11-12T20:42:00.49171176Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-mcvqh","uid":"aaa73d63-45e9-4fa9-b4fa-f69fa50c0271","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3169475"}, "reason": "UpdateTimeAssumedRebooted"} 2025-11-12T20:42:00.497006151Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "node name": "worker-0-2", "time left": "2m0.502997533s"} 2025-11-12T20:44:01.006175159Z INFO controllers.SelfNodeRemediation TimeAssumedRebooted is old. The unhealthy node assumed to been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T20:44:01.022073883Z INFO controllers.SelfNodeRemediation out-of-service taint added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-12T20:41:28Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-12T20:41:33Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-12T20:41:59Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-11-12T20:41:59Z"},{"key":"node.kubernetes.io/out-of-service","value":"nodeshutdown","effect":"NoExecute","timeAdded":"2025-11-12T20:44:01Z"}]} 2025-11-12T20:44:01.022810253Z DEBUG events [remediation] Remediation process - add out-of-service taint to unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3170226"}, "reason": "AddOutOfService"} 2025-11-12T20:44:01.028634013Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "pod name": "insights-runtime-extractor-pj48q", "phase": "Running"} 2025-11-12T20:44:06.041229719Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "pod name": "ingress-canary-lpscz", "phase": "Running"} 2025-11-12T20:44:11.052380762Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "pod name": "dns-default-sx2m6", "phase": "Running"} 2025-11-12T20:44:16.069083402Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "pod name": "ingress-canary-lpscz", "phase": "Running"} 2025-11-12T20:44:21.087340587Z INFO controllers.SelfNodeRemediation out-of-service taint removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-12T20:41:28Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-12T20:41:33Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-12T20:41:59Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-11-12T20:41:59Z"}]} 2025-11-12T20:44:21.087394765Z DEBUG events [remediation] Remediation process - remove out-of-service taint from node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3170385"}, "reason": "RemoveOutOfService"} 2025-11-12T20:44:21.087495897Z DEBUG events [remediation] Remediation process - finished deleting unhealthy node resources {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3170385"}, "reason": "DeleteResources"} 2025-11-12T20:46:27.682515671Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-11-12T20:46:27.682593902Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-11-12T20:46:27.682598248Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-11-12T20:46:27.683273126Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/11/12 20:46:27 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/11/12 20:46:27 update was successful 2025-11-12T20:46:57.628633921Z INFO selfnoderemediation-resource validate update {"name": "worker-0-2-mcvqh"} 2025-11-12T20:48:28.021185739Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}} 2025-11-12T20:48:28.027916657Z DEBUG events [remediation] Remediation process - mark healthy remediated node as schedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3172236"}, "reason": "MarkNodeSchedulable"} 2025-11-12T20:48:29.035203811Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}} 2025-11-12T20:48:29.045158991Z INFO controllers.SelfNodeRemediation NoExecute taint removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}, "new taints": null} 2025-11-12T20:48:29.045425892Z DEBUG events [remediation] Remediation process - remove NoExecute taint from healthy remediated node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3172246"}, "reason": "RemoveNoExecuteTaint"} 2025-11-12T20:48:29.059170461Z INFO controllers.SelfNodeRemediation finalizer removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}} 2025-11-12T20:48:29.059388357Z DEBUG events [remediation] Remediation process - remove finalizer from snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-mcvqh","uid":"aaa73d63-45e9-4fa9-b4fa-f69fa50c0271","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3172234"}, "reason": "RemoveFinalizer"} 2025-11-12T20:48:29.059419736Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-mcvqh","uid":"aaa73d63-45e9-4fa9-b4fa-f69fa50c0271","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3172234"}, "reason": "RemediationFinished"} 2025-11-12T20:48:29.06175508Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}} 2025-11-12T20:48:30.062287499Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-mcvqh","namespace":"openshift-workload-availability"}} 2025-11-12T20:50:19.548847093Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}} 2025-11-12T20:50:19.548898061Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-9tmzg","uid":"f551b6e7-4d06-42b7-bc3f-f32c1f2eed00","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3173125"}, "reason": "RemediationStarted"} 2025-11-12T20:50:19.554861546Z INFO controllers.SelfNodeRemediation finalizer added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}} 2025-11-12T20:50:19.554993275Z DEBUG events [remediation] Remediation process - successful adding finalizer {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-9tmzg","uid":"f551b6e7-4d06-42b7-bc3f-f32c1f2eed00","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3173128"}, "reason": "AddFinalizer"} 2025-11-12T20:50:19.559371407Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}} 2025-11-12T20:50:19.568344458Z INFO controllers.SelfNodeRemediation NoExecute taint added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-12T20:49:48Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-12T20:49:53Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-12T20:50:19Z"}]} 2025-11-12T20:50:19.568384883Z INFO controllers.SelfNodeRemediation Marking node as unschedulable {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T20:50:19.568506841Z DEBUG events [remediation] Remediation process - NoExecute taint added to the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3173132"}, "reason": "AddNoExecute"} 2025-11-12T20:50:19.575799573Z DEBUG events [remediation] Remediation process - unhealthy node marked as unschedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3173134"}, "reason": "MarkUnschedulable"} 2025-11-12T20:50:19.583121276Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}} 2025-11-12T20:50:19.583430392Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T20:50:19.590740154Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}} 2025-11-12T20:50:19.591042582Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T20:50:20.583318558Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}} 2025-11-12T20:50:20.583620387Z INFO rebootDurationCalculator Warning: Ignoring specified SafeTimeToAssumeNodeRebootedSeconds because it's lower than the calculated minimum safe reboot time {"specified time in seconds": 60, "calculated minimum time in seconds": 120} 2025-11-12T20:50:20.583633591Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "node name": "worker-0-2", "time": "2025-11-12 20:52:20.583633033 +0000 UTC m=+44012.721954232"} 2025-11-12T20:50:20.583737697Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-9tmzg","uid":"f551b6e7-4d06-42b7-bc3f-f32c1f2eed00","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3173143"}, "reason": "UpdateTimeAssumedRebooted"} 2025-11-12T20:50:20.588192032Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "node name": "worker-0-2", "time left": "2m0.411810166s"} 2025-11-12T20:52:21.005600625Z INFO controllers.SelfNodeRemediation TimeAssumedRebooted is old. The unhealthy node assumed to been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T20:52:21.019327442Z INFO controllers.SelfNodeRemediation out-of-service taint added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-12T20:49:48Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-12T20:49:53Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-12T20:50:19Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-11-12T20:50:19Z"},{"key":"node.kubernetes.io/out-of-service","value":"nodeshutdown","effect":"NoExecute","timeAdded":"2025-11-12T20:52:21Z"}]} 2025-11-12T20:52:21.019402442Z DEBUG events [remediation] Remediation process - add out-of-service taint to unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3173915"}, "reason": "AddOutOfService"} 2025-11-12T20:52:21.025174059Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "pod name": "ingress-canary-hd6m8", "phase": "Running"} 2025-11-12T20:52:26.036677066Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "pod name": "dns-default-6cbfg", "phase": "Running"} 2025-11-12T20:52:31.047758907Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "pod name": "ingress-canary-hd6m8", "phase": "Running"} 2025-11-12T20:52:36.063550314Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "pod name": "ingress-canary-hd6m8", "phase": "Running"} 2025-11-12T20:52:41.080715401Z INFO controllers.SelfNodeRemediation out-of-service taint removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-12T20:49:48Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-12T20:49:53Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-12T20:50:19Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-11-12T20:50:19Z"}]} 2025-11-12T20:52:41.080871077Z DEBUG events [remediation] Remediation process - remove out-of-service taint from node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3174065"}, "reason": "RemoveOutOfService"} 2025-11-12T20:52:41.080947727Z DEBUG events [remediation] Remediation process - finished deleting unhealthy node resources {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3174065"}, "reason": "DeleteResources"} 2025-11-12T20:55:26.371963307Z INFO selfnoderemediation-resource validate update {"name": "worker-0-2-9tmzg"} 2025-11-12T20:56:57.022767537Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}} 2025-11-12T20:56:57.029422701Z DEBUG events [remediation] Remediation process - mark healthy remediated node as schedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3175964"}, "reason": "MarkNodeSchedulable"} 2025-11-12T20:56:58.036519023Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}} 2025-11-12T20:56:58.043616877Z INFO controllers.SelfNodeRemediation NoExecute taint removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}, "new taints": null} 2025-11-12T20:56:58.043751598Z DEBUG events [remediation] Remediation process - remove NoExecute taint from healthy remediated node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3175969"}, "reason": "RemoveNoExecuteTaint"} 2025-11-12T20:56:58.052332286Z INFO selfnoderemediation-resource validate update {"name": "worker-0-2-9tmzg"} 2025-11-12T20:56:58.062011608Z INFO controllers.SelfNodeRemediation finalizer removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}} 2025-11-12T20:56:58.062157872Z DEBUG events [remediation] Remediation process - remove finalizer from snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-9tmzg","uid":"f551b6e7-4d06-42b7-bc3f-f32c1f2eed00","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3175962"}, "reason": "RemoveFinalizer"} 2025-11-12T20:56:58.062188534Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-9tmzg","uid":"f551b6e7-4d06-42b7-bc3f-f32c1f2eed00","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3175962"}, "reason": "RemediationFinished"} 2025-11-12T20:56:58.066803203Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}} 2025-11-12T20:56:59.066944627Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-9tmzg","namespace":"openshift-workload-availability"}} 2025-11-12T21:01:24.445879286Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}} 2025-11-12T21:01:24.445913452Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-rzwjq","uid":"dd528e06-c0e1-4cdd-8c1a-4ae22ea994bd","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3177876"}, "reason": "RemediationStarted"} 2025-11-12T21:01:24.45132742Z INFO controllers.SelfNodeRemediation finalizer added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}} 2025-11-12T21:01:24.451461204Z DEBUG events [remediation] Remediation process - successful adding finalizer {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-rzwjq","uid":"dd528e06-c0e1-4cdd-8c1a-4ae22ea994bd","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3177878"}, "reason": "AddFinalizer"} 2025-11-12T21:01:24.45557711Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}} 2025-11-12T21:01:24.463381607Z INFO controllers.SelfNodeRemediation NoExecute taint added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-12T21:00:53Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-12T21:00:59Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-12T21:01:24Z"}]} 2025-11-12T21:01:24.463426524Z INFO controllers.SelfNodeRemediation Marking node as unschedulable {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T21:01:24.463446854Z DEBUG events [remediation] Remediation process - NoExecute taint added to the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3177883"}, "reason": "AddNoExecute"} 2025-11-12T21:01:24.471241133Z DEBUG events [remediation] Remediation process - unhealthy node marked as unschedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3177886"}, "reason": "MarkUnschedulable"} 2025-11-12T21:01:24.478444872Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}} 2025-11-12T21:01:24.478731026Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T21:01:24.482822036Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}} 2025-11-12T21:01:24.48302495Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T21:01:25.479324295Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}} 2025-11-12T21:01:25.479632516Z INFO rebootDurationCalculator Warning: Ignoring specified SafeTimeToAssumeNodeRebootedSeconds because it's lower than the calculated minimum safe reboot time {"specified time in seconds": 60, "calculated minimum time in seconds": 120} 2025-11-12T21:01:25.479647628Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "node name": "worker-0-2", "time": "2025-11-12 21:03:25.479646978 +0000 UTC m=+44677.617968169"} 2025-11-12T21:01:25.480026075Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-rzwjq","uid":"dd528e06-c0e1-4cdd-8c1a-4ae22ea994bd","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3177889"}, "reason": "UpdateTimeAssumedRebooted"} 2025-11-12T21:01:25.484378244Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "node name": "worker-0-2", "time left": "2m0.515622867s"} 2025-11-12T21:03:26.005349939Z INFO controllers.SelfNodeRemediation TimeAssumedRebooted is old. The unhealthy node assumed to been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "node name": "worker-0-2"} 2025-11-12T21:03:26.01907911Z INFO controllers.SelfNodeRemediation out-of-service taint added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-12T21:00:53Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-12T21:00:59Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-12T21:01:24Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-11-12T21:01:24Z"},{"key":"node.kubernetes.io/out-of-service","value":"nodeshutdown","effect":"NoExecute","timeAdded":"2025-11-12T21:03:26Z"}]} 2025-11-12T21:03:26.019239063Z DEBUG events [remediation] Remediation process - add out-of-service taint to unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"2c78ae59-d650-43cd-a7c3-51f7ad6479e7","apiVersion":"v1","resourceVersion":"3178673"}, "reason": "AddOutOfService"} 2025-11-12T21:03:26.026141314Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Running"} 2025-11-12T21:03:31.038773573Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "ingress-canary-62879", "phase": "Running"} 2025-11-12T21:03:36.063254028Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "ingress-canary-62879", "phase": "Running"} 2025-11-12T21:03:41.07336058Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:03:46.082866771Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:03:51.095176156Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:03:56.10532739Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:01.117396762Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:06.129304793Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:11.139464449Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:16.153774517Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:21.1640172Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:26.174663419Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:26.184503914Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "b1a2fbaf-f0f8-4682-aa51-66adbb182531", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:04:26.189511207Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:26.193825513Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "f923924a-9eff-4714-ab53-014e13509d14", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:04:26.202606932Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:26.20732317Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "aa238d33-8426-4edd-a6e3-26b0d9dc555a", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:04:26.211659249Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:26.216157528Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "8f108f70-09d7-4b0a-8beb-37c5638b7a49", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:04:26.231271521Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:26.235554356Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "9ba8e191-000e-4895-988b-e6819d590858", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:04:26.319937318Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:26.325175441Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "befff08c-1e85-4848-9f1f-8ef481452be2", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:04:26.494504614Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:26.499787639Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "2dde32b6-8d52-4cf6-8d0b-46485a11b2f2", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:04:26.824856201Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:26.829736507Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "401e84db-90b5-4b4a-a026-14acd93769ad", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:04:27.474956232Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:27.479535692Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "59704494-f3ed-421b-b6a5-96b00e888ce5", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:04:28.765831846Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:28.771868213Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "aaedf84d-4d5f-4eac-bf0a-94aca07967c0", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:04:31.340690442Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:31.346337533Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "4242446a-6bdc-4a38-a637-ede69bb80099", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:04:36.471305345Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:36.475916749Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "125352f9-7d02-47b8-aa12-26d6053db789", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:04:46.721123793Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:04:46.725884912Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "94251b4d-1691-451e-9dc9-2f956b649604", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:05:07.215397974Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:05:07.220627089Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "09563420-8f76-4c8e-a823-7789a3606516", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:05:48.185567714Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:05:48.190274822Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "c1497dab-9500-4492-9d94-3b7517da7708", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:07:10.115128438Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:07:10.120657901Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "9e167314-7457-445c-9c25-07f8a4ed12ab", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:09:53.965490957Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:09:53.970630066Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "9e9388cd-3d05-4990-b769-7272baa56c1d", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:15:21.655588004Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:15:21.660636122Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "5367a255-6cdd-4aad-8cf2-dc4735a66f43", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:26:17.025336127Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "pod name": "stuck-terminating-pod", "phase": "Failed"} 2025-11-12T21:26:17.030118006Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-2-rzwjq", "reconcileID": "91799513-d867-4946-9023-24f2d0816fea", "error": "Not ready to delete out-of-service taint", "errorVerbose": "Not ready to delete out-of-service taint\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).useOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:447\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).handleRebootCompletedPhase\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:536\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithResourceRemoval\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:470\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).remediateWithOutOfServiceTaint\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:423\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:314\ngithub.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile\n\t/app/self-node-remediation/controllers/selfnoderemediation_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 2025-11-12T21:31:24.904868174Z INFO controllers.SelfNodeRemediation NHC added the timed-out annotation, remediation will be stopped {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}} 2025-11-12T21:31:24.905201276Z DEBUG events [remediation] NHC added the timed-out annotation, remediation will be stopped {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-rzwjq","uid":"dd528e06-c0e1-4cdd-8c1a-4ae22ea994bd","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3189532"}, "reason": "RemediationStopped"} 2025-11-12T21:31:24.910242443Z INFO controllers.SelfNodeRemediation NHC added the timed-out annotation, remediation will be stopped {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}} 2025-11-12T21:31:24.910462333Z DEBUG events [remediation] NHC added the timed-out annotation, remediation will be stopped {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-rzwjq","uid":"dd528e06-c0e1-4cdd-8c1a-4ae22ea994bd","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3189534"}, "reason": "RemediationStopped"} 2025-11-12T21:42:57.030544076Z INFO controllers.SelfNodeRemediation NHC added the timed-out annotation, remediation will be stopped {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-rzwjq","namespace":"openshift-workload-availability"}} 2025-11-12T21:42:57.03079676Z DEBUG events [remediation] NHC added the timed-out annotation, remediation will be stopped {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-rzwjq","uid":"dd528e06-c0e1-4cdd-8c1a-4ae22ea994bd","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3189534"}, "reason": "RemediationStopped"}