==================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc project openshift-workload-availability Already on project "openshift-workload-availability" on server "https://api.ocp-edge-cluster-0.qe.lab.redhat.com:6443". ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.16.0-0.nightly-2025-09-06-014223 True False 33h Cluster version is 4.16.0-0.nightly-2025-09-06-014223 ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get csv NAME DISPLAY VERSION REPLACES PHASE node-healthcheck-operator.v0.10.0 Node Health Check Operator 0.10.0 node-healthcheck-operator.v0.9.1 Succeeded self-node-remediation.v0.10.0 Self Node Remediation Operator 0.10.0 self-node-remediation.v0.9.0 Succeeded ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ PODS=$(oc get pods -o name | grep self-node) ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ echo $PODS pod/self-node-remediation-controller-manager-5889775bc6-2zx5j pod/self-node-remediation-ds-58dxp pod/self-node-remediation-ds-5l6q9 pod/self-node-remediation-ds-5m69c pod/self-node-remediation-ds-gg7lg pod/self-node-remediation-ds-m54lq pod/self-node-remediation-ds-qqdhn ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ for p in $PODS; do > echo "== $p" > oc get "$p" -o json | jq .spec.nodeName > done == pod/self-node-remediation-controller-manager-5889775bc6-2zx5j "worker-0-1" == pod/self-node-remediation-ds-58dxp "master-0-0" == pod/self-node-remediation-ds-5l6q9 "worker-0-0" == pod/self-node-remediation-ds-5m69c "worker-0-2" == pod/self-node-remediation-ds-gg7lg "master-0-1" == pod/self-node-remediation-ds-m54lq "worker-0-1" == pod/self-node-remediation-ds-qqdhn "master-0-2" ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get mhc -n openshift-machine-api -o yaml apiVersion: v1 items: - apiVersion: machine.openshift.io/v1beta1 kind: MachineHealthCheck metadata: annotations: capability.openshift.io/name: MachineAPI exclude.release.openshift.io/internal-openshift-hosted: "true" include.release.openshift.io/self-managed-high-availability: "true" creationTimestamp: "2025-09-07T20:38:00Z" generation: 1 labels: api: clusterapi k8s-app: termination-handler name: machine-api-termination-handler namespace: openshift-machine-api ownerReferences: - apiVersion: config.openshift.io/v1 controller: true kind: ClusterVersion name: version uid: 2fc8ea8b-4708-4bd3-acc1-900c89d1f124 resourceVersion: "26155" uid: cfb2ede7-e534-42b6-801d-1a33bcdc1a4d spec: maxUnhealthy: 100% nodeStartupTimeout: "0" selector: matchLabels: machine.openshift.io/interruptible-instance: "" unhealthyConditions: - status: "True" timeout: 0s type: Terminating status: conditions: - lastTransitionTime: "2025-09-07T20:38:00Z" status: "True" type: RemediationAllowed currentHealthy: 0 expectedMachines: 0 kind: List metadata: resourceVersion: "" ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 34h v1.29.14+c68a663 worker-0-1 Ready worker 34h v1.29.14+c68a663 worker-0-2 Ready worker 34h v1.29.14+c68a663 ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc debug node/worker-0-2 -- chroot /host bash -c "uptime -s" Temporary namespace openshift-debug-plfwq is created for debugging node... Starting pod/worker-0-2-debug-5mpmq ... To use host binaries, run `chroot /host` 2025-09-09 05:12:24 Removing debug pod ... Temporary namespace openshift-debug-plfwq was removed. ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc debug node/worker-0-2 -- chroot /host bash -c "systemctl stop kubelet" Temporary namespace openshift-debug-4c2jl is created for debugging node... ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ cat snr_mhc.yaml apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate metadata: namespace: openshift-machine-api name: selfnoderemediationtemplate-sample spec: template: spec: remediationStrategy: Automatic --- apiVersion: machine.openshift.io/v1beta1 kind: MachineHealthCheck metadata: name: mhc-snr-worker namespace: openshift-machine-api spec: selector: matchLabels: machine.openshift.io/cluster-api-machine-role: "worker" machine.openshift.io/cluster-api-machine-type: "worker" remediationTemplate: kind: SelfNodeRemediationTemplate apiVersion: self-node-remediation.medik8s.io/v1alpha1 name: selfnoderemediationtemplate-sample namespace: openshift-machine-api unhealthyConditions: - type: Ready status: "False" timeout: 90s - type: Ready status: "Unknown" timeout: 90s - type: MemoryPressure status: "True" timeout: 90s - type: DiskPressure status: "True" timeout: 90s maxUnhealthy: 100% nodeStartupTimeout: 10m ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc apply -f snr_mhc.yaml selfnoderemediationtemplate.self-node-remediation.medik8s.io/selfnoderemediationtemplate-sample created machinehealthcheck.machine.openshift.io/mhc-snr-worker created ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get snr ocp-edge-cluster-0-zjrl5-worker-0-jknsl -n openshift-machine-api -o yaml apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediation metadata: annotations: machine.openshift.io/cloned-from-groupkind: SelfNodeRemediationTemplate.self-node-remediation.medik8s.io machine.openshift.io/cloned-from-name: selfnoderemediationtemplate-sample creationTimestamp: "2025-09-09T07:20:35Z" finalizers: - self-node-remediation.medik8s.io/snr-finalizer generation: 1 name: ocp-edge-cluster-0-zjrl5-worker-0-jknsl namespace: openshift-machine-api ownerReferences: - apiVersion: machine.openshift.io/v1beta1 kind: Machine name: ocp-edge-cluster-0-zjrl5-worker-0-jknsl uid: 80e30b6f-3bff-40a4-b562-45cfca846815 resourceVersion: "862865" uid: 77394b2c-ef40-4e16-b5ef-112e07ae01fe spec: remediationStrategy: Automatic status: conditions: - lastTransitionTime: "2025-09-09T07:20:35Z" message: "" reason: RemediationStarted status: "True" type: Processing - lastTransitionTime: "2025-09-09T07:20:35Z" message: "" reason: RemediationStarted status: Unknown type: Succeeded phase: Pre-Reboot-Completed timeAssumedRebooted: "2025-09-09T07:22:36Z" ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 34h v1.29.14+c68a663 worker-0-1 Ready worker 34h v1.29.14+c68a663 worker-0-2 Ready worker 34h v1.29.14+c68a663 ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc debug node/worker-0-2 -- chroot /host bash -c "uptime -s" Temporary namespace openshift-debug-khvhl is created for debugging node... Starting pod/worker-0-2-debug-cm47w ... To use host binaries, run `chroot /host` 2025-09-09 07:21:12 Removing debug pod ... Temporary namespace openshift-debug-khvhl was removed. ==================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc logs pod/self-node-remediation-controller-manager-5889775bc6-2zx5j -n openshift-workload-availability Defaulted container "manager" out of: manager, kube-rbac-proxy 2025-09-08T12:40:39.330647863Z INFO setup Go Version: go1.23.2 (Red Hat 1.23.2-1.el9) X:strictfipsruntime 2025-09-08T12:40:39.330724491Z INFO setup Go OS/Arch: linux/amd64 2025-09-08T12:40:39.330727884Z INFO setup Operator Version: v0.10.0 2025-09-08T12:40:39.330729773Z INFO setup Git Commit: 2025-09-08T12:40:39.330731941Z INFO setup Build Date: 2025-01-13T11:55:12+00:00 2025-09-08T12:40:39.330733575Z INFO setup HTTP/2 for metrics and webhook server disabled 2025-09-08T12:40:39.331145562Z INFO controller-runtime.metrics Metrics server is starting to listen {"addr": "127.0.0.1:8080"} 2025-09-08T12:40:39.33586487Z INFO utils-taints out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 29} 2025-09-08T12:40:39.335912547Z INFO utils-taints out of service taint strategy {"isGA": true, "k8sMajorVersion": 1, "k8sMinorVersion": 29} 2025-09-08T12:40:39.335926595Z INFO setup Starting as a manager that installs the daemonset 2025-09-08T12:40:39.335943357Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig"} 2025-09-08T12:40:39.336004336Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2025-09-08T12:40:39.336098275Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2025-09-08T12:40:39.336156259Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-09-08T12:40:39.33620293Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-09-08T12:40:39.336232Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-09-08T12:40:39.33626736Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-09-08T12:40:39.336295722Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation"} 2025-09-08T12:40:39.336328259Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2025-09-08T12:40:39.336378068Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2025-09-08T12:40:39.336446166Z INFO setup starting manager 2025-09-08T12:40:39.336609862Z INFO controller-runtime.webhook.webhooks Starting webhook server 2025-09-08T12:40:39.336634169Z INFO Starting server {"kind": "health probe", "addr": "[::]:8081"} 2025-09-08T12:40:39.336728656Z INFO starting server {"path": "/metrics", "kind": "metrics", "addr": "127.0.0.1:8080"} I0908 12:40:39.336787 1 leaderelection.go:245] attempting to acquire leader lease openshift-workload-availability/547f6cb6.medik8s.io... 2025-09-08T12:40:39.336822032Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-09-08T12:40:39.336888962Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-09-08T12:40:39.33713535Z INFO controller-runtime.certwatcher Starting certificate watcher I0908 12:40:55.753695 1 leaderelection.go:255] successfully acquired lease openshift-workload-availability/547f6cb6.medik8s.io 2025-09-08T12:40:55.753755562Z DEBUG events self-node-remediation-controller-manager-5889775bc6-2zx5j_ba469cc2-53ff-4026-8c48-e0bfc6772b8b became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"547f6cb6.medik8s.io","uid":"c4647c9c-de7d-4622-b0bc-575a5774f3f4","apiVersion":"coordination.k8s.io/v1","resourceVersion":"397246"}, "reason": "LeaderElection"} 2025-09-08T12:40:55.754008105Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1alpha1.SelfNodeRemediationConfig"} 2025-09-08T12:40:55.754032273Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1.DaemonSet"} 2025-09-08T12:40:55.754035701Z INFO Starting Controller {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig"} 2025-09-08T12:40:55.754106859Z INFO Starting EventSource {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "source": "kind source: *v1alpha1.SelfNodeRemediation"} 2025-09-08T12:40:55.75414764Z INFO Starting Controller {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation"} 2025-09-08T12:40:55.761041835Z INFO selfnoderemediationconfig-resource validate create {"name": "self-node-remediation-config"} 2025-09-08T12:40:55.856885334Z INFO Starting workers {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "worker count": 1} 2025-09-08T12:40:55.856983265Z INFO Starting workers {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "worker count": 1} 2025-09-08T12:40:55.857135818Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-09-08T12:40:55.958940847Z INFO controllers.SelfNodeRemediationConfig Creating new certs 2025-09-08T12:40:58.52707615Z INFO controllers.SelfNodeRemediationConfig Storing certs in new secret 2025-09-08T12:40:58.533053054Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-09-08T12:40:58.533768277Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025-09-08T12:40:58.533804959Z INFO controllers.SelfNodeRemediationConfig snr didn't find old daemonset to be deleted 2025/09/08 12:40:58 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/09/08 12:40:58 does not exist, creating (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/09/08 12:40:58 successfully created (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025-09-08T12:40:58.571909722Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-09-08T12:40:58.571939528Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-09-08T12:40:58.571944966Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-09-08T12:40:58.57256258Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/09/08 12:40:58 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/09/08 12:40:58 update was successful 2025-09-08T12:41:05.758592907Z INFO selfnoderemediationtemplate-resource default {"name": "self-node-remediation-automatic-strategy-template"} 2025-09-08T12:41:05.761851351Z INFO selfnoderemediationtemplate-resource validate create {"name": "self-node-remediation-automatic-strategy-template"} 2025-09-08T14:14:59.588726008Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-09-08T14:14:59.588779939Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-09-08T14:14:59.58878348Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-09-08T14:14:59.589436439Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/09/08 14:14:59 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/09/08 14:14:59 update was successful 2025-09-08T15:05:22.491258822Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-09-08T15:05:22.491318599Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-09-08T15:05:22.49132176Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-09-08T15:05:22.491939001Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/09/08 15:05:22 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/09/08 15:05:22 update was successful 2025-09-08T15:13:56.616411712Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-09-08T15:13:56.616467524Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-09-08T15:13:56.616471096Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-09-08T15:13:56.617071147Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/09/08 15:13:56 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/09/08 15:13:56 update was successful 2025-09-08T18:29:50.561596941Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-09-08T18:29:50.561653998Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-09-08T18:29:50.561660115Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-09-08T18:29:50.562305053Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/09/08 18:29:50 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/09/08 18:29:50 update was successful 2025-09-08T19:55:38.833410883Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-09-08T19:55:38.833483505Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-09-08T19:55:38.833487626Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-09-08T19:55:38.834197894Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/09/08 19:55:38 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/09/08 19:55:38 update was successful 2025-09-09T05:04:35.879621614Z INFO selfnoderemediationtemplate-resource default {"name": "selfnoderemediationtemplate-sample"} 2025-09-09T05:04:35.882651556Z INFO selfnoderemediationtemplate-resource validate create {"name": "selfnoderemediationtemplate-sample"} 2025-09-09T05:11:48.250296752Z INFO selfnoderemediation-resource validate create {"name": "worker-0-2-d5fsx"} 2025-09-09T05:11:48.355732871Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:11:48.355770117Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:11:48.355785387Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"worker-0-2-d5fsx","uid":"4a9c3a81-2cbb-4dbf-9f79-b599d254eca4","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"806815"}, "reason": "RemediationStarted"} 2025-09-09T05:11:48.4602024Z INFO selfnoderemediation-resource validate update {"name": "worker-0-2-d5fsx"} 2025-09-09T05:11:48.464372717Z INFO controllers.SelfNodeRemediation finalizer added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:11:48.464518464Z DEBUG events [remediation] Remediation process - successful adding finalizer {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"worker-0-2-d5fsx","uid":"4a9c3a81-2cbb-4dbf-9f79-b599d254eca4","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"806819"}, "reason": "AddFinalizer"} 2025-09-09T05:11:48.469815438Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:11:48.46983789Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:11:48.478623489Z INFO controllers.SelfNodeRemediation NoExecute taint added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-09-09T05:11:17Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-09-09T05:11:23Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-09-09T05:11:48Z"}]} 2025-09-09T05:11:48.478692394Z INFO controllers.SelfNodeRemediation Marking node as unschedulable {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "node name": "worker-0-2"} 2025-09-09T05:11:48.478840318Z DEBUG events [remediation] Remediation process - NoExecute taint added to the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"806823"}, "reason": "AddNoExecute"} 2025-09-09T05:11:48.499519553Z DEBUG events [remediation] Remediation process - unhealthy node marked as unschedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"806826"}, "reason": "MarkUnschedulable"} 2025-09-09T05:11:48.510216255Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:11:48.510266721Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:11:48.510672706Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "node name": "worker-0-2"} 2025-09-09T05:11:48.525241501Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:11:48.52528025Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:11:48.525831129Z INFO rebootDurationCalculator No SafeTimeToAssumeNodeRebootedSeconds specified, using calculated minimum safe reboot time {"calculated minimum time in seconds": 120} 2025-09-09T05:11:48.525860749Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "node name": "worker-0-2", "time": "2025-09-09 05:13:48.525859432 +0000 UTC m=+59589.207321801"} 2025-09-09T05:11:48.526004811Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"worker-0-2-d5fsx","uid":"4a9c3a81-2cbb-4dbf-9f79-b599d254eca4","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"806845"}, "reason": "UpdateTimeAssumedRebooted"} 2025-09-09T05:11:48.536388409Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:11:48.536417498Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "node name": "worker-0-2", "time left": "2m0.463584577s"} 2025-09-09T05:11:49.510099991Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:11:49.510118473Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "node name": "worker-0-2", "time left": "1m59.489882399s"} 2025-09-09T05:13:37.65178843Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:13:37.651810976Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "node name": "worker-0-2", "time left": "11.348189976s"} 2025-09-09T05:13:49.008952125Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:13:49.008982Z INFO controllers.SelfNodeRemediation TimeAssumedRebooted is old. The unhealthy node assumed to been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "node name": "worker-0-2"} 2025-09-09T05:13:49.015346332Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:13:49.023564456Z INFO controllers.SelfNodeRemediation out-of-service taint added{"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "new taints": [{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-09-09T05:11:48Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-09-09T05:11:48Z"},{"key":"node.kubernetes.io/out-of-service","value":"nodeshutdown","effect":"NoExecute","timeAdded":"2025-09-09T05:13:49Z"}]} 2025-09-09T05:13:49.023640315Z DEBUG events [remediation] Remediation process - add out-of-service taint to unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"808557"}, "reason": "AddOutOfService"} 2025-09-09T05:13:49.030043503Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "pod name": "kube-state-metrics-6678955fdd-btpcr", "phase": "Running"} 2025-09-09T05:13:54.037403232Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:13:54.047036221Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "pod name": "self-node-remediation-ds-5m4l2", "phase": "Running"} 2025-09-09T05:13:59.052519234Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:13:59.056200656Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "pod name": "self-node-remediation-ds-5m4l2", "phase": "Running"} 2025-09-09T05:14:04.062178708Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:14:04.065818113Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "pod name": "self-node-remediation-ds-5m4l2", "phase": "Running"} 2025-09-09T05:14:09.072016757Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:14:09.075810134Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "pod name": "self-node-remediation-ds-5m4l2", "phase": "Running"} 2025-09-09T05:14:14.081147199Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:14:14.091363065Z INFO controllers.SelfNodeRemediation waiting for terminating pod {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "pod name": "self-node-remediation-ds-5m4l2", "phase": "Running"} 2025-09-09T05:14:19.099012544Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:14:19.21250506Z INFO controllers.SelfNodeRemediation out-of-service taint removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "new taints": [{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-09-09T05:11:48Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-09-09T05:11:48Z"}]} 2025-09-09T05:14:19.212623392Z DEBUG events [remediation] Remediation process - remove out-of-service taint from node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"809158"}, "reason": "RemoveOutOfService"} 2025-09-09T05:14:19.212653634Z DEBUG events [remediation] Remediation process - finished deleting unhealthy node resources {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"809158"}, "reason": "DeleteResources"} 2025-09-09T05:14:19.220858597Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:14:19.220875369Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:14:19.230424838Z DEBUG events [remediation] Remediation process - mark healthy remediated node as schedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"809163"}, "reason": "MarkNodeSchedulable"} 2025-09-09T05:14:20.237840764Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:14:20.23785788Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:14:20.244688586Z INFO controllers.SelfNodeRemediation NoExecute taint removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}, "new taints": null} 2025-09-09T05:14:20.244937402Z DEBUG events [remediation] Remediation process - remove NoExecute taint from healthy remediated node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"809181"}, "reason": "RemoveNoExecuteTaint"} 2025-09-09T05:14:20.252179556Z INFO selfnoderemediation-resource validate update {"name": "worker-0-2-d5fsx"} 2025-09-09T05:14:20.255738461Z INFO controllers.SelfNodeRemediation finalizer removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:14:20.255917192Z DEBUG events [remediation] Remediation process - remove finalizer from snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"worker-0-2-d5fsx","uid":"4a9c3a81-2cbb-4dbf-9f79-b599d254eca4","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"809160"}, "reason": "RemoveFinalizer"} 2025-09-09T05:14:20.255930189Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"worker-0-2-d5fsx","uid":"4a9c3a81-2cbb-4dbf-9f79-b599d254eca4","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"809160"}, "reason": "RemediationFinished"} 2025-09-09T05:14:20.258067206Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T05:14:21.258774099Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-2-d5fsx","namespace":"openshift-machine-api"}} 2025-09-09T06:42:07.233007133Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-09-09T06:42:07.233125476Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-09-09T06:42:07.233129522Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-09-09T06:42:07.233778152Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/09/09 06:42:07 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/09/09 06:42:07 update was successful 2025-09-09T07:14:18.533910403Z INFO selfnoderemediationtemplate-resource default {"name": "selfnoderemediationtemplate-sample"} 2025-09-09T07:14:18.537128958Z INFO selfnoderemediationtemplate-resource validate create {"name": "selfnoderemediationtemplate-sample"} 2025-09-09T07:14:18.570071686Z INFO selfnoderemediation-resource validate create {"name": "ocp-edge-cluster-0-zjrl5-worker-0-jknsl"} 2025-09-09T07:14:18.676009843Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:14:18.676031079Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:14:18.676058485Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","uid":"8f6ea494-31ff-4b5d-8410-76a68f3ada78","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"859765"}, "reason": "RemediationStarted"} 2025-09-09T07:14:18.678894165Z INFO selfnoderemediation-resource validate update {"name": "ocp-edge-cluster-0-zjrl5-worker-0-jknsl"} 2025-09-09T07:14:18.681929407Z INFO controllers.SelfNodeRemediation finalizer added {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:14:18.682085588Z DEBUG events [remediation] Remediation process - successful adding finalizer {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","uid":"8f6ea494-31ff-4b5d-8410-76a68f3ada78","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"859766"}, "reason": "AddFinalizer"} 2025-09-09T07:14:18.686449583Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:14:18.686470029Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:14:18.694880976Z INFO controllers.SelfNodeRemediation NoExecute taint added {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-09-09T07:09:44Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-09-09T07:09:49Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-09-09T07:14:18Z"}]} 2025-09-09T07:14:18.694934302Z INFO controllers.SelfNodeRemediation Marking node as unschedulable {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2"} 2025-09-09T07:14:18.695058013Z DEBUG events [remediation] Remediation process - NoExecute taint added to the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"859770"}, "reason": "AddNoExecute"} 2025-09-09T07:14:18.703558328Z DEBUG events [remediation] Remediation process - unhealthy node marked as unschedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"859771"}, "reason": "MarkUnschedulable"} 2025-09-09T07:14:18.711475853Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:14:18.711495267Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:14:18.711691732Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2"} 2025-09-09T07:14:18.719438386Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:14:18.719461295Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:14:18.719751124Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2"} 2025-09-09T07:14:19.71233251Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:14:19.712351393Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:14:19.712616654Z INFO rebootDurationCalculator No SafeTimeToAssumeNodeRebootedSeconds specified, using calculated minimum safe reboot time {"calculated minimum time in seconds": 120} 2025-09-09T07:14:19.712628058Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2", "time": "2025-09-09 07:16:19.712627534 +0000 UTC m=+66940.394089896"} 2025-09-09T07:14:19.712741701Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","uid":"8f6ea494-31ff-4b5d-8410-76a68f3ada78","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"859776"}, "reason": "UpdateTimeAssumedRebooted"} 2025-09-09T07:14:19.718646231Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:14:19.71866711Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2", "time left": "2m0.281333955s"} 2025-09-09T07:16:09.590633147Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:16:09.590664147Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2", "time left": "10.409337088s"} 2025-09-09T07:16:20.004566682Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:16:20.004594416Z INFO controllers.SelfNodeRemediation TimeAssumedRebooted is old. The unhealthy node assumed to been rebooted {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2"} 2025-09-09T07:16:20.010483762Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:16:20.01768086Z INFO controllers.SelfNodeRemediation out-of-service taint added{"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "new taints": [{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-09-09T07:14:18Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-09-09T07:14:18Z"},{"key":"node.kubernetes.io/out-of-service","value":"nodeshutdown","effect":"NoExecute","timeAdded":"2025-09-09T07:16:20Z"}]} 2025-09-09T07:16:20.01775002Z DEBUG events [remediation] Remediation process - add out-of-service taint to unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"860711"}, "reason": "AddOutOfService"} 2025-09-09T07:16:20.032391486Z INFO controllers.SelfNodeRemediation out-of-service taint removed {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "new taints": [{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-09-09T07:14:18Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-09-09T07:14:18Z"}]} 2025-09-09T07:16:20.032646101Z DEBUG events [remediation] Remediation process - remove out-of-service taint from node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"860715"}, "reason": "RemoveOutOfService"} 2025-09-09T07:16:20.032700274Z DEBUG events [remediation] Remediation process - finished deleting unhealthy node resources {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"860715"}, "reason": "DeleteResources"} 2025-09-09T07:16:20.039216103Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:16:20.039241966Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:16:20.045748344Z DEBUG events [remediation] Remediation process - mark healthy remediated node as schedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"860722"}, "reason": "MarkNodeSchedulable"} 2025-09-09T07:16:21.054204025Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:16:21.054229318Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:16:21.060998345Z INFO controllers.SelfNodeRemediation NoExecute taint removed {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "new taints": null} 2025-09-09T07:16:21.061347693Z DEBUG events [remediation] Remediation process - remove NoExecute taint from healthy remediated node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"860731"}, "reason": "RemoveNoExecuteTaint"} 2025-09-09T07:16:21.069657971Z INFO selfnoderemediation-resource validate update {"name": "ocp-edge-cluster-0-zjrl5-worker-0-jknsl"} 2025-09-09T07:16:21.076289716Z INFO controllers.SelfNodeRemediation finalizer removed {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:16:21.076485917Z DEBUG events [remediation] Remediation process - remove finalizer from snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","uid":"8f6ea494-31ff-4b5d-8410-76a68f3ada78","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"860719"}, "reason": "RemoveFinalizer"} 2025-09-09T07:16:21.076533566Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","uid":"8f6ea494-31ff-4b5d-8410-76a68f3ada78","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"860719"}, "reason": "RemediationFinished"} 2025-09-09T07:16:21.082247526Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:16:22.083292909Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:35.023017508Z INFO selfnoderemediation-resource validate create {"name": "ocp-edge-cluster-0-zjrl5-worker-0-jknsl"} 2025-09-09T07:20:35.026898794Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:35.02691882Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:35.026998084Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","uid":"77394b2c-ef40-4e16-b5ef-112e07ae01fe","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"862825"}, "reason": "RemediationStarted"} 2025-09-09T07:20:35.030606666Z INFO selfnoderemediation-resource validate update {"name": "ocp-edge-cluster-0-zjrl5-worker-0-jknsl"} 2025-09-09T07:20:35.033127838Z INFO controllers.SelfNodeRemediation finalizer added {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:35.033380818Z DEBUG events [remediation] Remediation process - successful adding finalizer {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","uid":"77394b2c-ef40-4e16-b5ef-112e07ae01fe","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"862827"}, "reason": "AddFinalizer"} 2025-09-09T07:20:35.037586356Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:35.037596244Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:35.044545275Z INFO controllers.SelfNodeRemediation NoExecute taint added {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-09-09T07:19:04Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-09-09T07:19:10Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-09-09T07:20:35Z"}]} 2025-09-09T07:20:35.044628387Z INFO controllers.SelfNodeRemediation Marking node as unschedulable {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2"} 2025-09-09T07:20:35.045013749Z DEBUG events [remediation] Remediation process - NoExecute taint added to the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"862830"}, "reason": "AddNoExecute"} 2025-09-09T07:20:35.057287265Z DEBUG events [remediation] Remediation process - unhealthy node marked as unschedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"862834"}, "reason": "MarkUnschedulable"} 2025-09-09T07:20:35.063342217Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:35.063358662Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:35.06356698Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2"} 2025-09-09T07:20:35.071610191Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:35.071626017Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:35.071875334Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2"} 2025-09-09T07:20:36.063903278Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:36.063920599Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:36.064340798Z INFO rebootDurationCalculator No SafeTimeToAssumeNodeRebootedSeconds specified, using calculated minimum safe reboot time {"calculated minimum time in seconds": 120} 2025-09-09T07:20:36.064353528Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2", "time": "2025-09-09 07:22:36.064352616 +0000 UTC m=+67316.745814978"} 2025-09-09T07:20:36.064453391Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","uid":"77394b2c-ef40-4e16-b5ef-112e07ae01fe","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"862845"}, "reason": "UpdateTimeAssumedRebooted"} 2025-09-09T07:20:36.069489939Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:20:36.06950569Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2", "time left": "2m0.930495103s"} 2025-09-09T07:22:25.91836719Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:22:25.918387696Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2", "time left": "11.081613219s"} 2025-09-09T07:22:37.004907706Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:22:37.004937751Z INFO controllers.SelfNodeRemediation TimeAssumedRebooted is old. The unhealthy node assumed to been rebooted {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "node name": "worker-0-2"} 2025-09-09T07:22:37.010239672Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:22:37.017141834Z INFO controllers.SelfNodeRemediation out-of-service taint added{"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "new taints": [{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-09-09T07:20:35Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-09-09T07:20:35Z"},{"key":"node.kubernetes.io/out-of-service","value":"nodeshutdown","effect":"NoExecute","timeAdded":"2025-09-09T07:22:37Z"}]} 2025-09-09T07:22:37.017497395Z DEBUG events [remediation] Remediation process - add out-of-service taint to unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"863823"}, "reason": "AddOutOfService"} 2025-09-09T07:22:37.033064414Z INFO controllers.SelfNodeRemediation out-of-service taint removed {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "new taints": [{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-09-09T07:20:35Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-09-09T07:20:35Z"}]} 2025-09-09T07:22:37.033219922Z DEBUG events [remediation] Remediation process - remove out-of-service taint from node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"863828"}, "reason": "RemoveOutOfService"} 2025-09-09T07:22:37.033240444Z DEBUG events [remediation] Remediation process - finished deleting unhealthy node resources {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"863828"}, "reason": "DeleteResources"} 2025-09-09T07:22:37.040822104Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:22:37.040843745Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:22:37.047274638Z DEBUG events [remediation] Remediation process - mark healthy remediated node as schedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"863834"}, "reason": "MarkNodeSchedulable"} 2025-09-09T07:22:38.053680736Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:22:38.053702766Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:22:38.061326943Z INFO controllers.SelfNodeRemediation NoExecute taint removed {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}, "new taints": null} 2025-09-09T07:22:38.061582652Z DEBUG events [remediation] Remediation process - remove NoExecute taint from healthy remediated node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"863842"}, "reason": "RemoveNoExecuteTaint"} 2025-09-09T07:22:38.068831493Z INFO selfnoderemediation-resource validate update {"name": "ocp-edge-cluster-0-zjrl5-worker-0-jknsl"} 2025-09-09T07:22:38.074801497Z INFO controllers.SelfNodeRemediation finalizer removed {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:22:38.074924516Z DEBUG events [remediation] Remediation process - remove finalizer from snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","uid":"77394b2c-ef40-4e16-b5ef-112e07ae01fe","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"863831"}, "reason": "RemoveFinalizer"} 2025-09-09T07:22:38.074950246Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","uid":"77394b2c-ef40-4e16-b5ef-112e07ae01fe","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"863831"}, "reason": "RemediationFinished"} 2025-09-09T07:22:38.079067298Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}} 2025-09-09T07:22:39.080171227Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-zjrl5-worker-0-jknsl","namespace":"openshift-machine-api"}}