============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.20.0-0.nightly-2025-09-01-101753 True False 11h Cluster version is 4.20.0-0.nightly-2025-09-01-101753 ============================================================================================================================== [kni@cert-rhosp-02 ~]$ ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get csv NAME DISPLAY VERSION REPLACES PHASE node-healthcheck-operator.v0.10.0 Node Health Check Operator 0.10.0 node-healthcheck-operator.v0.9.1 Succeeded self-node-remediation.v0.10.0 Self Node Remediation Operator 0.10.0 self-node-remediation.v0.9.0 Succeeded ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get pods | grep self self-node-remediation-controller-manager-56894b59db-cxq47 2/2 Running 1 (4m19s ago) 4m34s self-node-remediation-ds-c7fzq 1/1 Running 1 (3m54s ago) 4m2s self-node-remediation-ds-jz6jr 1/1 Running 1 (3m54s ago) 4m2s self-node-remediation-ds-mslr2 1/1 Running 1 (3m54s ago) 4m2s self-node-remediation-ds-p47rm 1/1 Running 2 (3m57s ago) 4m2s self-node-remediation-ds-tj85g 1/1 Running 2 (3m54s ago) 4m2s self-node-remediation-ds-vpdgx 1/1 Running 1 (3m53s ago) 4m2s ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 11h v1.33.3 worker-0-1 Ready worker 11h v1.33.3 worker-0-2 Ready worker 11h v1.33.3 ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get mhc -n openshift-machine-api NAME MAXUNHEALTHY EXPECTEDMACHINES CURRENTHEALTHY machine-api-termination-handler 100% 0 0 mhc-snr-worker 100% 3 3 ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get mhc/mhc-snr-worker -n openshift-machine-api -o yaml apiVersion: machine.openshift.io/v1beta1 kind: MachineHealthCheck metadata: creationTimestamp: "2025-09-05T20:11:07Z" generation: 1 name: mhc-snr-worker namespace: openshift-machine-api resourceVersion: "260106" uid: b4a92404-c587-4c66-9580-4533edec9358 spec: maxUnhealthy: 100% nodeStartupTimeout: 10m remediationTemplate: apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate name: selfnoderemediationtemplate-sample namespace: openshift-machine-api selector: matchLabels: machine.openshift.io/cluster-api-machine-role: worker machine.openshift.io/cluster-api-machine-type: worker unhealthyConditions: - status: "False" timeout: 90s type: Ready - status: Unknown timeout: 90s type: Ready - status: "True" timeout: 90s type: MemoryPressure - status: "True" timeout: 90s type: DiskPressure status: conditions: - lastTransitionTime: "2025-09-05T20:11:07Z" status: "True" type: RemediationAllowed currentHealthy: 3 expectedMachines: 3 remediationsAllowed: 3 ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get snr -n openshift-machine-api No resources found in openshift-machine-api namespace. ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get snrt -n openshift-machine-api NAME AGE selfnoderemediationtemplate-sample 5m50s ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get snrt/selfnoderemediationtemplate-sample -n openshift-machine-api -o yaml apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate metadata: annotations: remediation.medik8s.io/multiple-templates-support: "true" creationTimestamp: "2025-09-05T20:11:07Z" generation: 1 name: selfnoderemediationtemplate-sample namespace: openshift-machine-api resourceVersion: "260102" uid: 13974bd1-8172-4815-abab-d39f52ff3434 spec: template: spec: remediationStrategy: Automatic ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get pods/self-node-remediation-controller-manager-56894b59db-cxq47 -o json | jq .spec.nodeName "worker-0-2" ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc debug node/worker-0-1 -- chroot /host bash -c "uptime -s" Starting pod/worker-0-1-debug-szqql ... To use host binaries, run `chroot /host` 2025-09-05 15:57:51 Removing debug pod ... ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 11h v1.33.3 worker-0-1 Ready worker 11h v1.33.3 worker-0-2 Ready worker 11h v1.33.3 ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 11h v1.33.3 worker-0-1 NotReady worker 11h v1.33.3 worker-0-2 Ready worker 11h v1.33.3 ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get snr/ocp-edge-cluster-0-pjlrl-worker-0-qp62r -n openshift-machine-api -o yaml apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediation metadata: annotations: machine.openshift.io/cloned-from-groupkind: SelfNodeRemediationTemplate.self-node-remediation.medik8s.io machine.openshift.io/cloned-from-name: selfnoderemediationtemplate-sample creationTimestamp: "2025-09-05T20:21:37Z" deletionGracePeriodSeconds: 0 deletionTimestamp: "2025-09-05T20:23:25Z" finalizers: - self-node-remediation.medik8s.io/snr-finalizer generation: 2 name: ocp-edge-cluster-0-pjlrl-worker-0-qp62r namespace: openshift-machine-api ownerReferences: - apiVersion: machine.openshift.io/v1beta1 kind: Machine name: ocp-edge-cluster-0-pjlrl-worker-0-qp62r uid: 6b438338-d019-4f37-903a-5a734b08c595 resourceVersion: "264706" uid: 3f8d0151-1846-4dc0-85bc-b976409dc43c spec: remediationStrategy: Automatic status: conditions: - lastTransitionTime: "2025-09-05T20:21:37Z" message: "" reason: RemediationStarted status: "True" type: Processing - lastTransitionTime: "2025-09-05T20:21:37Z" message: "" reason: RemediationStarted status: Unknown type: Succeeded phase: Pre-Reboot-Completed timeAssumedRebooted: "2025-09-05T20:23:38Z" ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get nodes/worker-0-1 -o json | jq .spec.taints null ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get pods | grep self self-node-remediation-controller-manager-56894b59db-cxq47 2/2 Running 1 (14m ago) 14m self-node-remediation-ds-bjzbb 1/1 Running 0 75s self-node-remediation-ds-c7fzq 1/1 Running 1 (14m ago) 14m self-node-remediation-ds-jz6jr 1/1 Running 1 (14m ago) 14m self-node-remediation-ds-mslr2 1/1 Running 1 (14m ago) 14m self-node-remediation-ds-p47rm 1/1 Running 2 (14m ago) 14m self-node-remediation-ds-tj85g 1/1 Running 2 (14m ago) 14m ============================================================================================================================== [kni@cert-rhosp-02 ~]$ oc logs self-node-remediation-controller-manager-56894b59db-cxq47 Defaulted container "manager" out of: manager, kube-rbac-proxy 2025-09-05T20:10:12.141575893Z INFO setup Go Version: go1.23.2 (Red Hat 1.23.2-1.el9) X:strictfipsruntime 2025-09-05T20:10:12.141652219Z INFO setup Go OS/Arch: linux/amd64 2025-09-05T20:10:12.14165558Z INFO setup Operator Version: v0.10.0 2025-09-05T20:10:12.141657465Z INFO setup Git Commit: 2025-09-05T20:10:12.141659878Z INFO setup Build Date: 2025-01-13T11:55:12+00:00 2025-09-05T20:10:12.14166171Z INFO setup HTTP/2 for metrics and webhook server disabled 2025-09-05T20:10:12.142020926Z INFO controller-runtime.metrics Metrics server is starting to listen {"addr": "127.0.0.1:8080"} 2025-09-05T20:10:12.146454578Z INFO utils-taints out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 33} 2025-09-05T20:10:12.146473179Z INFO utils-taints out of service taint strategy {"isGA": true, "k8sMajorVersion": 1, "k8sMinorVersion": 33} 2025-09-05T20:10:12.146477859Z INFO setup Starting as a manager that installs the daemonset 2025-09-05T20:10:12.146484012Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig"} 2025-09-05T20:10:12.146519711Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2025-09-05T20:10:12.146598146Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2025-09-05T20:10:12.146654098Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-09-05T20:10:12.146690732Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-09-05T20:10:12.146739838Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-09-05T20:10:12.146775705Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-09-05T20:10:12.146802614Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation"} 2025-09-05T20:10:12.146823618Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2025-09-05T20:10:12.146854129Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2025-09-05T20:10:12.146898799Z INFO setup starting manager 2025-09-05T20:10:12.147124169Z INFO Starting server {"kind": "health probe", "addr": "[::]:8081"} 2025-09-05T20:10:12.147122366Z INFO controller-runtime.webhook.webhooks Starting webhook server 2025-09-05T20:10:12.147202752Z INFO starting server {"path": "/metrics", "kind": "metrics", "addr": "127.0.0.1:8080"} I0905 20:10:12.147295 1 leaderelection.go:245] attempting to acquire leader lease openshift-workload-availability/547f6cb6.medik8s.io... 2025-09-05T20:10:12.147355906Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-09-05T20:10:12.147397519Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-09-05T20:10:12.14743704Z INFO controller-runtime.certwatcher Starting certificate watcher I0905 20:10:27.542874 1 leaderelection.go:255] successfully acquired lease openshift-workload-availability/547f6cb6.medik8s.io 2025-09-05T20:10:27.543047021Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1alpha1.SelfNodeRemediationConfig"} 2025-09-05T20:10:27.543137006Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1.DaemonSet"} 2025-09-05T20:10:27.543148905Z INFO Starting Controller {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig"} 2025-09-05T20:10:27.543105733Z INFO Starting EventSource {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "source": "kind source: *v1alpha1.SelfNodeRemediation"} 2025-09-05T20:10:27.543174389Z INFO Starting Controller {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation"} 2025-09-05T20:10:27.543193102Z DEBUG events self-node-remediation-controller-manager-56894b59db-cxq47_8d3ca790-2e0a-4149-b5c6-90acedd2b793 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"547f6cb6.medik8s.io","uid":"77b8541f-0ce5-4604-8423-28854b8844bc","apiVersion":"coordination.k8s.io/v1","resourceVersion":"259715"}, "reason": "LeaderElection"} 2025-09-05T20:10:27.549438322Z INFO selfnoderemediationconfig-resource validate create {"name": "self-node-remediation-config"} 2025-09-05T20:10:27.645172132Z INFO Starting workers {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "worker count": 1} 2025-09-05T20:10:27.645306603Z INFO Starting workers {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "worker count": 1} 2025-09-05T20:10:27.645433399Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-09-05T20:10:27.748163457Z INFO controllers.SelfNodeRemediationConfig Creating new certs 2025-09-05T20:10:29.408649292Z INFO controllers.SelfNodeRemediationConfig Storing certs in new secret 2025-09-05T20:10:29.412728917Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-09-05T20:10:29.413408627Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025-09-05T20:10:29.413443235Z INFO controllers.SelfNodeRemediationConfig snr didn't find old daemonset to be deleted 2025/09/05 20:10:29 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/09/05 20:10:29 does not exist, creating (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/09/05 20:10:29 successfully created (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025-09-05T20:10:29.426541196Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-09-05T20:10:29.426566291Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-09-05T20:10:29.426571208Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-09-05T20:10:29.42708056Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/09/05 20:10:29 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/09/05 20:10:29 update was successful 2025-09-05T20:10:37.548130457Z INFO selfnoderemediationtemplate-resource default {"name": "self-node-remediation-automatic-strategy-template"} 2025-09-05T20:10:37.552039509Z INFO selfnoderemediationtemplate-resource validate create {"name": "self-node-remediation-automatic-strategy-template"} 2025-09-05T20:11:07.068884933Z INFO selfnoderemediationtemplate-resource default {"name": "selfnoderemediationtemplate-sample"} 2025-09-05T20:11:07.070419572Z INFO selfnoderemediationtemplate-resource validate create {"name": "selfnoderemediationtemplate-sample"} 2025-09-05T20:21:37.022712403Z INFO selfnoderemediation-resource validate create {"name": "ocp-edge-cluster-0-pjlrl-worker-0-qp62r"} 2025-09-05T20:21:37.229040391Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:21:37.229068042Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:21:37.229187894Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","uid":"3f8d0151-1846-4dc0-85bc-b976409dc43c","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"263834"}, "reason": "RemediationStarted"} 2025-09-05T20:21:37.33319791Z INFO selfnoderemediation-resource validate update {"name": "ocp-edge-cluster-0-pjlrl-worker-0-qp62r"} 2025-09-05T20:21:37.336697781Z INFO controllers.SelfNodeRemediation finalizer added {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:21:37.336807229Z DEBUG events [remediation] Remediation process - successful adding finalizer{"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","uid":"3f8d0151-1846-4dc0-85bc-b976409dc43c","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"263840"}, "reason": "AddFinalizer"} 2025-09-05T20:21:37.341849235Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:21:37.341861334Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:21:37.35258228Z INFO controllers.SelfNodeRemediation NoExecute taint added {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-09-05T20:20:06Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-09-05T20:20:11Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-09-05T20:21:37Z"}]} 2025-09-05T20:21:37.352655999Z INFO controllers.SelfNodeRemediation Marking node as unschedulable {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}, "node name": "worker-0-1"} 2025-09-05T20:21:37.352720194Z DEBUG events [remediation] Remediation process - NoExecute taint added to the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-1","uid":"d53e1282-e281-4d1f-9d2d-4d612a800767","apiVersion":"v1","resourceVersion":"263843"}, "reason": "AddNoExecute"} 2025-09-05T20:21:37.362130353Z DEBUG events [remediation] Remediation process - unhealthy node marked as unschedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-1","uid":"d53e1282-e281-4d1f-9d2d-4d612a800767","apiVersion":"v1","resourceVersion":"263846"}, "reason": "MarkUnschedulable"} 2025-09-05T20:21:37.375830408Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:21:37.375843495Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:21:37.376033019Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}, "node name": "worker-0-1"} 2025-09-05T20:21:38.376437725Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:21:38.376455368Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:21:38.376697032Z INFO rebootDurationCalculator No SafeTimeToAssumeNodeRebootedSeconds specified, using calculated minimum safe reboot time {"calculated minimum time in seconds": 120} 2025-09-05T20:21:38.376705428Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}, "node name": "worker-0-1", "time": "2025-09-05 20:23:38.376704932 +0000 UTC m=+806.247294217"} 2025-09-05T20:21:38.376814793Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","uid":"3f8d0151-1846-4dc0-85bc-b976409dc43c","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"263850"}, "reason": "UpdateTimeAssumedRebooted"} 2025-09-05T20:21:38.382352794Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:21:38.382370678Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}, "node name": "worker-0-1", "time left": "2m0.617630562s"} 2025-09-05T20:23:25.185964123Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:23:25.185979619Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}, "node name": "worker-0-1", "time left": "13.814021332s"} 2025-09-05T20:23:39.006322969Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:23:39.006342949Z INFO controllers.SelfNodeRemediation TimeAssumedRebooted is old. The unhealthy node assumed to been rebooted {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}, "node name": "worker-0-1"} 2025-09-05T20:23:39.01180446Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:23:39.019252466Z INFO controllers.SelfNodeRemediation out-of-service taint added {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}, "new taints": [{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-09-05T20:21:37Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-09-05T20:21:37Z"},{"key":"node.kubernetes.io/out-of-service","value":"nodeshutdown","effect":"NoExecute","timeAdded":"2025-09-05T20:23:39Z"}]} 2025-09-05T20:23:39.019550165Z DEBUG events [remediation] Remediation process - add out-of-service taint to unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-1","uid":"d53e1282-e281-4d1f-9d2d-4d612a800767","apiVersion":"v1","resourceVersion":"265368"}, "reason": "AddOutOfService"} 2025-09-05T20:23:39.136757176Z INFO controllers.SelfNodeRemediation out-of-service taint removed {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}, "new taints": [{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-09-05T20:21:37Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-09-05T20:21:37Z"}]} 2025-09-05T20:23:39.136918827Z DEBUG events [remediation] Remediation process - remove out-of-service taint from node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-1","uid":"d53e1282-e281-4d1f-9d2d-4d612a800767","apiVersion":"v1","resourceVersion":"265375"}, "reason": "RemoveOutOfService"} 2025-09-05T20:23:39.136961Z DEBUG events [remediation] Remediation process - finished deleting unhealthy node resources {"type": "Normal", "object": {"kind":"Node","name":"worker-0-1","uid":"d53e1282-e281-4d1f-9d2d-4d612a800767","apiVersion":"v1","resourceVersion":"265375"}, "reason": "DeleteResources"} 2025-09-05T20:23:39.14510019Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:23:39.145121259Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:23:39.153253428Z DEBUG events [remediation] Remediation process - mark healthy remediated node as schedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-1","uid":"d53e1282-e281-4d1f-9d2d-4d612a800767","apiVersion":"v1","resourceVersion":"265379"}, "reason": "MarkNodeSchedulable"} 2025-09-05T20:23:40.15920568Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:23:40.159283643Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:23:40.166737888Z INFO controllers.SelfNodeRemediation NoExecute taint removed {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}, "new taints": null} 2025-09-05T20:23:40.166930135Z DEBUG events [remediation] Remediation process - remove NoExecute taint from healthy remediated node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-1","uid":"d53e1282-e281-4d1f-9d2d-4d612a800767","apiVersion":"v1","resourceVersion":"265385"}, "reason": "RemoveNoExecuteTaint"} 2025-09-05T20:23:40.173438615Z INFO selfnoderemediation-resource validate update {"name": "ocp-edge-cluster-0-pjlrl-worker-0-qp62r"} 2025-09-05T20:23:40.179434466Z INFO controllers.SelfNodeRemediation finalizer removed {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:23:40.179629199Z DEBUG events [remediation] Remediation process - remove finalizer from snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","uid":"3f8d0151-1846-4dc0-85bc-b976409dc43c","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"265376"}, "reason": "RemoveFinalizer"} 2025-09-05T20:23:40.179710683Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-machine-api","name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","uid":"3f8d0151-1846-4dc0-85bc-b976409dc43c","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"265376"}, "reason": "RemediationFinished"} 2025-09-05T20:23:40.184251098Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}} 2025-09-05T20:23:41.185090307Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"ocp-edge-cluster-0-pjlrl-worker-0-qp62r","namespace":"openshift-machine-api"}}