[root@cert-rhosp-02 ~]# oc project openshift-workload-availability Now using project "openshift-workload-availability" on server "https://api.ocp-edge-cluster-0.qe.lab.redhat.com:6443". [root@cert-rhosp-02 ~]# oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.20.0-0.nightly-2025-11-14-130157 True False 29h Cluster version is 4.20.0-0.nightly-2025-11-14-130157 [root@cert-rhosp-02 ~]# oc get csv NAME DISPLAY VERSION REPLACES PHASE machine-deletion-remediation.v0.5.0 Machine Deletion Remediation operator 0.5.0 machine-deletion-remediation.v0.4.1 Succeeded node-healthcheck-operator.v0.10.1 Node Health Check Operator 0.10.1 node-healthcheck-operator.v0.10.0 Succeeded self-node-remediation.v0.11.0 Self Node Remediation Operator 0.11.0 self-node-remediation.v0.10.1 Succeeded [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep node-healthcheck-controller-manager) [root@cert-rhosp-02 ~]# echo $PODS pod/node-healthcheck-controller-manager-79965596bb-7pdwc pod/node-healthcheck-controller-manager-79965596bb-tfgfx [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/node-healthcheck-controller-manager-79965596bb-7pdwc "master-0-2" == pod/node-healthcheck-controller-manager-79965596bb-tfgfx "master-0-1" [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep self-node-remediation-controller-manager) [root@cert-rhosp-02 ~]# echo $PODS pod/self-node-remediation-controller-manager-77c688fb7b-c6wjg pod/self-node-remediation-controller-manager-77c688fb7b-s8rvc [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/self-node-remediation-controller-manager-77c688fb7b-c6wjg "worker-0-2" == pod/self-node-remediation-controller-manager-77c688fb7b-s8rvc "worker-0-1" [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep machine-deletion-remediation-controller) [root@cert-rhosp-02 ~]# echo $PODS pod/machine-deletion-remediation-controller-manager-b468d785d-gwtmq [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/machine-deletion-remediation-controller-manager-b468d785d-gwtmq "worker-0-1" [root@cert-rhosp-02 ~]# vi test.yaml [root@cert-rhosp-02 ~]# cat test.yaml --- kind: MachineDeletionRemediationTemplate apiVersion: machine-deletion-remediation.medik8s.io/v1alpha1 metadata: name: mdr-template namespace: openshift-workload-availability spec: template: spec: {} --- apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate metadata: namespace: openshift-workload-availability name: selfnoderemediationtemplate-sample spec: template: spec: remediationStrategy: Automatic --- apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: name: nhc-mdr-snr spec: minHealthy: 30% healthyDelay: 5s escalatingRemediations: - remediationTemplate: apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate name: selfnoderemediationtemplate-sample namespace: openshift-workload-availability order: 0 timeout: 60s - remediationTemplate: apiVersion: machine-deletion-remediation.medik8s.io/v1alpha1 kind: MachineDeletionRemediationTemplate name: mdr-template namespace: openshift-workload-availability order: -1 timeout: 500s selector: matchExpressions: - key: node-role.kubernetes.io/worker operator: Exists unhealthyConditions: - type: Ready status: "False" duration: 30s - type: Ready status: Unknown duration: 30s [root@cert-rhosp-02 ~]# oc apply -f test.yaml machinedeletionremediationtemplate.machine-deletion-remediation.medik8s.io/mdr-template created selfnoderemediationtemplate.self-node-remediation.medik8s.io/selfnoderemediationtemplate-sample created nodehealthcheck.remediation.medik8s.io/nhc-mdr-snr created [root@cert-rhosp-02 ~]# oc get node worker-0-0 -o json | jq .spec.taints null [root@cert-rhosp-02 ~]# oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 30h v1.33.5 worker-0-1 Ready worker 30h v1.33.5 worker-0-2 Ready worker 30h v1.33.5 [root@cert-rhosp-02 ~]# oc get snr No resources found in openshift-workload-availability namespace. [root@cert-rhosp-02 ~]# oc get mdr No resources found in openshift-workload-availability namespace. [root@cert-rhosp-02 ~]# oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 NotReady worker 34h v1.33.5 worker-0-1 Ready worker 34h v1.33.5 worker-0-2 Ready worker 34h v1.33.5 [root@cert-rhosp-02 ~]# oc get mdr -o yaml apiVersion: v1 items: - apiVersion: machine-deletion-remediation.medik8s.io/v1alpha1 kind: MachineDeletionRemediation metadata: annotations: machine-deletion-remediation.medik8s.io/machineNameNamespace: openshift-machine-api/ocp-edge-cluster-0-6bcbf-worker-0-6qhcr machine-deletion-remediation.medik8s.io/machineOwner: MachineSet/ocp-edge-cluster-0-6bcbf-worker-0 remediation.medik8s.io/node-name: worker-0-0 remediation.medik8s.io/template-name: mdr-template creationTimestamp: "2025-11-16T10:56:06Z" generation: 1 labels: app.kubernetes.io/part-of: node-healthcheck-controller name: worker-0-0 namespace: openshift-workload-availability ownerReferences: - apiVersion: remediation.medik8s.io/v1alpha1 controller: false kind: NodeHealthCheck name: nhc-mdr-snr uid: 5cc37621-86e0-4a9a-8751-641e5567058f resourceVersion: "575410" uid: c60d8235-77ba-41f8-964a-4ff9016784a8 spec: {} status: conditions: - lastTransitionTime: "2025-11-16T10:56:06Z" message: "" reason: RemediationStarted status: "True" type: Processing - lastTransitionTime: "2025-11-16T10:56:06Z" message: "" reason: RemediationStarted status: Unknown type: Succeeded - lastTransitionTime: "2025-11-16T10:56:06Z" message: 'Machine will be deleted and the unhealthy node replaced. This is a BareMetal cluster provider: the new node is NOT expected to have a new name' reason: MachineDeletionOnBareMetalProviderKeepsNodeName status: "False" type: PermanentNodeDeletionExpected kind: List metadata: resourceVersion: "" [root@cert-rhosp-02 ~]# oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 38h v1.33.5 worker-0-1 Ready worker 38h v1.33.5 worker-0-2 Ready worker 38h v1.33.5 [root@cert-rhosp-02 ~]# oc get mdr No resources found in openshift-workload-availability namespace. [root@cert-rhosp-02 ~]# oc get snr No resources found in openshift-workload-availability namespace. [root@cert-rhosp-02 ~]# oc get nhc NAME AGE nhc-mdr-snr 8h [root@cert-rhosp-02 ~]# oc get nhc -o yaml apiVersion: v1 items: - apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","metadata":{"annotations":{},"name":"nhc-mdr-snr"},"spec":{"escalatingRemediations":[{"order":0,"remediationTemplate":{"apiVersion":"self-node-remediation.medik8s.io/v1alpha1","kind":"SelfNodeRemediationTemplate","name":"selfnoderemediationtemplate-sample","namespace":"openshift-workload-availability"},"timeout":"60s"},{"order":-1,"remediationTemplate":{"apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1","kind":"MachineDeletionRemediationTemplate","name":"mdr-template","namespace":"openshift-workload-availability"},"timeout":"500s"}],"healthyDelay":"5s","minHealthy":"30%","selector":{"matchExpressions":[{"key":"node-role.kubernetes.io/worker","operator":"Exists"}]},"unhealthyConditions":[{"duration":"30s","status":"False","type":"Ready"},{"duration":"30s","status":"Unknown","type":"Ready"}]}} creationTimestamp: "2025-11-16T06:25:20Z" generation: 1 name: nhc-mdr-snr resourceVersion: "580661" uid: 5cc37621-86e0-4a9a-8751-641e5567058f spec: escalatingRemediations: - order: 0 remediationTemplate: apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate name: selfnoderemediationtemplate-sample namespace: openshift-workload-availability timeout: 60s - order: -1 remediationTemplate: apiVersion: machine-deletion-remediation.medik8s.io/v1alpha1 kind: MachineDeletionRemediationTemplate name: mdr-template namespace: openshift-workload-availability timeout: 500s healthyDelay: 5s minHealthy: 30% selector: matchExpressions: - key: node-role.kubernetes.io/worker operator: Exists unhealthyConditions: - duration: 30s status: "False" type: Ready - duration: 30s status: Unknown type: Ready status: conditions: - lastTransitionTime: "2025-11-16T06:25:20Z" message: No issues found, NodeHealthCheck is enabled. reason: NodeHealthCheckEnabled status: "False" type: Disabled healthyNodes: 3 lastUpdateTime: "2025-11-16T11:10:35Z" observedNodes: 3 phase: Enabled reason: NHC is enabled, no ongoing remediation kind: List metadata: resourceVersion: "" [root@cert-rhosp-02 ~]# oc get node worker-0-0 -o json | jq .spec.taints null MDR logs: [root@cert-rhosp-02 ~]# oc logs pod/machine-deletion-remediation-controller-manager-b468d785d-gwtmq -c manager 2025-11-16T06:19:33.672565089Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-11-16T06:19:33.672694927Z INFO setup Go OS/Arch: linux/amd64 2025-11-16T06:19:33.672698299Z INFO setup Operator Version: 2c7f718 2025-11-16T06:19:33.672700344Z INFO setup Git Commit: 2c7f718ad98018b682353cefdf55662191b8fe8d 2025-11-16T06:19:33.672702293Z INFO setup Build Date: 2025-11-11T13:09:46+00:00 2025-11-16T06:19:33.674496194Z INFO setup starting manager 2025-11-16T06:19:33.674670259Z INFO controller-runtime.metrics Starting metrics server 2025-11-16T06:19:33.674736474Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": "127.0.0.1:8080", "secure": false} 2025-11-16T06:19:33.674780795Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} I1116 06:19:33.674842 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/285d4098.example.com... I1116 06:19:33.685999 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/285d4098.example.com 2025-11-16T06:19:33.686081487Z DEBUG events machine-deletion-remediation-controller-manager-b468d785d-gwtmq_418464e3-6d18-4d05-80ca-878908d9e242 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"285d4098.example.com","uid":"870b4024-45e4-48e0-b964-5b6156cbb17f","apiVersion":"coordination.k8s.io/v1","resourceVersion":"482033"}, "reason": "LeaderElection"} 2025-11-16T06:19:33.686301352Z INFO Starting EventSource {"controller": "machinedeletionremediation", "controllerGroup": "machine-deletion-remediation.medik8s.io", "controllerKind": "MachineDeletionRemediation", "source": "kind source: *v1alpha1.MachineDeletionRemediation"} 2025-11-16T06:19:33.686389735Z INFO Starting Controller {"controller": "machinedeletionremediation", "controllerGroup": "machine-deletion-remediation.medik8s.io", "controllerKind": "MachineDeletionRemediation"} 2025-11-16T06:19:33.872618132Z INFO Starting workers {"controller": "machinedeletionremediation", "controllerGroup": "machine-deletion-remediation.medik8s.io", "controllerKind": "MachineDeletionRemediation", "worker count": 1} 2025-11-16T10:56:06.460127481Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:56:06.460222688Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T10:56:06.460231152Z INFO controllers.MachineDeletionRemediation updating Status Condition {"processingConditionStatus": "True", "succededConditionStatus": "Unknown", "reason": "RemediationStarted"} 2025-11-16T10:56:06.464670678Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:56:06.464693254Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T10:56:06.56676063Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T10:56:06.668593141Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T10:56:06.668637194Z INFO controllers.MachineDeletionRemediation updating Status Condition {"PermanentNodeDeletionExpected": "False", "reason": "MachineDeletionOnBareMetalProviderKeepsNodeName", "message": "Machine will be deleted and the unhealthy node replaced. This is a BareMetal cluster provider: the new node is NOT expected to have a new name"} 2025-11-16T10:56:06.668649945Z INFO controllers.MachineDeletionRemediation Machine will be deleted and the unhealthy node replaced. This is a BareMetal cluster provider: the new node is NOT expected to have a new name {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:56:06.668890799Z DEBUG events [remediation] Machine will be deleted and the unhealthy node replaced. This is a BareMetal cluster provider: the new node is NOT expected to have a new name {"type": "Normal", "object": {"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1","resourceVersion":"575406"}, "reason": "PermanentNodeDeletionExpected"} 2025-11-16T10:56:06.678586944Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:56:06.678621343Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T10:56:06.678653259Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T10:56:06.678667089Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T10:56:06.685783753Z INFO controllers.MachineDeletionRemediation request machine deletion {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "remediation name": "worker-0-0"} 2025-11-16T10:56:06.698427022Z DEBUG events [remediation] Remediation started {"type": "Normal", "object": {"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1","resourceVersion":"575410"}, "reason": "RemediationStarted"} 2025-11-16T10:56:06.713255247Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:56:06.713283783Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T10:56:06.713289418Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T10:56:06.713311009Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T10:56:06.713313772Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T10:56:06.72294717Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:56:06.722969049Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T10:56:06.72297315Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T10:56:06.722984949Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T10:56:06.722987133Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T10:56:36.723495845Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:56:36.723536771Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T10:56:36.723542536Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T10:56:36.723557538Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T10:56:36.723560092Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T10:57:06.729593886Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:57:06.729631075Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T10:57:06.72963609Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T10:57:06.729654081Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T10:57:06.729657059Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T10:57:36.734727873Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:57:36.734765829Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T10:57:36.734772132Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T10:57:36.734857824Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T10:57:36.734866333Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T10:58:06.740312479Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:58:06.740351366Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T10:58:06.740356503Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T10:58:06.740370156Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T10:58:06.740372601Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T10:58:36.747072678Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:58:36.747115428Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T10:58:36.747121003Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T10:58:36.747135682Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T10:58:36.747138149Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T10:59:06.7524841Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:59:06.752524795Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T10:59:06.752530102Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T10:59:06.752548598Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T10:59:06.752551539Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T10:59:36.759198386Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T10:59:36.759246497Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T10:59:36.759253367Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T10:59:36.759267722Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T10:59:36.759270191Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T11:00:06.765196045Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:00:06.765241837Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:00:06.765247479Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T11:00:06.76526555Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T11:00:06.76526811Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T11:00:36.770457865Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:00:36.770496293Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:00:36.770501682Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T11:00:36.770517698Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T11:00:36.770520423Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T11:01:06.776420794Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:01:06.776473406Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:01:06.776480066Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T11:01:06.776502861Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T11:01:06.776505799Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T11:01:36.782247955Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:01:36.782285821Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:01:36.782291359Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T11:01:36.782305719Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T11:01:36.782308354Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T11:02:06.788313712Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:02:06.788363397Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:02:06.788369301Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T11:02:06.788388354Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T11:02:06.788391035Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T11:02:36.793453702Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:02:36.7935309Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:02:36.793540574Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T11:02:36.793559347Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T11:02:36.793562149Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T11:03:06.799391099Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:03:06.799570172Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:03:06.799592151Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T11:03:06.799693773Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T11:03:06.79971343Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T11:03:36.804438648Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:03:36.804517072Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:03:36.804524047Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T11:03:36.804551871Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T11:03:36.804558165Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T11:04:06.810855508Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:06.81092554Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:04:06.810933739Z INFO controllers.MachineDeletionRemediation Looking for the target Machine {"machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "namespace": "openshift-machine-api"} 2025-11-16T11:04:06.810954648Z INFO controllers.MachineDeletionRemediation target machine found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr"} 2025-11-16T11:04:06.810957399Z INFO controllers.MachineDeletionRemediation target machine was not deleted yet {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "machine": "ocp-edge-cluster-0-6bcbf-worker-0-6qhcr", "machine status.phase": "Deleting"} 2025-11-16T11:04:26.336946132Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:26.336986518Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:04:26.336991155Z INFO controllers.MachineDeletionRemediation updating Status Condition {"processingConditionStatus": "False", "succededConditionStatus": "False", "reason": "RemediationStoppedByNHC"} 2025-11-16T11:04:26.336997087Z INFO controllers.MachineDeletionRemediation NHC time out annotation found, stopping remediation {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:26.337214133Z DEBUG events [remediation] NHC added the timed-out annotation, remediation will be stopped {"type": "Normal", "object": {"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1","resourceVersion":"578326"}, "reason": "RemediationStopped"} 2025-11-16T11:04:26.342709789Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:26.342751438Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:04:36.816764317Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:36.816851427Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:10:28.720711162Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:10:28.720776973Z INFO controllers.MachineDeletionRemediation Machine Deletion Remediation CR found {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}, "name": "worker-0-0"} 2025-11-16T11:10:34.049357402Z INFO controllers.MachineDeletionRemediation reconciling... {"machinedeletionremediation": {"name":"worker-0-0","namespace":"openshift-workload-availability"}} 2025-11-16T11:10:34.049516091Z INFO controllers.MachineDeletionRemediation MDR already deleted, nothing to do NHC logs: [root@cert-rhosp-02 ~]# oc logs pod/node-healthcheck-controller-manager-79965596bb-tfgfx 2025-11-16T06:16:31.973841327Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-11-16T06:16:31.973988499Z INFO setup Go OS/Arch: linux/amd64 2025-11-16T06:16:31.973993453Z INFO setup Operator Version: 2ef588d 2025-11-16T06:16:31.973996078Z INFO setup Git Commit: 2ef588d65de4087c46447401cd6e757459d79210 2025-11-16T06:16:31.973998713Z INFO setup Build Date: 2025-11-11T13:40:45+00:00 2025-11-16T06:16:31.974001274Z INFO setup HTTP/2 for metrics and webhook server disabled 2025-11-16T06:16:32.000317622Z INFO setup supported control plane topology {"topology": "HighlyAvailable"} 2025-11-16T06:16:32.00034876Z INFO setup Cluster capabilities {"IsOnOpenshift": true, "HasMachineAPI": true} 2025-11-16T06:16:32.000598607Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "remediation.medik8s.io/v1alpha1, Kind=NodeHealthCheck", "path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2025-11-16T06:16:32.000667179Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2025-11-16T06:16:32.00071946Z INFO setup starting manager 2025-11-16T06:16:32.00076582Z INFO controller-runtime.metrics Starting metrics server 2025-11-16T06:16:32.000833496Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-11-16T06:16:32.00086093Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": "127.0.0.1:8080", "secure": false} 2025-11-16T06:16:32.000907293Z INFO controller-runtime.webhook Starting webhook server 2025-11-16T06:16:32.001129801Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-11-16T06:16:32.001240882Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} 2025-11-16T06:16:32.001262783Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} I1116 06:16:32.101996 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/e1f13584.medik8s.io... I1116 06:16:32.108335 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/e1f13584.medik8s.io 2025-11-16T06:16:32.108408542Z DEBUG events node-healthcheck-controller-manager-79965596bb-tfgfx_d74d675a-cd78-4036-9c82-aba4feaef3b4 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"e1f13584.medik8s.io","uid":"ddc0b050-aa35-4403-a90f-4cc1fd4c2d28","apiVersion":"coordination.k8s.io/v1","resourceVersion":"479283"}, "reason": "LeaderElection"} 2025-11-16T06:16:32.10869523Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "channel source: 0xc000740070"} 2025-11-16T06:16:32.108729263Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1alpha1.NodeHealthCheck"} 2025-11-16T06:16:32.108730471Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "channel source: 0xc000740310"} 2025-11-16T06:16:32.108761909Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.MachineHealthCheck"} 2025-11-16T06:16:32.108776894Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.Machine"} 2025-11-16T06:16:32.108809606Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1.Node"} I1116 06:16:32.109852 1 shared_informer.go:313] Waiting for caches to sync for feature gate accessor 2025-11-16T06:16:32.114421002Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1.Node"} 2025-11-16T06:16:32.208895221Z INFO MHCChecker found termination handler MHC, will ignore Nodes with Terminating condition 2025-11-16T06:16:32.208938334Z INFO MHCChecker MHC Checker status changed, notifying NHC controller 2025-11-16T06:16:32.208999722Z INFO adding all NHCs to reconcile queue for handling MHC event I1116 06:16:32.210661 1 shared_informer.go:320] Caches are synced for feature gate accessor I1116 06:16:32.210685 1 simple_featuregate_reader.go:171] Starting feature-gate-detector 2025-11-16T06:16:32.210792647Z INFO FeatureGateAccessor FeatureGates initialized I1116 06:16:32.210888 1 recorder_logging.go:49] &Event{ObjectMeta:{dummy.1878682bfa7adb10.57258c66 dummy 0 0001-01-01 00:00:00 +0000 UTC map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Pod,Namespace:dummy,Name:dummy,UID:,APIVersion:v1,ResourceVersion:,FieldPath:,},Reason:FeatureGatesInitialized,Message:FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "NewOLM", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}},Source:EventSource{Component:,Host:,},FirstTimestamp:2025-11-16 06:16:32.210787088 +0000 UTC m=+0.262344869,LastTimestamp:2025-11-16 06:16:32.210787088 +0000 UTC m=+0.262344869,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:,ReportingInstance:,} 2025-11-16T06:16:32.309220444Z INFO Starting Controller {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck"} 2025-11-16T06:16:32.309253629Z INFO Starting workers {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "worker count": 1} 2025-11-16T06:16:32.317383904Z INFO Starting Controller {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck"} 2025-11-16T06:16:32.317468044Z INFO Starting workers {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "worker count": 1} 2025-11-16T06:16:32.521836222Z INFO console-plugin successfully created / updated console plugin resources 2025-11-16T06:25:20.633999806Z INFO nodehealthcheck-resource validate create {"name": "nhc-mdr-snr"} 2025-11-16T06:25:20.636683682Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T06:25:20.66936727Z INFO controllers.NodeHealthCheck enabling NHC, valid config, no conflicting MHC configured in the cluster {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T06:25:20.669461378Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-11-16T06:25:20.669480779Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation template CRs {"kind": "SelfNodeRemediationTemplate"} 2025-11-16T06:25:20.669488719Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-11-16T06:25:20.669492353Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation CRs {"kind": "SelfNodeRemediation"} 2025-11-16T06:25:20.669498695Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-11-16T06:25:20.669502108Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation template CRs {"kind": "MachineDeletionRemediationTemplate"} 2025-11-16T06:25:20.669506329Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-11-16T06:25:20.669510265Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation CRs {"kind": "MachineDeletionRemediation"} 2025-11-16T06:25:20.669752505Z DEBUG events [remediation] No issues found, NodeHealthCheck is enabled. {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"484139"}, "reason": "Enabled"} 2025-11-16T06:25:20.676025721Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T06:25:20Z" level=info msg="invalidating lease" time="2025-11-16T06:25:20Z" level=info msg="getting lease" 2025-11-16T06:25:20.877140744Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-11-16T06:25:20Z" level=info msg="invalidating lease" time="2025-11-16T06:25:20Z" level=info msg="getting lease" 2025-11-16T06:25:20.882034055Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T06:25:20Z" level=info msg="invalidating lease" time="2025-11-16T06:25:20Z" level=info msg="getting lease" 2025-11-16T06:25:20.885632556Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":3,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation"}, "patch": "{\"status\":{\"conditions\":[{\"lastTransitionTime\":\"2025-11-16T06:25:20Z\",\"message\":\"No issues found, NodeHealthCheck is enabled.\",\"reason\":\"NodeHealthCheckEnabled\",\"status\":\"False\",\"type\":\"Disabled\"}],\"healthyNodes\":3,\"observedNodes\":3,\"phase\":\"Enabled\",\"reason\":\"NHC is enabled, no ongoing remediation\"}}"} 2025-11-16T06:25:21.090957893Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-11-16T10:55:35.866447921Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-mdr-snr"} 2025-11-16T10:55:35.866515104Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-mdr-snr"} 2025-11-16T10:55:35.866573745Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T10:55:35.879437136Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown", "duration left": "29.120565434s"} 2025-11-16T10:55:35.892551682Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T10:55:35Z" level=info msg="invalidating lease" time="2025-11-16T10:55:35Z" level=info msg="getting lease" 2025-11-16T10:55:35.993311738Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T10:55:36Z" level=info msg="invalidating lease" time="2025-11-16T10:55:36Z" level=info msg="getting lease" 2025-11-16T10:55:36.094924532Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-11-16T06:25:20Z"}, "patch": "{\"status\":{\"healthyNodes\":2}}"} 2025-11-16T10:55:36.306131964Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "30.120565434s"} 2025-11-16T10:56:06.427367594Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T10:56:06.4340327Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-11-16T10:56:06.434213207Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-0", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"575169"}, "reason": "DetectedUnhealthy"} 2025-11-16T10:56:06.439364057Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T10:56:06Z" level=info msg="invalidating lease" time="2025-11-16T10:56:06Z" level=info msg="getting lease" 2025-11-16T10:56:06.441959926Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T10:56:06Z" level=info msg="invalidating lease" time="2025-11-16T10:56:06Z" level=info msg="getting lease" 2025-11-16T10:56:06.447079623Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T10:56:06.450896945Z INFO controllers.NodeHealthCheck.resource manager Attempting to obtain Node Lease {"Node name": "worker-0-0"} time="2025-11-16T10:56:06Z" level=info msg="request lease" time="2025-11-16T10:56:06Z" level=info msg="getting lease" time="2025-11-16T10:56:06Z" level=info msg="create lease" 2025-11-16T10:56:06.454660383Z INFO controllers.NodeHealthCheck.resource manager Creating a remediation CR {"CR name": "worker-0-0", "CR kind": "MachineDeletionRemediation", "namespace": "openshift-workload-availability"} 2025-11-16T10:56:06.459018692Z DEBUG events [remediation] Created remediation object for node worker-0-0 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"575169"}, "reason": "RemediationCreated"} 2025-11-16T10:56:06.464351308Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T10:56:06.464408292Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-11-16T10:56:06.464420279Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T10:56:06.464438608Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-11-16T10:56:06.539525004Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T10:56:06Z","templateName":"mdr-template"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-16T10:55:36Z"}, "patch": "{\"spec\":{\"escalatingRemediations\":[{\"order\":-1,\"remediationTemplate\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediationTemplate\",\"name\":\"mdr-template\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"8m20s\"},{\"order\":0,\"remediationTemplate\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediationTemplate\",\"name\":\"selfnoderemediationtemplate-sample\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"1m0s\"}]},\"status\":{\"phase\":\"Remediating\",\"reason\":\"NHC is remediating 1 nodes\",\"unhealthyNodes\":[{\"name\":\"worker-0-0\",\"remediations\":[{\"resource\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediation\",\"name\":\"worker-0-0\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"c60d8235-77ba-41f8-964a-4ff9016784a8\"},\"started\":\"2025-11-16T10:56:06Z\",\"templateName\":\"mdr-template\"}]}]}}"} 2025-11-16T10:56:06.680265972Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T10:56:06.680370986Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-11-16T10:56:06.680393685Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T10:56:06.680407371Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-11-16T10:56:06.689940799Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T10:56:06.68997168Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-11-16T10:56:06.689980882Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T10:56:06.689985102Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-11-16T10:56:06.745767208Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "8m20s"} 2025-11-16T10:56:06.745865307Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T10:56:06.761982223Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-11-16T10:56:06.774997129Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T10:56:06.775038456Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T10:56:06.775049235Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T10:56:06Z" level=info msg="invalidating lease" time="2025-11-16T10:56:06Z" level=info msg="getting lease" 2025-11-16T10:56:06.779124452Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T10:56:06Z" level=info msg="invalidating lease" time="2025-11-16T10:56:06Z" level=info msg="getting lease" 2025-11-16T10:56:06.875236598Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T10:56:06.882955315Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-0", "kind": "MachineDeletionRemediation", "namespace": "openshift-workload-availability"} time="2025-11-16T10:56:06Z" level=info msg="getting lease" 2025-11-16T10:56:06.88307695Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-0", "NHC is lease owner": true, "lease expiration time": "8m20s"} 2025-11-16T10:56:06.883116856Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T10:56:06.977825293Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T10:56:06Z","templateName":"mdr-template"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-16T10:56:06Z"}, "patch": "{\"spec\":{\"escalatingRemediations\":[{\"order\":-1,\"remediationTemplate\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediationTemplate\",\"name\":\"mdr-template\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"8m20s\"},{\"order\":0,\"remediationTemplate\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediationTemplate\",\"name\":\"selfnoderemediationtemplate-sample\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"1m0s\"}]}}"} 2025-11-16T10:56:07.183738919Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "8m19.116888371s"} 2025-11-16T11:04:26.301071282Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:04:26.30844793Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-11-16T11:04:26.315543263Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:04:26.315577535Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:04:26.315585776Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:04:26Z" level=info msg="invalidating lease" time="2025-11-16T11:04:26Z" level=info msg="getting lease" 2025-11-16T11:04:26.319436795Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:04:26Z" level=info msg="invalidating lease" time="2025-11-16T11:04:26Z" level=info msg="getting lease" 2025-11-16T11:04:26.323529842Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T11:04:26.327998547Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-0", "kind": "MachineDeletionRemediation", "namespace": "openshift-workload-availability"} time="2025-11-16T11:04:26Z" level=info msg="getting lease" 2025-11-16T11:04:26.328084242Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-0", "NHC is lease owner": true, "lease expiration time": "8m20s"} time="2025-11-16T11:04:26Z" level=info msg="request lease" time="2025-11-16T11:04:26Z" level=info msg="getting lease" time="2025-11-16T11:04:26Z" level=info msg="renew lease owned by NodeHealthCheck-nhc-mdr-snr setAcquireTime=false" 2025-11-16T11:04:26.331570039Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:04:26.331598709Z INFO controllers.NodeHealthCheck remediation timed out {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:04:26.331603247Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:04:26.336600173Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:04:26.336646619Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-11-16T11:04:26.336657944Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:04:26.336661954Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-11-16T11:04:26.342515831Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:04:26.342551978Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-11-16T11:04:26.342564132Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:04:26.342569795Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-11-16T11:04:26.413795987Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T10:56:06Z","timedOut":"2025-11-16T11:04:26Z","templateName":"mdr-template"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-16T10:56:06Z"}, "patch": "{\"spec\":{\"escalatingRemediations\":[{\"order\":-1,\"remediationTemplate\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediationTemplate\",\"name\":\"mdr-template\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"8m20s\"},{\"order\":0,\"remediationTemplate\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediationTemplate\",\"name\":\"selfnoderemediationtemplate-sample\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"1m0s\"}]},\"status\":{\"unhealthyNodes\":[{\"name\":\"worker-0-0\",\"remediations\":[{\"resource\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediation\",\"name\":\"worker-0-0\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"c60d8235-77ba-41f8-964a-4ff9016784a8\"},\"started\":\"2025-11-16T10:56:06Z\",\"templateName\":\"mdr-template\",\"timedOut\":\"2025-11-16T11:04:26Z\"}]}]}}"} 2025-11-16T11:04:26.620050082Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "1s"} 2025-11-16T11:04:26.620137212Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:04:26.62602342Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-11-16T11:04:26.63171472Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:04:26.631749837Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:04:26.63175751Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:04:26Z" level=info msg="invalidating lease" time="2025-11-16T11:04:26Z" level=info msg="getting lease" 2025-11-16T11:04:26.634838549Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:04:26Z" level=info msg="invalidating lease" time="2025-11-16T11:04:26Z" level=info msg="getting lease" 2025-11-16T11:04:26.638106518Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T11:04:26.641200115Z INFO controllers.NodeHealthCheck.resource manager Attempting to obtain Node Lease {"Node name": "worker-0-0"} time="2025-11-16T11:04:26Z" level=info msg="request lease" time="2025-11-16T11:04:26Z" level=info msg="getting lease" 2025-11-16T11:04:26.641263419Z INFO controllers.NodeHealthCheck.resource manager Creating a remediation CR {"CR name": "", "CR kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} 2025-11-16T11:04:26.65108654Z DEBUG events [remediation] Created remediation object for node worker-0-0 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"578330"}, "reason": "RemediationCreated"} 2025-11-16T11:04:26.73549585Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T10:56:06Z","timedOut":"2025-11-16T11:04:26Z","templateName":"mdr-template"},{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T11:04:26Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-16T11:04:26Z"}, "patch": "{\"spec\":{\"escalatingRemediations\":[{\"order\":-1,\"remediationTemplate\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediationTemplate\",\"name\":\"mdr-template\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"8m20s\"},{\"order\":0,\"remediationTemplate\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediationTemplate\",\"name\":\"selfnoderemediationtemplate-sample\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"1m0s\"}]},\"status\":{\"unhealthyNodes\":[{\"name\":\"worker-0-0\",\"remediations\":[{\"resource\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediation\",\"name\":\"worker-0-0\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"c60d8235-77ba-41f8-964a-4ff9016784a8\"},\"started\":\"2025-11-16T10:56:06Z\",\"templateName\":\"mdr-template\",\"timedOut\":\"2025-11-16T11:04:26Z\"},{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-0-s79k4\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"c9d21434-2f9f-4654-8868-f8cddc87be5e\"},\"started\":\"2025-11-16T11:04:26Z\",\"templateName\":\"selfnoderemediationtemplate-sample\"}]}]}}"} 2025-11-16T11:04:26.859443234Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:04:26.859497701Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:04:26.859511736Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:04:26.859516358Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:04:26.863915523Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:04:26.863953147Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:04:26.863965286Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:04:26.863971011Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:04:26.894635192Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:04:26.894680827Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:04:26.894693016Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:04:26.894699071Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:04:26.942268682Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "1m0s"} 2025-11-16T11:04:26.942389443Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:04:26.979101304Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-11-16T11:04:26.987438888Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:04:26.987469936Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:04:26.987477107Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:04:26.987482767Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:04:27Z" level=info msg="invalidating lease" time="2025-11-16T11:04:27Z" level=info msg="getting lease" 2025-11-16T11:04:27.088595771Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:04:27Z" level=info msg="invalidating lease" time="2025-11-16T11:04:27Z" level=info msg="getting lease" 2025-11-16T11:04:27.092558643Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T11:04:27.096218899Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-0-s79k4", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2025-11-16T11:04:27Z" level=info msg="getting lease" 2025-11-16T11:04:27.096266379Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-0", "NHC is lease owner": true, "lease expiration time": "1m0s"} 2025-11-16T11:04:27.096287504Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:04:27.100157165Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T10:56:06Z","timedOut":"2025-11-16T11:04:26Z","templateName":"mdr-template"},{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T11:04:26Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-16T11:04:26Z"}, "patch": "{\"spec\":{\"escalatingRemediations\":[{\"order\":-1,\"remediationTemplate\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediationTemplate\",\"name\":\"mdr-template\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"8m20s\"},{\"order\":0,\"remediationTemplate\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediationTemplate\",\"name\":\"selfnoderemediationtemplate-sample\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"1m0s\"}]}}"} 2025-11-16T11:04:27.306176432Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "58.903715495s"} 2025-11-16T11:04:27.621045475Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:04:27.627890798Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-11-16T11:04:27.636744916Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:04:27.636772663Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:04:27.636779727Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:04:27.636786504Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:04:27Z" level=info msg="invalidating lease" time="2025-11-16T11:04:27Z" level=info msg="getting lease" 2025-11-16T11:04:27.640314079Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:04:27Z" level=info msg="invalidating lease" time="2025-11-16T11:04:27Z" level=info msg="getting lease" 2025-11-16T11:04:27.644219897Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T11:04:27.651262469Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-0-s79k4", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2025-11-16T11:04:27Z" level=info msg="getting lease" 2025-11-16T11:04:27.651386773Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-0", "NHC is lease owner": true, "lease expiration time": "1m0s"} 2025-11-16T11:04:27.651423068Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:04:27.6554491Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T10:56:06Z","timedOut":"2025-11-16T11:04:26Z","templateName":"mdr-template"},{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T11:04:26Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-16T11:04:27Z"}, "patch": "{\"spec\":{\"escalatingRemediations\":[{\"order\":-1,\"remediationTemplate\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediationTemplate\",\"name\":\"mdr-template\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"8m20s\"},{\"order\":0,\"remediationTemplate\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediationTemplate\",\"name\":\"selfnoderemediationtemplate-sample\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"1m0s\"}]}}"} 2025-11-16T11:04:27.861824887Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "58.348582096s"} 2025-11-16T11:05:26.21147047Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:05:26.21766416Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-11-16T11:05:26.224255487Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:05:26.224293016Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:05:26.22429996Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:05:26.224306058Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:05:26Z" level=info msg="invalidating lease" time="2025-11-16T11:05:26Z" level=info msg="getting lease" 2025-11-16T11:05:26.228116437Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:05:26Z" level=info msg="invalidating lease" time="2025-11-16T11:05:26Z" level=info msg="getting lease" 2025-11-16T11:05:26.231807723Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T11:05:26.235597666Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-0-s79k4", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2025-11-16T11:05:26Z" level=info msg="getting lease" 2025-11-16T11:05:26.235668Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-0", "NHC is lease owner": true, "lease expiration time": "1m0s"} 2025-11-16T11:05:26.235695591Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:05:26.235709287Z INFO controllers.NodeHealthCheck remediation timed out {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:05:26.235713316Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:05:26.242889185Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:05:26.242922499Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:05:26.242936758Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:05:26.242943319Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:05:26.248169212Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:05:26.248200889Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:05:26.248211012Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:05:26.248215281Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:05:26.328548575Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T10:56:06Z","timedOut":"2025-11-16T11:04:26Z","templateName":"mdr-template"},{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T11:04:26Z","timedOut":"2025-11-16T11:05:26Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-16T11:04:27Z"}, "patch": "{\"spec\":{\"escalatingRemediations\":[{\"order\":-1,\"remediationTemplate\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediationTemplate\",\"name\":\"mdr-template\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"8m20s\"},{\"order\":0,\"remediationTemplate\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediationTemplate\",\"name\":\"selfnoderemediationtemplate-sample\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"1m0s\"}]},\"status\":{\"unhealthyNodes\":[{\"name\":\"worker-0-0\",\"remediations\":[{\"resource\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediation\",\"name\":\"worker-0-0\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"c60d8235-77ba-41f8-964a-4ff9016784a8\"},\"started\":\"2025-11-16T10:56:06Z\",\"templateName\":\"mdr-template\",\"timedOut\":\"2025-11-16T11:04:26Z\"},{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-0-s79k4\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"c9d21434-2f9f-4654-8868-f8cddc87be5e\"},\"started\":\"2025-11-16T11:04:26Z\",\"templateName\":\"selfnoderemediationtemplate-sample\",\"timedOut\":\"2025-11-16T11:05:26Z\"}]}]}}"} 2025-11-16T11:05:26.53586191Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "1s"} 2025-11-16T11:05:26.535972106Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:05:26.543422297Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-11-16T11:05:26.550304148Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:05:26Z"} 2025-11-16T11:05:26.550335803Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:05:26.550341877Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:05:26.55034754Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:05:26Z" level=info msg="invalidating lease" time="2025-11-16T11:05:26Z" level=info msg="getting lease" 2025-11-16T11:05:26.55406422Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:05:26Z" level=info msg="invalidating lease" time="2025-11-16T11:05:26Z" level=info msg="getting lease" 2025-11-16T11:05:26.557639434Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T11:05:26.557665048Z ERROR controllers.NodeHealthCheck Remediation timed out, and no template left to try {"NodeHealthCheck name": "nhc-mdr-snr", "error": "didn't find a template to use for NHC nhc-mdr-snr and node worker-0-0"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).remediate /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:536 github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:365 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-16T11:05:26.557803446Z DEBUG events [remediation] Remediation timed out, and no template left to try. didn't find a template to use for NHC nhc-mdr-snr and node worker-0-0 {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"578678"}, "reason": "NoTemplateLeft"} 2025-11-16T11:05:26.649395546Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T10:56:06Z","timedOut":"2025-11-16T11:04:26Z","templateName":"mdr-template"},{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T11:04:26Z","timedOut":"2025-11-16T11:05:26Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-16T11:05:26Z"}, "patch": "{\"spec\":{\"escalatingRemediations\":[{\"order\":-1,\"remediationTemplate\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediationTemplate\",\"name\":\"mdr-template\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"8m20s\"},{\"order\":0,\"remediationTemplate\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediationTemplate\",\"name\":\"selfnoderemediationtemplate-sample\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"1m0s\"}]}}"} 2025-11-16T11:05:26.855489623Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "47h51m39.350828308s"} 2025-11-16T11:05:27.536750923Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:05:27.542697926Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-11-16T11:05:27.549216136Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:05:26Z"} 2025-11-16T11:05:27.549239249Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:05:27.549245918Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:05:27.54925112Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:05:27Z" level=info msg="invalidating lease" time="2025-11-16T11:05:27Z" level=info msg="getting lease" 2025-11-16T11:05:27.552898696Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:05:27Z" level=info msg="invalidating lease" time="2025-11-16T11:05:27Z" level=info msg="getting lease" 2025-11-16T11:05:27.556310245Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T11:05:27.556326186Z ERROR controllers.NodeHealthCheck Remediation timed out, and no template left to try {"NodeHealthCheck name": "nhc-mdr-snr", "error": "didn't find a template to use for NHC nhc-mdr-snr and node worker-0-0"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).remediate /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:536 github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:365 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-16T11:05:27.556443633Z DEBUG events [remediation] Remediation timed out, and no template left to try. didn't find a template to use for NHC nhc-mdr-snr and node worker-0-0 {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"578678"}, "reason": "NoTemplateLeft"} 2025-11-16T11:05:27.560078075Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T10:56:06Z","timedOut":"2025-11-16T11:04:26Z","templateName":"mdr-template"},{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T11:04:26Z","timedOut":"2025-11-16T11:05:26Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-16T11:05:26Z"}, "patch": "{\"spec\":{\"escalatingRemediations\":[{\"order\":-1,\"remediationTemplate\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediationTemplate\",\"name\":\"mdr-template\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"8m20s\"},{\"order\":0,\"remediationTemplate\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediationTemplate\",\"name\":\"selfnoderemediationtemplate-sample\",\"namespace\":\"openshift-workload-availability\"},\"timeout\":\"1m0s\"}]}}"} 2025-11-16T11:05:27.765375627Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "47h51m38.44018186s"} 2025-11-16T11:10:27.462462757Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-mdr-snr"} 2025-11-16T11:10:27.46253371Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-mdr-snr"} 2025-11-16T11:10:27.462605802Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:10:27.472283003Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "False", "duration left": "29.527719184s"} 2025-11-16T11:10:27.482286608Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:05:26Z"} 2025-11-16T11:10:27.482351955Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:10:27.482362975Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:10:27.482371753Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:10:27Z" level=info msg="invalidating lease" time="2025-11-16T11:10:27Z" level=info msg="getting lease" 2025-11-16T11:10:27.582322968Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:10:27Z" level=info msg="invalidating lease" time="2025-11-16T11:10:27Z" level=info msg="getting lease" 2025-11-16T11:10:27.585728641Z INFO controllers.NodeHealthCheck Ignoring node, because it was unhealthy, and is likely to be unhealthy again. {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T11:10:27.585959166Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "30.527719184s"} 2025-11-16T11:10:28.680204811Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-mdr-snr"} 2025-11-16T11:10:28.680263988Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-mdr-snr"} 2025-11-16T11:10:28.680315931Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:10:28.701340997Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:05:26Z"} 2025-11-16T11:10:28.701383645Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:10:28.701392196Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:10:28.701399829Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T11:10:28.70571115Z INFO controllers.NodeHealthCheck.resource manager setting a delay for node getting healthy {"node name": "worker-0-0", "delay in seconds": 5} 2025-11-16T11:10:28.7144101Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:10:28.714463887Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:10:28.714474922Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:10:28.714479342Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:10:28.714824053Z INFO controllers.NodeHealthCheck.resource manager setting a delay for node getting healthy {"node name": "worker-0-0", "delay in seconds": 5} 2025-11-16T11:10:28.720330673Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-11-16T11:10:28.720423464Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:10:28.720482045Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-11-16T11:10:28.720494942Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:10:28.72050003Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} time="2025-11-16T11:10:28Z" level=info msg="invalidating lease" time="2025-11-16T11:10:28Z" level=info msg="getting lease" 2025-11-16T11:10:28.800540334Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:10:28Z" level=info msg="invalidating lease" time="2025-11-16T11:10:28Z" level=info msg="getting lease" 2025-11-16T11:10:28.804548747Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-0","uid":"c60d8235-77ba-41f8-964a-4ff9016784a8","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T10:56:06Z","timedOut":"2025-11-16T11:04:26Z","templateName":"mdr-template"},{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-11-16T11:04:26Z","timedOut":"2025-11-16T11:05:26Z","templateName":"selfnoderemediationtemplate-sample"}],"conditionsHealthyTimestamp":"2025-11-16T11:10:28Z","healthyDelayed":true}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-16T11:05:27Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"conditionsHealthyTimestamp\":\"2025-11-16T11:10:28Z\",\"healthyDelayed\":true,\"name\":\"worker-0-0\",\"remediations\":[{\"resource\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediation\",\"name\":\"worker-0-0\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"c60d8235-77ba-41f8-964a-4ff9016784a8\"},\"started\":\"2025-11-16T10:56:06Z\",\"templateName\":\"mdr-template\",\"timedOut\":\"2025-11-16T11:04:26Z\"},{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-0-s79k4\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"c9d21434-2f9f-4654-8868-f8cddc87be5e\"},\"started\":\"2025-11-16T11:04:26Z\",\"templateName\":\"selfnoderemediationtemplate-sample\",\"timedOut\":\"2025-11-16T11:05:26Z\"}]}]}}"} 2025-11-16T11:10:29.01062082Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "6s"} 2025-11-16T11:10:29.010697158Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:10:29.024740596Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:05:26Z"} 2025-11-16T11:10:29.024780744Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:10:29.02478919Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:10:29.024795557Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T11:10:29.126234966Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 3.873767713} 2025-11-16T11:10:29.126261857Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 3.873738993} 2025-11-16T11:10:29.126269378Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:10:29Z" level=info msg="invalidating lease" time="2025-11-16T11:10:29Z" level=info msg="getting lease" 2025-11-16T11:10:29.129988402Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:10:29Z" level=info msg="invalidating lease" time="2025-11-16T11:10:29Z" level=info msg="getting lease" 2025-11-16T11:10:29.133701266Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "4.873738993s"} 2025-11-16T11:10:34.007629841Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:10:34.021810907Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:05:26Z"} 2025-11-16T11:10:34.021843839Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T10:56:06Z"} 2025-11-16T11:10:34.021849476Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-11-16T11:04:26Z"} 2025-11-16T11:10:34.02185418Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:10:34Z" level=info msg="invalidating lease" time="2025-11-16T11:10:34Z" level=info msg="getting lease" 2025-11-16T11:10:34.025240162Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:10:34Z" level=info msg="invalidating lease" time="2025-11-16T11:10:34Z" level=info msg="getting lease" 2025-11-16T11:10:34.028850361Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T11:10:34.032584909Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-0"} 2025-11-16T11:10:34.039170601Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:10:34.039196568Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:10:34.039206564Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:10:34.03921238Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:10:34.039391405Z INFO controllers.NodeHealthCheck.resource manager deleted remediation CR {"name": "worker-0-0-s79k4"} 2025-11-16T11:10:34.03940375Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-0"} 2025-11-16T11:10:34.039406908Z DEBUG events [remediation] Deleted remediation CR of kind SelfNodeRemediation with name worker-0-0-s79k4 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"580351"}, "reason": "RemediationRemoved"} 2025-11-16T11:10:34.048765255Z INFO controllers.NodeHealthCheck.resource manager deleted remediation CR {"name": "worker-0-0"} 2025-11-16T11:10:34.048927843Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:10:34.04895344Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-11-16T11:10:34.049035736Z DEBUG events [remediation] Deleted remediation CR of kind MachineDeletionRemediation with name worker-0-0 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"580351"}, "reason": "RemediationRemoved"} 2025-11-16T11:10:34.049123495Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-11-16T11:10:34.049202194Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:10:34.123646542Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:10:34Z" level=info msg="invalidating lease" time="2025-11-16T11:10:34Z" level=info msg="getting lease" 2025-11-16T11:10:34.126769266Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-11-16T11:10:34.130186524Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-0"} 2025-11-16T11:10:34.132399653Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:10:34Z" level=info msg="invalidating lease" time="2025-11-16T11:10:34Z" level=info msg="getting lease" 2025-11-16T11:10:34.224122385Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "17s"} 2025-11-16T11:10:35.073617793Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"5cc37621-86e0-4a9a-8751-641e5567058f","controller":false}} 2025-11-16T11:10:35.073650396Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-0-s79k4", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-16T11:10:35.073706621Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:10:35.087986139Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:10:35Z" level=info msg="invalidating lease" time="2025-11-16T11:10:35Z" level=info msg="getting lease" 2025-11-16T11:10:35.188622063Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-11-16T11:10:35Z" level=info msg="invalidating lease" time="2025-11-16T11:10:35Z" level=info msg="getting lease" 2025-11-16T11:10:35.291641725Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:10:35Z" level=info msg="invalidating lease" time="2025-11-16T11:10:35Z" level=info msg="getting lease" 2025-11-16T11:10:35.38998443Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":3,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-16T06:25:20Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-11-16T11:10:28Z"}, "patch": "{\"status\":{\"healthyNodes\":3,\"phase\":\"Enabled\",\"reason\":\"NHC is enabled, no ongoing remediation\",\"unhealthyNodes\":null}}"} 2025-11-16T11:10:35.599610597Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-11-16T11:10:45.04983417Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-11-16T11:10:45.061474831Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-11-16T11:10:45Z" level=info msg="invalidating lease" time="2025-11-16T11:10:45Z" level=info msg="getting lease" 2025-11-16T11:10:45.064620889Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-11-16T11:10:45Z" level=info msg="invalidating lease" time="2025-11-16T11:10:45Z" level=info msg="getting lease" 2025-11-16T11:10:45.067540096Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-11-16T11:10:45Z" level=info msg="invalidating lease" time="2025-11-16T11:10:45Z" level=info msg="getting lease" 2025-11-16T11:10:45.070425051Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "0s"} SNR logs: [root@cert-rhosp-02 ~]# oc logs pod/self-node-remediation-controller-manager-77c688fb7b-c6wjg Defaulted container "manager" out of: manager, kube-rbac-proxy 2025-11-16T06:18:55.503485577Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-11-16T06:18:55.503555745Z INFO setup Go OS/Arch: linux/amd64 2025-11-16T06:18:55.503558595Z INFO setup Operator Version: d80706e 2025-11-16T06:18:55.503560459Z INFO setup Git Commit: d80706e1c305b9b65e00dee1b7cb653dcb1705e7 2025-11-16T06:18:55.503562047Z INFO setup Build Date: 2025-11-03T07:49:14+00:00 2025-11-16T06:18:55.503563634Z INFO setup HTTP/2 for metrics and webhook server disabled 2025-11-16T06:18:55.511267648Z INFO utils-taints out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 33} 2025-11-16T06:18:55.511291985Z INFO utils-taints out of service taint strategy {"isGA": true, "k8sMajorVersion": 1, "k8sMinorVersion": 33} 2025-11-16T06:18:55.511297347Z INFO setup Starting as a manager that installs the daemonset 2025-11-16T06:18:55.511304175Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig"} 2025-11-16T06:18:55.511336535Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2025-11-16T06:18:55.511400051Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2025-11-16T06:18:55.511464481Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-11-16T06:18:55.511496199Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-11-16T06:18:55.511510829Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-11-16T06:18:55.511535825Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-11-16T06:18:55.511554689Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation"} 2025-11-16T06:18:55.511570176Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2025-11-16T06:18:55.511595444Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2025-11-16T06:18:55.511641898Z INFO setup starting manager 2025-11-16T06:18:55.511729676Z INFO controller-runtime.metrics Starting metrics server 2025-11-16T06:18:55.511816331Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": "127.0.0.1:8080", "secure": false} 2025-11-16T06:18:55.511851828Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-11-16T06:18:55.511866927Z INFO controller-runtime.webhook Starting webhook server I1116 06:18:55.511948 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/547f6cb6.medik8s.io... 2025-11-16T06:18:55.512061228Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-11-16T06:18:55.51214902Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-11-16T06:18:55.512207836Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} I1116 06:19:10.967028 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/547f6cb6.medik8s.io 2025-11-16T06:19:10.967263288Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1alpha1.SelfNodeRemediationConfig"} 2025-11-16T06:19:10.967293155Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1.DaemonSet"} 2025-11-16T06:19:10.967299003Z INFO Starting Controller {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig"} 2025-11-16T06:19:10.967546522Z INFO Starting EventSource {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "source": "kind source: *v1alpha1.SelfNodeRemediation"} 2025-11-16T06:19:10.967814778Z INFO Starting Controller {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation"} 2025-11-16T06:19:10.967066467Z DEBUG events self-node-remediation-controller-manager-77c688fb7b-c6wjg_ed6849d4-c982-4e6e-890f-aec643d933a9 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"547f6cb6.medik8s.io","uid":"f5705f4d-03f2-49b0-9b85-e664cb4492df","apiVersion":"coordination.k8s.io/v1","resourceVersion":"481415"}, "reason": "LeaderElection"} 2025-11-16T06:19:11.070669314Z INFO Starting workers {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "worker count": 1} 2025-11-16T06:19:11.070699871Z INFO Starting workers {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "worker count": 1} 2025-11-16T06:19:11.07090805Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-11-16T06:19:11.17362493Z INFO controllers.SelfNodeRemediationConfig Creating new certs 2025-11-16T06:19:14.50782855Z INFO controllers.SelfNodeRemediationConfig Storing certs in new secret 2025-11-16T06:19:14.512255885Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-11-16T06:19:14.513067992Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025-11-16T06:19:14.51309846Z INFO controllers.SelfNodeRemediationConfig snr didn't find old daemonset to be deleted 2025/11/16 06:19:14 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/11/16 06:19:14 does not exist, creating (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/11/16 06:19:14 successfully created (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025-11-16T06:19:14.525926254Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-11-16T06:19:14.52595545Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-11-16T06:19:14.525961446Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-11-16T06:19:14.526738815Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/11/16 06:19:14 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/11/16 06:19:14 update was successful 2025-11-16T06:25:20.617321636Z INFO selfnoderemediationtemplate-resource default {"name": "selfnoderemediationtemplate-sample"} 2025-11-16T11:04:26.751937741Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:26.751971166Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:26.75201829Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"578331"}, "reason": "RemediationStarted"} 2025-11-16T11:04:26.859407252Z INFO controllers.SelfNodeRemediation finalizer added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:26.859591443Z DEBUG events [remediation] Remediation process - successful adding finalizer {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"578335"}, "reason": "AddFinalizer"} 2025-11-16T11:04:26.86422137Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:26.864234969Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:26.876173456Z INFO controllers.SelfNodeRemediation NoExecute taint added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-16T10:55:35Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-16T10:55:41Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-11-16T10:56:06Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-16T11:04:26Z"}]} 2025-11-16T11:04:26.876408972Z INFO rebootDurationCalculator No SafeTimeToAssumeNodeRebootedSeconds specified, using calculated minimum safe reboot time {"calculated minimum time in seconds": 120} 2025-11-16T11:04:26.876440836Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}, "node name": "worker-0-0", "time": "2025-11-16 11:06:26.87643986 +0000 UTC m=+17251.400665943"} 2025-11-16T11:04:26.876639412Z DEBUG events [remediation] Remediation process - NoExecute taint added to the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-0","uid":"81d10fd3-71fd-47f7-b1d2-f9cbfde06a54","apiVersion":"v1","resourceVersion":"578338"}, "reason": "AddNoExecute"} 2025-11-16T11:04:26.876710214Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"578337"}, "reason": "UpdateTimeAssumedRebooted"} 2025-11-16T11:04:26.894848549Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:26.894870506Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:26.895253081Z INFO rebootDurationCalculator No SafeTimeToAssumeNodeRebootedSeconds specified, using calculated minimum safe reboot time {"calculated minimum time in seconds": 120} 2025-11-16T11:04:26.895267412Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}, "node name": "worker-0-0", "time": "2025-11-16 11:06:26.895266463 +0000 UTC m=+17251.419492550"} 2025-11-16T11:04:26.895505112Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"578337"}, "reason": "UpdateTimeAssumedRebooted"} 2025-11-16T11:04:26.908698896Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:26.90872283Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}, "node name": "worker-0-0", "time left": "2m0.091277829s"} 2025-11-16T11:04:27.909649996Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:04:27.909666916Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}, "node name": "worker-0-0", "time left": "1m59.090334941s"} 2025-11-16T11:05:26.243622387Z INFO controllers.SelfNodeRemediation NHC added the timed-out annotation, remediation will be stopped {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:05:26.243927609Z DEBUG events [remediation] NHC added the timed-out annotation, remediation will be stopped {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"578674"}, "reason": "RemediationStopped"} 2025-11-16T11:05:26.247996251Z INFO controllers.SelfNodeRemediation NHC added the timed-out annotation, remediation will be stopped {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:05:26.248216399Z DEBUG events [remediation] NHC added the timed-out annotation, remediation will be stopped {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"578676"}, "reason": "RemediationStopped"} 2025-11-16T11:06:27.005863666Z INFO controllers.SelfNodeRemediation NHC added the timed-out annotation, remediation will be stopped {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:06:27.006072216Z DEBUG events [remediation] NHC added the timed-out annotation, remediation will be stopped {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"578676"}, "reason": "RemediationStopped"} 2025-11-16T11:10:28.711118096Z INFO selfnoderemediation-resource validate update {"name": "worker-0-0-s79k4"} 2025-11-16T11:10:28.714293864Z INFO controllers.SelfNodeRemediation NHC added the timed-out annotation, remediation will be stopped {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:10:28.714463463Z DEBUG events [remediation] NHC added the timed-out annotation, remediation will be stopped {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"580347"}, "reason": "RemediationStopped"} 2025-11-16T11:10:34.039325957Z INFO controllers.SelfNodeRemediation Cleaning up a timed-out remediation which is deleted by NHC {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}, "remediation name": "worker-0-0-s79k4"} 2025-11-16T11:10:34.039378965Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:10:34.045520972Z DEBUG events [remediation] Remediation process - mark healthy remediated node as schedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-0","uid":"81d10fd3-71fd-47f7-b1d2-f9cbfde06a54","apiVersion":"v1","resourceVersion":"580513"}, "reason": "MarkNodeSchedulable"} 2025-11-16T11:10:35.054397093Z INFO controllers.SelfNodeRemediation Cleaning up a timed-out remediation which is deleted by NHC {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}, "remediation name": "worker-0-0-s79k4"} 2025-11-16T11:10:35.054457753Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:10:35.064001823Z INFO controllers.SelfNodeRemediation NoExecute taint removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}, "new taints": null} 2025-11-16T11:10:35.06433259Z DEBUG events [remediation] Remediation process - remove NoExecute taint from healthy remediated node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-0","uid":"81d10fd3-71fd-47f7-b1d2-f9cbfde06a54","apiVersion":"v1","resourceVersion":"580591"}, "reason": "RemoveNoExecuteTaint"} 2025-11-16T11:10:35.068030117Z INFO selfnoderemediation-resource validate update {"name": "worker-0-0-s79k4"} 2025-11-16T11:10:35.073577256Z INFO controllers.SelfNodeRemediation finalizer removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:10:35.073824245Z DEBUG events [remediation] Remediation process - remove finalizer from snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"580510"}, "reason": "RemoveFinalizer"} 2025-11-16T11:10:35.073860139Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-s79k4","uid":"c9d21434-2f9f-4654-8868-f8cddc87be5e","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"580510"}, "reason": "RemediationFinished"} 2025-11-16T11:10:35.075815658Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}} 2025-11-16T11:10:36.076450555Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-0-s79k4","namespace":"openshift-workload-availability"}}