[root@cert-rhosp-02 ~]# oc project openshift-workload-availability Already on project "openshift-workload-availability" on server "https://api.ocp-edge-cluster-0.qe.lab.redhat.com:6443". [root@cert-rhosp-02 ~]# oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.21.0-0.nightly-2026-02-12-134401 True False 7d8h Cluster version is 4.21.0-0.nightly-2026-02-12-134401 [root@cert-rhosp-02 ~]# oc get csv NAME DISPLAY VERSION REPLACES PHASE fence-agents-remediation.v0.7.0 Fence Agents Remediation Operator 0.7.0 fence-agents-remediation.v0.6.0 Succeeded node-healthcheck-operator.v0.11.0 Node Health Check Operator 0.11.0 node-healthcheck-operator.v0.10.1 Succeeded [root@cert-rhosp-02 ~]# oc get fartemplate -o yaml apiVersion: v1 items: [] kind: List metadata: resourceVersion: "" selfLink: "" [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 7d8h v1.34.2 master-0-1 Ready control-plane,master 7d8h v1.34.2 master-0-2 Ready control-plane,master 7d8h v1.34.2 worker-0-0 Ready worker 7d8h v1.34.2 worker-0-1 Ready worker 7d8h v1.34.2 worker-0-2 Ready worker 7d8h v1.34.2 [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep fence-agents-remediation-controller-manager) [root@cert-rhosp-02 ~]# echo $PODS pod/fence-agents-remediation-controller-manager-7f56bb9d9b-kq5hq pod/fence-agents-remediation-controller-manager-7f56bb9d9b-xhrdg [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/fence-agents-remediation-controller-manager-7f56bb9d9b-kq5hq "worker-0-0" == pod/fence-agents-remediation-controller-manager-7f56bb9d9b-xhrdg "worker-0-1" [root@cert-rhosp-02 ~]# oc get node worker-0-2 -o yaml | yq .spec.taints null [root@cert-rhosp-02 ~]# oc debug node/worker-0-2 -- chroot /host bash -c "date" W0223 13:42:10.624378 902626 warnings.go:70] would violate PodSecurity "restricted:latest": host namespaces (hostNetwork=true, hostPID=true), privileged (container "container-00" must not set securityContext.privileged=true), allowPrivilegeEscalation != false (container "container-00" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container "container-00" must set securityContext.capabilities.drop=["ALL"]), restricted volume types (volume "host" uses restricted volume type "hostPath"), runAsNonRoot != true (pod or container "container-00" must set securityContext.runAsNonRoot=true), runAsUser=0 (container "container-00" must not set runAsUser=0) Starting pod/worker-0-2-debug ... To use host binaries, run `chroot /host` Mon Feb 23 18:42:10 UTC 2026 Removing debug pod ... [root@cert-rhosp-02 ~]# oc debug node/worker-0-2 -- chroot /host bash -c "systemctl stop kubelet" W0223 13:42:49.002054 903024 warnings.go:70] would violate PodSecurity "restricted:latest": host namespaces (hostNetwork=true, hostPID=true), privileged (container "container-00" must not set securityContext.privileged=true), allowPrivilegeEscalation != false (container "container-00" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container "container-00" must set securityContext.capabilities.drop=["ALL"]), restricted volume types (volume "host" uses restricted volume type "hostPath"), runAsNonRoot != true (pod or container "container-00" must set securityContext.runAsNonRoot=true), runAsUser=0 (container "container-00" must not set runAsUser=0) Starting pod/worker-0-2-debug ... To use host binaries, run `chroot /host` [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 7d11h v1.34.2 master-0-1 Ready control-plane,master 7d11h v1.34.2 master-0-2 Ready control-plane,master 7d11h v1.34.2 worker-0-0 Ready worker 7d11h v1.34.2 worker-0-1 Ready worker 7d11h v1.34.2 worker-0-2 NotReady worker 7d11h v1.34.2 [root@cert-rhosp-02 ~]# oc get node worker-0-2 -o yaml | yq .spec.taints - effect: NoSchedule key: node.kubernetes.io/unreachable timeAdded: "2026-02-23T18:43:42Z" - effect: NoExecute key: node.kubernetes.io/unreachable timeAdded: "2026-02-23T18:43:42Z" [root@cert-rhosp-02 ~]# oc get events -A --field-selector involvedObject.name=worker-0-2 NAMESPACE LAST SEEN TYPE REASON OBJECT MESSAGE default 2m19s Normal NodeNotReady node/worker-0-2 Node worker-0-2 status is now: NodeNotReady [root@cert-rhosp-02 ~]# cat far_nhc_nodesecret.yaml apiVersion: v1 kind: Secret stringData: --password: password metadata: name: worker-0-cred namespace: openshift-workload-availability --- apiVersion: v1 kind: Secret stringData: --password: password metadata: name: worker-1-cred namespace: openshift-workload-availability type: Opaque --- apiVersion: v1 kind: Secret stringData: --password: password metadata: name: worker-2-cred namespace: openshift-workload-availability type: Opaque --- apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate metadata: name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability spec: template: spec: agent: fence_ipmilan retrycount: 5 retryinterval: 10s timeout: 300s nodeparameters: '--ipport': master-0-0: '6230' master-0-1: '6231' master-0-2: '6232' worker-0-0: '6233' worker-0-1: '6234' worker-0-2: '6235' sharedparameters: '--action': "reboot" '--lanplus': '' '--ip': 192.168.123.1 '--username': admin nodeSecretNames: worker-0-0: worker-0-cred worker-0-1: worker-1-cred worker-0-2: worker-2-cred [root@cert-rhosp-02 ~]# oc apply -f far_nhc_nodesecret.yaml secret/worker-0-cred created secret/worker-1-cred created secret/worker-2-cred created fenceagentsremediationtemplate.fence-agents-remediation.medik8s.io/fenceagentsremediationtemplate-test created [root@cert-rhosp-02 ~]# oc apply -f far_nhc_nodesecret.yaml secret/worker-0-cred configured secret/worker-1-cred configured secret/worker-2-cred configured fenceagentsremediationtemplate.fence-agents-remediation.medik8s.io/fenceagentsremediationtemplate-test configured nodehealthcheck.remediation.medik8s.io/nhc-far-worker created [root@cert-rhosp-02 ~]# oc get nhc -oyaml apiVersion: v1 items: - apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","metadata":{"annotations":{},"name":"nhc-far-worker"},"spec":{"maxUnhealthy":2,"remediationTemplate":{"apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","kind":"FenceAgentsRemediationTemplate","name":"fenceagentsremediationtemplate-test","namespace":"openshift-workload-availability"},"selector":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"DoesNotExist","values":[]},{"key":"node-role.kubernetes.io/master","operator":"DoesNotExist","values":[]}]},"unhealthyConditions":[{"duration":"30s","status":"False","type":"Ready"},{"duration":"30s","status":"Unknown","type":"Ready"}]}} creationTimestamp: "2026-02-23T19:00:28Z" generation: 1 name: nhc-far-worker resourceVersion: "3893542" uid: 306d174f-61fc-4c33-8455-1bfd86b62596 spec: maxUnhealthy: 2 remediationTemplate: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/control-plane operator: DoesNotExist values: [] - key: node-role.kubernetes.io/master operator: DoesNotExist values: [] unhealthyConditions: - duration: 30s status: "False" type: Ready - duration: 30s status: Unknown type: Ready status: conditions: - lastTransitionTime: "2026-02-23T19:00:28Z" message: No issues found, NodeHealthCheck is enabled. reason: NodeHealthCheckEnabled status: "False" type: Disabled healthyNodes: 2 lastUpdateTime: "2026-02-23T19:00:29Z" observedNodes: 3 phase: Remediating reason: NHC is remediating 1 nodes unhealthyNodes: - name: worker-0-2 remediations: - resource: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediation name: worker-0-2-7wb76 namespace: openshift-workload-availability uid: b7dc8183-0dbc-4088-be66-73629ef54156 started: "2026-02-23T19:00:29Z" templateName: fenceagentsremediationtemplate-test kind: List metadata: resourceVersion: "" selfLink: "" [root@cert-rhosp-02 ~]# oc get far -oyaml apiVersion: v1 items: - apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediation metadata: annotations: remediation.medik8s.io/node-name: worker-0-2 remediation.medik8s.io/template-name: fenceagentsremediationtemplate-test creationTimestamp: "2026-02-23T19:00:29Z" finalizers: - fence-agents-remediation.medik8s.io/far-finalizer generateName: worker-0-2- generation: 1 labels: app.kubernetes.io/part-of: node-healthcheck-controller name: worker-0-2-7wb76 namespace: openshift-workload-availability ownerReferences: - apiVersion: remediation.medik8s.io/v1alpha1 controller: false kind: NodeHealthCheck name: nhc-far-worker uid: 306d174f-61fc-4c33-8455-1bfd86b62596 resourceVersion: "3893720" uid: b7dc8183-0dbc-4088-be66-73629ef54156 spec: agent: fence_ipmilan nodeSecretNames: worker-0-0: worker-0-cred worker-0-1: worker-1-cred worker-0-2: worker-2-cred nodeparameters: --ipport: master-0-0: "6230" master-0-1: "6231" master-0-2: "6232" worker-0-0: "6233" worker-0-1: "6234" worker-0-2: "6235" remediationStrategy: ResourceDeletion retrycount: 5 retryinterval: 10s sharedparameters: --action: reboot --ip: 192.168.123.1 --lanplus: "" --username: admin timeout: 5m0s status: conditions: - lastTransitionTime: "2026-02-23T19:00:35Z" message: The unhealthy node was fully remediated (it was tainted, fenced using the fence agent and all the node resources have been deleted) reason: RemediationFinishedSuccessfully status: "False" type: Processing - lastTransitionTime: "2026-02-23T19:00:34Z" message: FAR taint was added and the fence agent command has been created and executed successfully reason: FenceAgentSucceeded status: "True" type: FenceAgentActionSucceeded - lastTransitionTime: "2026-02-23T19:00:35Z" message: The unhealthy node was fully remediated (it was tainted, fenced using the fence agent and all the node resources have been deleted) reason: RemediationFinishedSuccessfully status: "True" type: Succeeded lastUpdateTime: "2026-02-23T19:00:35Z" kind: List metadata: resourceVersion: "" selfLink: "" [root@cert-rhosp-02 ~]# oc get node worker-0-2 -o yaml | yq .spec.taints - effect: NoSchedule key: node.kubernetes.io/unreachable timeAdded: "2026-02-23T18:43:42Z" - effect: NoExecute key: node.kubernetes.io/unreachable timeAdded: "2026-02-23T18:43:42Z" - effect: NoSchedule key: remediation.medik8s.io/fence-agents-remediation timeAdded: "2026-02-23T19:00:29Z" [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 7d12h v1.34.2 master-0-1 Ready control-plane,master 7d12h v1.34.2 master-0-2 Ready control-plane,master 7d12h v1.34.2 worker-0-0 Ready worker 7d11h v1.34.2 worker-0-1 Ready worker 7d11h v1.34.2 worker-0-2 Ready worker 7d11h v1.34.2 [root@cert-rhosp-02 ~]# oc get far No resources found in openshift-workload-availability namespace. [root@cert-rhosp-02 ~]# oc get events -A --field-selector involvedObject.name=worker-0-2 NAMESPACE LAST SEEN TYPE REASON OBJECT MESSAGE default 19m Normal NodeNotReady node/worker-0-2 Node worker-0-2 status is now: NodeNotReady default 102s Normal RemoveRemediationTaint node/worker-0-2 [remediation] Remediation taint was removed default 3m Normal AddRemediationTaint node/worker-0-2 [remediation] Remediation taint was added default 2m55s Normal DeleteResources node/worker-0-2 [remediation] Manually delete pods from the unhealthy node default 2m54s Normal NodeRemediationCompleted node/worker-0-2 [remediation] Unhealthy node remediation was completed default 104s Normal Starting node/worker-0-2 Starting kubelet. default 104s Normal NodeHasSufficientMemory node/worker-0-2 Node worker-0-2 status is now: NodeHasSufficientMemory default 104s Normal NodeHasNoDiskPressure node/worker-0-2 Node worker-0-2 status is now: NodeHasNoDiskPressure default 104s Normal NodeHasSufficientPID node/worker-0-2 Node worker-0-2 status is now: NodeHasSufficientPID default 104s Normal NodeAllocatableEnforced node/worker-0-2 Updated Node Allocatable limit across pods default 103s Warning Rebooted node/worker-0-2 Node worker-0-2 has been rebooted, boot id: d79aca2d-a0a1-4ad8-8053-aa6e4a5f9079 openshift-machine-config-operator 77s Normal Uncordon node/worker-0-2 Update completed for config rendered-worker-fe64d0bf10634489a4afb88e57b2cbc7 and node has been uncordoned openshift-machine-config-operator 77s Normal NodeDone node/worker-0-2 Setting node worker-0-2, currentConfig rendered-worker-fe64d0bf10634489a4afb88e57b2cbc7 to Done openshift-machine-config-operator 77s Normal ConfigDriftMonitorStarted node/worker-0-2 Config Drift Monitor started, watching against rendered-worker-fe64d0bf10634489a4afb88e57b2cbc7 [root@cert-rhosp-02 ~]# oc get node worker-0-2 -o yaml | yq .spec.taints null [root@cert-rhosp-02 ~]# cat far_nhc_nodesecret.yaml apiVersion: v1 kind: Secret stringData: --password: password metadata: name: worker-0-cred namespace: openshift-workload-availability --- apiVersion: v1 kind: Secret stringData: --password: password metadata: name: worker-1-cred namespace: openshift-workload-availability type: Opaque --- apiVersion: v1 kind: Secret stringData: --password: password metadata: name: worker-2-cred namespace: openshift-workload-availability type: Opaque --- apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate metadata: name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability spec: template: spec: agent: fence_ipmilan retrycount: 5 retryinterval: 10s timeout: 300s remediationStrategy: OutOfServiceTaint nodeparameters: '--ipport': master-0-0: '6230' master-0-1: '6231' master-0-2: '6232' worker-0-0: '6233' worker-0-1: '6234' worker-0-2: '6235' sharedparameters: '--action': "reboot" '--lanplus': '' '--ip': 192.168.123.1 '--username': admin nodeSecretNames: worker-0-0: worker-0-cred worker-0-1: worker-1-cred worker-0-2: worker-2-cred --- apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: name: nhc-far-worker spec: maxUnhealthy: 2 remediationTemplate: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/control-plane operator: DoesNotExist values: [] - key: node-role.kubernetes.io/master operator: DoesNotExist values: [] unhealthyConditions: - duration: 30s status: 'False' type: Ready - duration: 30s status: Unknown type: Ready [root@cert-rhosp-02 ~]# oc apply -f far_nhc_nodesecret.yaml secret/worker-0-cred configured secret/worker-1-cred configured secret/worker-2-cred configured fenceagentsremediationtemplate.fence-agents-remediation.medik8s.io/fenceagentsremediationtemplate-test configured nodehealthcheck.remediation.medik8s.io/nhc-far-worker unchanged [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 7d12h v1.34.2 master-0-1 Ready control-plane,master 7d12h v1.34.2 master-0-2 Ready control-plane,master 7d12h v1.34.2 worker-0-0 Ready worker 7d11h v1.34.2 worker-0-1 Ready worker 7d11h v1.34.2 worker-0-2 Ready worker 7d11h v1.34.2 [root@cert-rhosp-02 ~]# oc debug node/worker-0-2 -- chroot /host bash -c "date" W0223 14:09:46.467025 913005 warnings.go:70] would violate PodSecurity "restricted:latest": host namespaces (hostNetwork=true, hostPID=true), privileged (container "container-00" must not set securityContext.privileged=true), allowPrivilegeEscalation != false (container "container-00" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container "container-00" must set securityContext.capabilities.drop=["ALL"]), restricted volume types (volume "host" uses restricted volume type "hostPath"), runAsNonRoot != true (pod or container "container-00" must set securityContext.runAsNonRoot=true), runAsUser=0 (container "container-00" must not set runAsUser=0) Starting pod/worker-0-2-debug ... To use host binaries, run `chroot /host` ^[[AMon Feb 23 19:09:46 UTC 2026 Removing debug pod ... [root@cert-rhosp-02 ~]# oc debug node/worker-0-2 -- chroot /host bash -c "systemctl stop kubelet" W0223 14:09:52.333479 913048 warnings.go:70] would violate PodSecurity "restricted:latest": host namespaces (hostNetwork=true, hostPID=true), privileged (container "container-00" must not set securityContext.privileged=true), allowPrivilegeEscalation != false (container "container-00" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container "container-00" must set securityContext.capabilities.drop=["ALL"]), restricted volume types (volume "host" uses restricted volume type "hostPath"), runAsNonRoot != true (pod or container "container-00" must set securityContext.runAsNonRoot=true), runAsUser=0 (container "container-00" must not set runAsUser=0) Starting pod/worker-0-2-debug ... To use host binaries, run `chroot /host` [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 7d12h v1.34.2 master-0-1 Ready control-plane,master 7d12h v1.34.2 master-0-2 Ready control-plane,master 7d12h v1.34.2 worker-0-0 Ready worker 7d11h v1.34.2 worker-0-1 Ready worker 7d11h v1.34.2 worker-0-2 NotReady worker 7d11h v1.34.2 [root@cert-rhosp-02 ~]# oc get node worker-0-2 -o yaml | yq .spec.taints - effect: NoSchedule key: node.kubernetes.io/unreachable timeAdded: "2026-02-23T19:10:37Z" - effect: NoExecute key: node.kubernetes.io/unreachable timeAdded: "2026-02-23T19:10:37Z" [root@cert-rhosp-02 ~]# oc get far NAME AGE worker-0-2-ptq2z 2s [root@cert-rhosp-02 ~]# oc get node worker-0-2 -o yaml | yq .spec.taints - effect: NoSchedule key: node.kubernetes.io/unreachable timeAdded: "2026-02-23T19:10:37Z" - effect: NoExecute key: node.kubernetes.io/unreachable timeAdded: "2026-02-23T19:10:37Z" - effect: NoSchedule key: remediation.medik8s.io/fence-agents-remediation timeAdded: "2026-02-23T19:11:08Z" - effect: NoExecute key: node.kubernetes.io/out-of-service timeAdded: "2026-02-23T19:11:13Z" value: nodeshutdown [root@cert-rhosp-02 ~]# oc get node worker-0-2 -o yaml | yq .spec.taints null [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 7d12h v1.34.2 master-0-1 Ready control-plane,master 7d12h v1.34.2 master-0-2 Ready control-plane,master 7d12h v1.34.2 worker-0-0 Ready worker 7d11h v1.34.2 worker-0-1 Ready worker 7d11h v1.34.2 worker-0-2 Ready worker 7d11h v1.34.2 [root@cert-rhosp-02 ~]# oc get node worker-0-2 -o yaml | yq .spec.taints null [root@cert-rhosp-02 ~]# oc get events -A --field-selector involvedObject.name=worker-0-2 NAMESPACE LAST SEEN TYPE REASON OBJECT MESSAGE default 2m33s Normal NodeNotReady node/worker-0-2 Node worker-0-2 status is now: NodeNotReady default 42s Normal RemoveOutOfServiceTaint node/worker-0-2 [remediation] The out-of-service taint was removed default 42s Normal RemoveRemediationTaint node/worker-0-2 [remediation] Remediation taint was removed default 2m2s Normal AddRemediationTaint node/worker-0-2 [remediation] Remediation taint was added default 12m Normal DeleteResources node/worker-0-2 [remediation] Manually delete pods from the unhealthy node default 117s Normal NodeRemediationCompleted node/worker-0-2 [remediation] Unhealthy node remediation was completed default 11m Normal Starting node/worker-0-2 Starting kubelet. default 11m Normal NodeHasSufficientMemory node/worker-0-2 Node worker-0-2 status is now: NodeHasSufficientMemory default 11m Normal NodeHasNoDiskPressure node/worker-0-2 Node worker-0-2 status is now: NodeHasNoDiskPressure default 11m Normal NodeHasSufficientPID node/worker-0-2 Node worker-0-2 status is now: NodeHasSufficientPID default 11m Normal NodeAllocatableEnforced node/worker-0-2 Updated Node Allocatable limit across pods default 11m Warning Rebooted node/worker-0-2 Node worker-0-2 has been rebooted, boot id: d79aca2d-a0a1-4ad8-8053-aa6e4a5f9079 default 117s Normal AddOutOfServiceTaint node/worker-0-2 [remediation] The out-of-service taint was added default 46s Normal Starting node/worker-0-2 Starting kubelet. default 46s Normal NodeHasSufficientMemory node/worker-0-2 Node worker-0-2 status is now: NodeHasSufficientMemory default 46s Normal NodeHasNoDiskPressure node/worker-0-2 Node worker-0-2 status is now: NodeHasNoDiskPressure default 46s Normal NodeHasSufficientPID node/worker-0-2 Node worker-0-2 status is now: NodeHasSufficientPID default 46s Normal NodeAllocatableEnforced node/worker-0-2 Updated Node Allocatable limit across pods default 45s Warning Rebooted node/worker-0-2 Node worker-0-2 has been rebooted, boot id: c451a4a3-7600-480a-a6c8-fd9a41837422 openshift-machine-config-operator 10m Normal Uncordon node/worker-0-2 Update completed for config rendered-worker-fe64d0bf10634489a4afb88e57b2cbc7 and node has been uncordoned openshift-machine-config-operator 10m Normal NodeDone node/worker-0-2 Setting node worker-0-2, currentConfig rendered-worker-fe64d0bf10634489a4afb88e57b2cbc7 to Done openshift-machine-config-operator 10m Normal ConfigDriftMonitorStarted node/worker-0-2 Config Drift Monitor started, watching against rendered-worker-fe64d0bf10634489a4afb88e57b2cbc7 openshift-machine-config-operator 30s Normal Uncordon node/worker-0-2 Update completed for config rendered-worker-fe64d0bf10634489a4afb88e57b2cbc7 and node has been uncordoned openshift-machine-config-operator 30s Normal NodeDone node/worker-0-2 Setting node worker-0-2, currentConfig rendered-worker-fe64d0bf10634489a4afb88e57b2cbc7 to Done openshift-machine-config-operator 30s Normal ConfigDriftMonitorStarted node/worker-0-2 Config Drift Monitor started, watching against rendered-worker-fe64d0bf10634489a4afb88e57b2cbc7 [root@cert-rhosp-02 ~]# oc logs pod/fence-agents-remediation-controller-manager-7f56bb9d9b-xhrdg 2026-02-23T15:15:13.699133216Z INFO setup Go Version: go1.25.3 (Red Hat 1.25.3-1.el9_7) X:strictfipsruntime 2026-02-23T15:15:13.699290887Z INFO setup Go OS/Arch: linux/amd64 2026-02-23T15:15:13.699300094Z INFO setup Operator Version: 10619ba 2026-02-23T15:15:13.699303382Z INFO setup Git Commit: 10619bab37f2c246d0044a467d7b220d177f4ba5 2026-02-23T15:15:13.69930635Z INFO setup Build Date: 2026-02-23T12:03:02+00:00 2026-02-23T15:15:13.699333972Z INFO setup HTTP/2 for webhooks disabled 2026-02-23T15:15:13.708302048Z INFO validation out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 34} 2026-02-23T15:15:13.708339248Z INFO setup out-of-service taint is supported on this cluster 2026-02-23T15:15:13.708449958Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation", "path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2026-02-23T15:15:13.7085615Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2026-02-23T15:15:13.708595693Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2026-02-23T15:15:13.708644628Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2026-02-23T15:15:13.708708602Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2026-02-23T15:15:13.708754652Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2026-02-23T15:15:13.708779975Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2026-02-23T15:15:13.708820291Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2026-02-23T15:15:13.708868663Z INFO setup starting manager 2026-02-23T15:15:13.709016506Z INFO controller-runtime.metrics Starting metrics server 2026-02-23T15:15:13.709098615Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-02-23T15:15:13.709151158Z INFO controller-runtime.webhook Starting webhook server 2026-02-23T15:15:13.709153522Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} I0223 15:15:13.709223 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/cb305759.medik8s.io... 2026-02-23T15:15:13.709476078Z INFO controller-runtime.certwatcher Updated current TLS certificate 2026-02-23T15:15:13.709552616Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-02-23T15:15:13.709624766Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} 2026-02-23T15:15:29.485389081Z DEBUG events fence-agents-remediation-controller-manager-7f56bb9d9b-xhrdg_2bdba2df-955b-4155-9007-7cb7508d9f06 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"cb305759.medik8s.io","uid":"ee55c474-b094-4e6c-8ae9-cefd48a17595","apiVersion":"coordination.k8s.io/v1","resourceVersion":"3826381"}, "reason": "LeaderElection"} I0223 15:15:29.485357 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/cb305759.medik8s.io 2026-02-23T15:15:29.4858596Z INFO Starting EventSource {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation", "source": "kind source: *v1alpha1.FenceAgentsRemediation"} 2026-02-23T15:15:29.485875043Z INFO Starting EventSource {"controller": "fenceagentsremediationtemplate", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediationTemplate", "source": "kind source: *v1alpha1.FenceAgentsRemediationTemplate"} 2026-02-23T15:15:29.595049494Z INFO Starting Controller {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation"} 2026-02-23T15:15:29.595081625Z INFO Starting workers {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation", "worker count": 1} 2026-02-23T15:15:29.59509899Z INFO Starting Controller {"controller": "fenceagentsremediationtemplate", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediationTemplate"} 2026-02-23T15:15:29.595101867Z INFO Starting workers {"controller": "fenceagentsremediationtemplate", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediationTemplate", "worker count": 1} 2026-02-23T15:15:29.595180902Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T15:15:29.595236065Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T15:15:29.696536385Z INFO controllers.FenceAgentsRemediation CR's deletion timestamp is not zero, and FAR finalizer exists {"CR Name": "worker-0-2"} 2026-02-23T15:15:29.696585611Z INFO controllers.FenceAgentsRemediation Removing out-of-service taint {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2"} 2026-02-23T15:15:29.703726442Z INFO taints Taint was removed {"taint effect": "NoExecute", "taint list": [{"key":"medik8s.io/fence-agents-remediation","effect":"NoExecute","timeAdded":"2026-02-23T14:58:16Z"}]} 2026-02-23T15:15:29.703908325Z INFO controllers.FenceAgentsRemediation out-of-service taint was removed {"Node Name": "worker-0-2"} 2026-02-23T15:15:29.704081491Z INFO controllers.FenceAgentsRemediation FAR remediation taint was removed {"Node Name": "worker-0-2"} 2026-02-23T15:15:29.704073265Z DEBUG events [remediation] The out-of-service taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"935ced32-7e99-4d30-af6f-2e31fc7f40c5","apiVersion":"v1","resourceVersion":"3825441"}, "reason": "RemoveOutOfServiceTaint"} 2026-02-23T15:15:29.704296135Z DEBUG events [remediation] Remediation taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"935ced32-7e99-4d30-af6f-2e31fc7f40c5","apiVersion":"v1","resourceVersion":"3825441"}, "reason": "RemoveRemediationTaint"} 2026-02-23T15:15:29.711204172Z INFO fenceagentsremediation-resource default {"name": "worker-0-2"} 2026-02-23T15:15:29.812262296Z INFO fenceagentsremediationtemplate-resource Removing SharedSecretName old default value as the secret does not exist {"secretName": "fence-agents-credentials-shared"} 2026-02-23T15:15:29.817542928Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-0"} 2026-02-23T15:15:29.817557531Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-0"} 2026-02-23T15:15:29.817560377Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-0"} 2026-02-23T15:15:29.817576682Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-0"} 2026-02-23T15:15:29.817579135Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-1"} 2026-02-23T15:15:29.817580989Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-1"} 2026-02-23T15:15:29.817582996Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-1"} 2026-02-23T15:15:29.817596211Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-1"} 2026-02-23T15:15:29.817598174Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-2"} 2026-02-23T15:15:29.817599812Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-2"} 2026-02-23T15:15:29.817601529Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-2"} 2026-02-23T15:15:29.817614486Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-2"} 2026-02-23T15:15:29.817620836Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-0"} 2026-02-23T15:15:29.817622793Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-0"} 2026-02-23T15:15:29.817624655Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-0"} 2026-02-23T15:15:29.817636931Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-0"} 2026-02-23T15:15:29.817641438Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-1"} 2026-02-23T15:15:29.817643626Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-1"} 2026-02-23T15:15:29.817645449Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-1"} 2026-02-23T15:15:29.817659871Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-1"} 2026-02-23T15:15:29.817664802Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-2"} 2026-02-23T15:15:29.81766685Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-2"} 2026-02-23T15:15:29.817668851Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-2"} 2026-02-23T15:15:29.817683435Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-2"} 2026-02-23T15:15:29.821790732Z INFO controllers.FenceAgentsRemediation Finalizer was removed {"CR Name": "worker-0-2"} 2026-02-23T15:15:29.821847976Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T15:15:29.821914871Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T15:15:29.821879611Z DEBUG events [remediation] Finalizer was removed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"18041ac4-c3cd-4153-a2af-bf33a9fbcd20","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3823585"}, "reason": "RemoveFinalizer"} 2026-02-23T15:15:29.821939807Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR was not found {"CR Name": "worker-0-2", "CR Namespace": "openshift-workload-availability"} 2026-02-23T15:15:29.821947212Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T18:57:44.03199715Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-1"} 2026-02-23T18:57:44.032025203Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-1"} 2026-02-23T18:57:44.032052694Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-1-cred", "parameter name": "--password"} 2026-02-23T18:57:44.032056319Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-1"} 2026-02-23T18:57:44.032072926Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-1"} 2026-02-23T18:57:44.032075585Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-2"} 2026-02-23T18:57:44.032078004Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-2"} 2026-02-23T18:57:44.032083266Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-2-cred", "parameter name": "--password"} 2026-02-23T18:57:44.032085603Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-2"} 2026-02-23T18:57:44.032101066Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-2"} 2026-02-23T18:57:44.032102987Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-0"} 2026-02-23T18:57:44.032104973Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-0"} 2026-02-23T18:57:44.032107021Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-0"} 2026-02-23T18:57:44.032120023Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-0"} 2026-02-23T18:57:44.032121946Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-1"} 2026-02-23T18:57:44.032123622Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-1"} 2026-02-23T18:57:44.032125436Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-1"} 2026-02-23T18:57:44.03213701Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-1"} 2026-02-23T18:57:44.032139079Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-2"} 2026-02-23T18:57:44.032140783Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-2"} 2026-02-23T18:57:44.032142538Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-2"} 2026-02-23T18:57:44.032157133Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-2"} 2026-02-23T18:57:44.03215919Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-0"} 2026-02-23T18:57:44.032161903Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-0"} 2026-02-23T18:57:44.032166585Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-0-cred", "parameter name": "--password"} 2026-02-23T18:57:44.03216877Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-0"} 2026-02-23T18:57:44.032181954Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-0"} 2026-02-23T18:57:44.035038705Z INFO controllers.FenceAgentsRemediationTemplate Begin FenceAgentsRemediationTemplate Reconcile 2026-02-23T18:57:44.03510378Z INFO controllers.FenceAgentsRemediationTemplate status validation skipped, StatusValidationSample is zero or undefined 2026-02-23T18:57:44.039769953Z INFO controllers.FenceAgentsRemediationTemplate Finish FenceAgentsRemediationTemplate Reconcile 2026-02-23T19:00:28.835715058Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-2"} 2026-02-23T19:00:28.83574439Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-2"} 2026-02-23T19:00:28.835770222Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-2-cred", "parameter name": "--password"} 2026-02-23T19:00:28.835775779Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-2"} 2026-02-23T19:00:28.835794061Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-2"} 2026-02-23T19:00:28.835796708Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-0"} 2026-02-23T19:00:28.835798447Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-0"} 2026-02-23T19:00:28.835801097Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-0"} 2026-02-23T19:00:28.835813296Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-0"} 2026-02-23T19:00:28.835815208Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-1"} 2026-02-23T19:00:28.835816885Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-1"} 2026-02-23T19:00:28.835819185Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-1"} 2026-02-23T19:00:28.835832639Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-1"} 2026-02-23T19:00:28.835834434Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-2"} 2026-02-23T19:00:28.835836127Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-2"} 2026-02-23T19:00:28.835837889Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-2"} 2026-02-23T19:00:28.835850462Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-2"} 2026-02-23T19:00:28.8358528Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-0"} 2026-02-23T19:00:28.835854416Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-0"} 2026-02-23T19:00:28.835859392Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-0-cred", "parameter name": "--password"} 2026-02-23T19:00:28.83586175Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-0"} 2026-02-23T19:00:28.835873312Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-0"} 2026-02-23T19:00:28.835875213Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-1"} 2026-02-23T19:00:28.835876815Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-1"} 2026-02-23T19:00:28.835882464Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-1-cred", "parameter name": "--password"} 2026-02-23T19:00:28.835884739Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-1"} 2026-02-23T19:00:28.835898078Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-1"} 2026-02-23T19:00:29.073690155Z INFO fenceagentsremediation-resource default {"name": ""} 2026-02-23T19:00:29.082865099Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:00:29.082931826Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:00:29.094579261Z INFO controllers.FenceAgentsRemediation Finalizer was added {"CR Name": "worker-0-2-7wb76"} 2026-02-23T19:00:29.094620547Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "True", "fenceAgentActionSucceededConditionStatus": "Unknown", "succeededConditionStatus": "Unknown", "reason": "RemediationStarted", "LastUpdateTime": "2026-02-23T19:00:29.094619427Z"} 2026-02-23T19:00:29.094656671Z DEBUG events [remediation] Remediation started {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-7wb76","uid":"b7dc8183-0dbc-4088-be66-73629ef54156","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3893543"}, "reason": "RemediationStarted"} 2026-02-23T19:00:29.094713435Z DEBUG events [remediation] Finalizer was added {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-7wb76","uid":"b7dc8183-0dbc-4088-be66-73629ef54156","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3893543"}, "reason": "AddFinalizer"} 2026-02-23T19:00:29.300555859Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:00:29.300664249Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:00:29.300674672Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:00:29.311609936Z INFO taints Taint was added {"taint effect": "NoSchedule", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2026-02-23T18:43:42Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-23T18:43:42Z"},{"key":"remediation.medik8s.io/fence-agents-remediation","effect":"NoSchedule","timeAdded":"2026-02-23T19:00:29Z"}]} 2026-02-23T19:00:29.311670406Z INFO controllers.FenceAgentsRemediation FAR remediation taint was added {"Node Name": "worker-0-2"} 2026-02-23T19:00:29.311710269Z INFO controllers.FenceAgentsRemediation Build fence agent command line {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2"} 2026-02-23T19:00:29.311716471Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-2-7wb76"} 2026-02-23T19:00:29.311720677Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-2"} 2026-02-23T19:00:29.311744766Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-2-cred", "parameter name": "--password"} 2026-02-23T19:00:29.311751888Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-2"} 2026-02-23T19:00:29.311787816Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-2-7wb76"} 2026-02-23T19:00:29.311800856Z INFO controllers.FenceAgentsRemediation Execute the fence agent {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "b7dc8183-0dbc-4088-be66-73629ef54156", "Parameters": ["--ipport","--password","--action","--ip","--lanplus","--username"]} 2026-02-23T19:00:29.312131884Z DEBUG events [remediation] Remediation taint was added {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"935ced32-7e99-4d30-af6f-2e31fc7f40c5","apiVersion":"v1","resourceVersion":"3893265"}, "reason": "AddRemediationTaint"} 2026-02-23T19:00:29.312154382Z DEBUG events [remediation] Fence agent was executed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-7wb76","uid":"b7dc8183-0dbc-4088-be66-73629ef54156","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3893544"}, "reason": "FenceAgentExecuted"} 2026-02-23T19:00:29.312162452Z INFO executer fence agent start {"uid": "b7dc8183-0dbc-4088-be66-73629ef54156", "fence_agent": "fence_ipmilan", "retryCount": 5, "retryInterval": "10s", "timeout": "5m0s"} 2026-02-23T19:00:29.319143598Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:00:29.319245545Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:00:29.319264457Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:00:29.319326338Z INFO controllers.FenceAgentsRemediation A Fence Agent is already running {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "b7dc8183-0dbc-4088-be66-73629ef54156"} 2026-02-23T19:00:29.324289025Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:00:34.251702366Z INFO executer command completed {"uid": "b7dc8183-0dbc-4088-be66-73629ef54156", "response": "Success: Rebooted\n", "errMessage": "", "err": null} 2026-02-23T19:00:34.251749382Z INFO executer fence agent done {"uid": "b7dc8183-0dbc-4088-be66-73629ef54156", "fence_agent": "fence_ipmilan", "stdout": "Success: Rebooted\n", "stderr": "", "err": null} 2026-02-23T19:00:34.251754975Z INFO executer updating status {"FAR uid": "b7dc8183-0dbc-4088-be66-73629ef54156"} 2026-02-23T19:00:34.251834168Z INFO executer Updating Status Condition {"processingConditionStatus": "", "fenceAgentActionSucceededConditionStatus": "True", "succeededConditionStatus": "", "reason": "FenceAgentSucceeded", "LastUpdateTime": "2026-02-23T19:00:34.251832946Z"} 2026-02-23T19:00:34.251918938Z DEBUG events [remediation] Fence agent was succeeded {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-7wb76","uid":"b7dc8183-0dbc-4088-be66-73629ef54156","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3893544"}, "reason": "FenceAgentSucceeded"} 2026-02-23T19:00:34.257681324Z INFO executer status updated {"FAR uid": "b7dc8183-0dbc-4088-be66-73629ef54156"} 2026-02-23T19:00:34.258196281Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:00:34.25822461Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:00:34.258315211Z INFO controllers.FenceAgentsRemediation Remediation strategy is ResourceDeletion which explicitly deletes resources - manually deleting workload {"Node Name": "worker-0-2-7wb76"} 2026-02-23T19:00:34.258441479Z DEBUG events [remediation] Manually delete pods from the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"935ced32-7e99-4d30-af6f-2e31fc7f40c5","apiVersion":"v1","resourceVersion":"3893548"}, "reason": "DeleteResources"} 2026-02-23T19:00:34.359849524Z INFO commons-resource starting to delete pods {"node name": "worker-0-2"} 2026-02-23T19:00:35.273831715Z INFO commons-resource done deleting pods {"node name": "worker-0-2"} 2026-02-23T19:00:35.273960753Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "False", "fenceAgentActionSucceededConditionStatus": "", "succeededConditionStatus": "True", "reason": "RemediationFinishedSuccessfully", "LastUpdateTime": "2026-02-23T19:00:35.273950806Z"} 2026-02-23T19:00:35.273989026Z INFO executer cancelling fence agent routine {"uid": "b7dc8183-0dbc-4088-be66-73629ef54156"} 2026-02-23T19:00:35.274005556Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR has completed to remediate the node {"Node Name": "worker-0-2"} 2026-02-23T19:00:35.27415106Z DEBUG events [remediation] Unhealthy node remediation was completed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"935ced32-7e99-4d30-af6f-2e31fc7f40c5","apiVersion":"v1","resourceVersion":"3893548"}, "reason": "NodeRemediationCompleted"} 2026-02-23T19:00:35.274211139Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-7wb76","uid":"b7dc8183-0dbc-4088-be66-73629ef54156","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3893572"}, "reason": "RemediationFinished"} 2026-02-23T19:00:35.279195701Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:00:35.279284656Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:00:35.279303788Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:00:35.284068097Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:01:47.379267357Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:01:47.379318035Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:01:47.379347378Z INFO controllers.FenceAgentsRemediation CR's deletion timestamp is not zero, and FAR finalizer exists {"CR Name": "worker-0-2-7wb76"} 2026-02-23T19:01:47.389224872Z INFO taints Taint was removed {"taint effect": "NoSchedule", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-23T18:43:42Z"}]} 2026-02-23T19:01:47.389276691Z INFO controllers.FenceAgentsRemediation FAR remediation taint was removed {"Node Name": "worker-0-2"} 2026-02-23T19:01:47.389589013Z DEBUG events [remediation] Remediation taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"935ced32-7e99-4d30-af6f-2e31fc7f40c5","apiVersion":"v1","resourceVersion":"3894080"}, "reason": "RemoveRemediationTaint"} 2026-02-23T19:01:47.40970083Z INFO controllers.FenceAgentsRemediation Finalizer was removed {"CR Name": "worker-0-2-7wb76"} 2026-02-23T19:01:47.409755967Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:01:47.409784288Z DEBUG events [remediation] Finalizer was removed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-7wb76","uid":"b7dc8183-0dbc-4088-be66-73629ef54156","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3894112"}, "reason": "RemoveFinalizer"} 2026-02-23T19:01:47.41044378Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:01:47.410509487Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR was not found {"CR Name": "worker-0-2-7wb76", "CR Namespace": "openshift-workload-availability"} 2026-02-23T19:01:47.410515442Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:09:11.370437192Z INFO fenceagentsremediationtemplate-resource default {"name": "fenceagentsremediationtemplate-test"} 2026-02-23T19:09:11.374684608Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-2"} 2026-02-23T19:09:11.374702117Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-2"} 2026-02-23T19:09:11.37470515Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-2"} 2026-02-23T19:09:11.374721524Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-2"} 2026-02-23T19:09:11.374725468Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-0"} 2026-02-23T19:09:11.374727404Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-0"} 2026-02-23T19:09:11.374875913Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-0-cred", "parameter name": "--password"} 2026-02-23T19:09:11.374886705Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-0"} 2026-02-23T19:09:11.374922629Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-0"} 2026-02-23T19:09:11.374929525Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-1"} 2026-02-23T19:09:11.374931693Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-1"} 2026-02-23T19:09:11.374948598Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-1-cred", "parameter name": "--password"} 2026-02-23T19:09:11.374954907Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-1"} 2026-02-23T19:09:11.375011659Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-1"} 2026-02-23T19:09:11.375018668Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-2"} 2026-02-23T19:09:11.375020643Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-2"} 2026-02-23T19:09:11.375026016Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-2-cred", "parameter name": "--password"} 2026-02-23T19:09:11.375029311Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-2"} 2026-02-23T19:09:11.375073527Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-2"} 2026-02-23T19:09:11.375080097Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-0"} 2026-02-23T19:09:11.37508226Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-0"} 2026-02-23T19:09:11.375084307Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-0"} 2026-02-23T19:09:11.375099231Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-0"} 2026-02-23T19:09:11.375104622Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-1"} 2026-02-23T19:09:11.375106749Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-1"} 2026-02-23T19:09:11.375108765Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-1"} 2026-02-23T19:09:11.375135744Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-1"} 2026-02-23T19:09:11.378509284Z INFO controllers.FenceAgentsRemediationTemplate Begin FenceAgentsRemediationTemplate Reconcile 2026-02-23T19:09:11.378575517Z INFO controllers.FenceAgentsRemediationTemplate status validation skipped, StatusValidationSample is zero or undefined 2026-02-23T19:09:11.384215615Z INFO controllers.FenceAgentsRemediationTemplate Finish FenceAgentsRemediationTemplate Reconcile 2026-02-23T19:11:08.442192586Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:11:08.442256358Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:11:08.447522188Z INFO fenceagentsremediation-resource default {"name": "worker-0-2-ptq2z"} 2026-02-23T19:11:08.45114053Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-0"} 2026-02-23T19:11:08.451162362Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-0"} 2026-02-23T19:11:08.451181205Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-0-cred", "parameter name": "--password"} 2026-02-23T19:11:08.451185047Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-0"} 2026-02-23T19:11:08.451202066Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-0"} 2026-02-23T19:11:08.45120464Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-1"} 2026-02-23T19:11:08.451206676Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-1"} 2026-02-23T19:11:08.451214107Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-1-cred", "parameter name": "--password"} 2026-02-23T19:11:08.451217848Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-1"} 2026-02-23T19:11:08.451231605Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-1"} 2026-02-23T19:11:08.451237472Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-2"} 2026-02-23T19:11:08.451239664Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-2"} 2026-02-23T19:11:08.451245503Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-2-cred", "parameter name": "--password"} 2026-02-23T19:11:08.451248465Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-2"} 2026-02-23T19:11:08.451261633Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-2"} 2026-02-23T19:11:08.451266575Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-0"} 2026-02-23T19:11:08.451268698Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-0"} 2026-02-23T19:11:08.451271049Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-0"} 2026-02-23T19:11:08.451281312Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-0"} 2026-02-23T19:11:08.4512862Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-1"} 2026-02-23T19:11:08.451288165Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-1"} 2026-02-23T19:11:08.451290219Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-1"} 2026-02-23T19:11:08.451309975Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-1"} 2026-02-23T19:11:08.451318755Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-2"} 2026-02-23T19:11:08.451320697Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-2"} 2026-02-23T19:11:08.451322742Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-2"} 2026-02-23T19:11:08.451417714Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-2"} 2026-02-23T19:11:08.454462525Z INFO controllers.FenceAgentsRemediation Finalizer was added {"CR Name": "worker-0-2-ptq2z"} 2026-02-23T19:11:08.454487911Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "True", "fenceAgentActionSucceededConditionStatus": "Unknown", "succeededConditionStatus": "Unknown", "reason": "RemediationStarted", "LastUpdateTime": "2026-02-23T19:11:08.454486582Z"} 2026-02-23T19:11:08.454545942Z DEBUG events [remediation] Remediation started {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-ptq2z","uid":"c9488d8b-e526-44fb-aeb5-297d5a697ab3","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3897329"}, "reason": "RemediationStarted"} 2026-02-23T19:11:08.454620161Z DEBUG events [remediation] Finalizer was added {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-ptq2z","uid":"c9488d8b-e526-44fb-aeb5-297d5a697ab3","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3897329"}, "reason": "AddFinalizer"} 2026-02-23T19:11:08.66034982Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:11:08.660457443Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:11:08.660467533Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:11:08.668137491Z INFO taints Taint was added {"taint effect": "NoSchedule", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2026-02-23T19:10:37Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-23T19:10:37Z"},{"key":"remediation.medik8s.io/fence-agents-remediation","effect":"NoSchedule","timeAdded":"2026-02-23T19:11:08Z"}]} 2026-02-23T19:11:08.668188985Z INFO controllers.FenceAgentsRemediation FAR remediation taint was added {"Node Name": "worker-0-2"} 2026-02-23T19:11:08.668263065Z INFO controllers.FenceAgentsRemediation Build fence agent command line {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2"} 2026-02-23T19:11:08.668269814Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-2-ptq2z"} 2026-02-23T19:11:08.668273978Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-2"} 2026-02-23T19:11:08.668299747Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-2-cred", "parameter name": "--password"} 2026-02-23T19:11:08.668307432Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-2"} 2026-02-23T19:11:08.668347115Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-2-ptq2z"} 2026-02-23T19:11:08.668362346Z INFO controllers.FenceAgentsRemediation Execute the fence agent {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "c9488d8b-e526-44fb-aeb5-297d5a697ab3", "Parameters": ["--ip","--lanplus","--ipport","--password","--username","--action"]} 2026-02-23T19:11:08.668632172Z DEBUG events [remediation] Remediation taint was added {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"935ced32-7e99-4d30-af6f-2e31fc7f40c5","apiVersion":"v1","resourceVersion":"3897088"}, "reason": "AddRemediationTaint"} 2026-02-23T19:11:08.668649717Z DEBUG events [remediation] Fence agent was executed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-ptq2z","uid":"c9488d8b-e526-44fb-aeb5-297d5a697ab3","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3897331"}, "reason": "FenceAgentExecuted"} 2026-02-23T19:11:08.668658837Z INFO executer fence agent start {"uid": "c9488d8b-e526-44fb-aeb5-297d5a697ab3", "fence_agent": "fence_ipmilan", "retryCount": 5, "retryInterval": "10s", "timeout": "5m0s"} 2026-02-23T19:11:08.674503919Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:11:08.674578817Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:11:08.67459004Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:11:08.67463846Z INFO controllers.FenceAgentsRemediation A Fence Agent is already running {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "c9488d8b-e526-44fb-aeb5-297d5a697ab3"} 2026-02-23T19:11:08.680454105Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:11:13.620211709Z INFO executer command completed {"uid": "c9488d8b-e526-44fb-aeb5-297d5a697ab3", "response": "Success: Rebooted\n", "errMessage": "", "err": null} 2026-02-23T19:11:13.620265983Z INFO executer fence agent done {"uid": "c9488d8b-e526-44fb-aeb5-297d5a697ab3", "fence_agent": "fence_ipmilan", "stdout": "Success: Rebooted\n", "stderr": "", "err": null} 2026-02-23T19:11:13.620271575Z INFO executer updating status {"FAR uid": "c9488d8b-e526-44fb-aeb5-297d5a697ab3"} 2026-02-23T19:11:13.620380946Z INFO executer Updating Status Condition {"processingConditionStatus": "", "fenceAgentActionSucceededConditionStatus": "True", "succeededConditionStatus": "", "reason": "FenceAgentSucceeded", "LastUpdateTime": "2026-02-23T19:11:13.620380315Z"} 2026-02-23T19:11:13.620767228Z DEBUG events [remediation] Fence agent was succeeded {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-ptq2z","uid":"c9488d8b-e526-44fb-aeb5-297d5a697ab3","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3897331"}, "reason": "FenceAgentSucceeded"} 2026-02-23T19:11:13.626165098Z INFO executer status updated {"FAR uid": "c9488d8b-e526-44fb-aeb5-297d5a697ab3"} 2026-02-23T19:11:13.62634856Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:11:13.626375767Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:11:13.626448771Z INFO controllers.FenceAgentsRemediation Remediation strategy is OutOfServiceTaint which implicitly deletes resources - adding out-of-service taint {"Node Name": "worker-0-2-ptq2z"} 2026-02-23T19:11:13.634425267Z INFO taints Taint was added {"taint effect": "NoExecute", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2026-02-23T19:10:37Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-23T19:10:37Z"},{"key":"remediation.medik8s.io/fence-agents-remediation","effect":"NoSchedule","timeAdded":"2026-02-23T19:11:08Z"},{"key":"node.kubernetes.io/out-of-service","value":"nodeshutdown","effect":"NoExecute","timeAdded":"2026-02-23T19:11:13Z"}]} 2026-02-23T19:11:13.634465068Z INFO controllers.FenceAgentsRemediation out-of-service taint was added {"Node Name": "worker-0-2"} 2026-02-23T19:11:13.634484822Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "False", "fenceAgentActionSucceededConditionStatus": "", "succeededConditionStatus": "True", "reason": "RemediationFinishedSuccessfully", "LastUpdateTime": "2026-02-23T19:11:13.634483779Z"} 2026-02-23T19:11:13.634493733Z INFO executer cancelling fence agent routine {"uid": "c9488d8b-e526-44fb-aeb5-297d5a697ab3"} 2026-02-23T19:11:13.63450077Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR has completed to remediate the node {"Node Name": "worker-0-2"} 2026-02-23T19:11:13.63458514Z DEBUG events [remediation] The out-of-service taint was added {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"935ced32-7e99-4d30-af6f-2e31fc7f40c5","apiVersion":"v1","resourceVersion":"3897333"}, "reason": "AddOutOfServiceTaint"} 2026-02-23T19:11:13.634622922Z DEBUG events [remediation] Unhealthy node remediation was completed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"935ced32-7e99-4d30-af6f-2e31fc7f40c5","apiVersion":"v1","resourceVersion":"3897333"}, "reason": "NodeRemediationCompleted"} 2026-02-23T19:11:13.634630355Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-ptq2z","uid":"c9488d8b-e526-44fb-aeb5-297d5a697ab3","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3897364"}, "reason": "RemediationFinished"} 2026-02-23T19:11:13.64077282Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:11:13.641012661Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:11:13.641049068Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:11:13.65136171Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:12:27.599350984Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:12:27.599416624Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:12:27.599456579Z INFO controllers.FenceAgentsRemediation CR's deletion timestamp is not zero, and FAR finalizer exists {"CR Name": "worker-0-2-ptq2z"} 2026-02-23T19:12:27.599460778Z INFO controllers.FenceAgentsRemediation Removing out-of-service taint {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2"} 2026-02-23T19:12:27.609009885Z INFO taints Taint was removed {"taint effect": "NoExecute", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-23T19:10:37Z"},{"key":"remediation.medik8s.io/fence-agents-remediation","effect":"NoSchedule","timeAdded":"2026-02-23T19:11:08Z"}]} 2026-02-23T19:12:27.609083095Z INFO controllers.FenceAgentsRemediation out-of-service taint was removed {"Node Name": "worker-0-2"} 2026-02-23T19:12:27.609244453Z DEBUG events [remediation] The out-of-service taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"935ced32-7e99-4d30-af6f-2e31fc7f40c5","apiVersion":"v1","resourceVersion":"3897750"}, "reason": "RemoveOutOfServiceTaint"} 2026-02-23T19:12:27.616034878Z INFO controllers.FenceAgentsRemediation Failed to remove taint from node due to node update, retrying... , {"node name": "worker-0-2", "taint key": "remediation.medik8s.io/fence-agents-remediation", "taint effect": "NoSchedule"} 2026-02-23T19:12:27.621837244Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:12:28.622648943Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:12:28.622726531Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T19:12:28.622786057Z INFO controllers.FenceAgentsRemediation CR's deletion timestamp is not zero, and FAR finalizer exists {"CR Name": "worker-0-2-ptq2z"} 2026-02-23T19:12:28.622792976Z INFO controllers.FenceAgentsRemediation Removing out-of-service taint {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2"} 2026-02-23T19:12:28.622815895Z INFO controllers.FenceAgentsRemediation out-of-service taint was removed {"Node Name": "worker-0-2"} 2026-02-23T19:12:28.622990028Z DEBUG events [remediation] The out-of-service taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"935ced32-7e99-4d30-af6f-2e31fc7f40c5","apiVersion":"v1","resourceVersion":"3897760"}, "reason": "RemoveOutOfServiceTaint"} 2026-02-23T19:12:28.631304937Z INFO taints Taint was removed {"taint effect": "NoSchedule", "taint list": null} 2026-02-23T19:12:28.631357609Z INFO controllers.FenceAgentsRemediation FAR remediation taint was removed {"Node Name": "worker-0-2"} 2026-02-23T19:12:28.631641661Z DEBUG events [remediation] Remediation taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"935ced32-7e99-4d30-af6f-2e31fc7f40c5","apiVersion":"v1","resourceVersion":"3897760"}, "reason": "RemoveRemediationTaint"} 2026-02-23T19:12:28.634759999Z INFO fenceagentsremediation-resource default {"name": "worker-0-2-ptq2z"} 2026-02-23T19:12:28.637554791Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-1"} 2026-02-23T19:12:28.637584128Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-1"} 2026-02-23T19:12:28.637589485Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-1"} 2026-02-23T19:12:28.63761245Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-1"} 2026-02-23T19:12:28.637616259Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-2"} 2026-02-23T19:12:28.637619836Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-2"} 2026-02-23T19:12:28.637623526Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-2"} 2026-02-23T19:12:28.637640581Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-2"} 2026-02-23T19:12:28.637643666Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-0"} 2026-02-23T19:12:28.637646339Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-0"} 2026-02-23T19:12:28.637667561Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-0-cred", "parameter name": "--password"} 2026-02-23T19:12:28.637672348Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-0"} 2026-02-23T19:12:28.637692195Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-0"} 2026-02-23T19:12:28.637700482Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-1"} 2026-02-23T19:12:28.637703625Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-1"} 2026-02-23T19:12:28.637713146Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-1-cred", "parameter name": "--password"} 2026-02-23T19:12:28.637717322Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-1"} 2026-02-23T19:12:28.637741422Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-1"} 2026-02-23T19:12:28.637749753Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-2"} 2026-02-23T19:12:28.637753028Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-2"} 2026-02-23T19:12:28.637764047Z INFO fenceagentsremediation-params found a value from secret {"secret name": "worker-2-cred", "parameter name": "--password"} 2026-02-23T19:12:28.637767825Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-2"} 2026-02-23T19:12:28.637787138Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-2"} 2026-02-23T19:12:28.637793865Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-0"} 2026-02-23T19:12:28.637796893Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-0"} 2026-02-23T19:12:28.637799782Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-0"} 2026-02-23T19:12:28.637821577Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-0"} 2026-02-23T19:12:28.646233756Z INFO controllers.FenceAgentsRemediation Finalizer was removed {"CR Name": "worker-0-2-ptq2z"} 2026-02-23T19:12:28.646294655Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T19:12:28.646323543Z DEBUG events [remediation] Finalizer was removed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-ptq2z","uid":"c9488d8b-e526-44fb-aeb5-297d5a697ab3","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3897753"}, "reason": "RemoveFinalizer"} 2026-02-23T19:12:28.648943867Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T19:12:28.649003475Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR was not found {"CR Name": "worker-0-2-ptq2z", "CR Namespace": "openshift-workload-availability"} 2026-02-23T19:12:28.649007951Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile