[root@cert-rhosp-02 ~]# oc project openshift-workload-availability Already on project "openshift-workload-availability" on server "https://api.ocp-edge-cluster-0.qe.lab.redhat.com:6443". [root@cert-rhosp-02 ~]# oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.21.0-0.nightly-2026-02-12-134401 True False 7d6h Cluster version is 4.21.0-0.nightly-2026-02-12-134401 [root@cert-rhosp-02 ~]# oc get csv NAME DISPLAY VERSION REPLACES PHASE fence-agents-remediation.v0.7.0 Fence Agents Remediation Operator 0.7.0 fence-agents-remediation.v0.6.0 Succeeded node-healthcheck-operator.v0.11.0 Node Health Check Operator 0.11.0 node-healthcheck-operator.v0.10.1 Succeeded [root@cert-rhosp-02 ~]# oc get fartemplate -o yaml apiVersion: v1 items: [] kind: List metadata: resourceVersion: "" selfLink: "" [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 7d7h v1.34.2 master-0-1 Ready control-plane,master 7d7h v1.34.2 master-0-2 Ready control-plane,master 7d7h v1.34.2 worker-0-0 Ready worker 7d6h v1.34.2 worker-0-1 Ready worker 7d6h v1.34.2 worker-0-2 Ready worker 7d6h v1.34.2 [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep fence-agents-remediation-controller-manager) [root@cert-rhosp-02 ~]# echo $PODS pod/fence-agents-remediation-controller-manager-6647b79dbb-2xpt9 pod/fence-agents-remediation-controller-manager-6647b79dbb-jlcbv [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/fence-agents-remediation-controller-manager-6647b79dbb-2xpt9 "worker-0-1" == pod/fence-agents-remediation-controller-manager-6647b79dbb-jlcbv "worker-0-2" [root@cert-rhosp-02 ~]# oc debug node/worker-0-0 -- chroot /host bash -c "date" W0223 09:11:54.255770 811580 warnings.go:70] would violate PodSecurity "restricted:latest": host namespaces (hostNetwork=true, hostPID=true), privileged (container "container-00" must not set securityContext.privileged=true), allowPrivilegeEscalation != false (container "container-00" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container "container-00" must set securityContext.capabilities.drop=["ALL"]), restricted volume types (volume "host" uses restricted volume type "hostPath"), runAsNonRoot != true (pod or container "container-00" must set securityContext.runAsNonRoot=true), runAsUser=0 (container "container-00" must not set runAsUser=0) Starting pod/worker-0-0-debug ... To use host binaries, run `chroot /host` Mon Feb 23 14:11:54 UTC 2026 Removing debug pod ... [root@cert-rhosp-02 ~]# oc debug node/worker-0-0 -- chroot /host bash -c "systemctl stop kubelet" W0223 09:12:03.476423 811630 warnings.go:70] would violate PodSecurity "restricted:latest": host namespaces (hostNetwork=true, hostPID=true), privileged (container "container-00" must not set securityContext.privileged=true), allowPrivilegeEscalation != false (container "container-00" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container "container-00" must set securityContext.capabilities.drop=["ALL"]), restricted volume types (volume "host" uses restricted volume type "hostPath"), runAsNonRoot != true (pod or container "container-00" must set securityContext.runAsNonRoot=true), runAsUser=0 (container "container-00" must not set runAsUser=0) Starting pod/worker-0-0-debug ... To use host binaries, run `chroot /host` [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 7d7h v1.34.2 master-0-1 Ready control-plane,master 7d7h v1.34.2 master-0-2 Ready control-plane,master 7d7h v1.34.2 worker-0-0 NotReady worker 7d6h v1.34.2 worker-0-1 Ready worker 7d6h v1.34.2 worker-0-2 Ready worker 7d6h v1.34.2 [root@cert-rhosp-02 ~]# vi far_nhc_sharedsecret.yaml [root@cert-rhosp-02 ~]# cat far_nhc_sharedsecret.yaml apiVersion: v1 kind: Secret stringData: '--password': password '--username': admin metadata: name: test-far-shared namespace: openshift-workload-availability type: Opaque --- apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate metadata: name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability spec: template: spec: agent: fence_ipmilan retrycount: 5 retryinterval: 10s timeout: 300s remediationStrategy: OutOfServiceTaint nodeparameters: '--ipport': master-0-0: '6230' master-0-1: '6231' master-0-2: '6232' worker-0-0: '6233' worker-0-1: '6234' worker-0-2: '6235' sharedparameters: '--action': reboot '--lanplus': '' '--ip': 192.168.123.1 sharedSecretName: test-far-shared --- apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: name: nhc-far-worker spec: maxUnhealthy: 2 remediationTemplate: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/control-plane operator: DoesNotExist values: [] - key: node-role.kubernetes.io/master operator: DoesNotExist values: [] unhealthyConditions: - duration: 30s status: 'False' type: Ready - duration: 30s status: Unknown type: Ready [root@cert-rhosp-02 ~]# oc apply -f far_nhc_sharedsecret.yaml secret/test-far-shared created fenceagentsremediationtemplate.fence-agents-remediation.medik8s.io/fenceagentsremediationtemplate-test created nodehealthcheck.remediation.medik8s.io/nhc-far-worker created [root@cert-rhosp-02 ~]# oc get fartemplate NAME AGE fenceagentsremediationtemplate-test 14s [root@cert-rhosp-02 ~]# oc get far NAME AGE worker-0-0-5bl95 17s [root@cert-rhosp-02 ~]# oc get far -oyaml apiVersion: v1 items: - apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediation metadata: annotations: remediation.medik8s.io/node-name: worker-0-0 remediation.medik8s.io/template-name: fenceagentsremediationtemplate-test creationTimestamp: "2026-02-23T14:19:08Z" finalizers: - fence-agents-remediation.medik8s.io/far-finalizer generateName: worker-0-0- generation: 1 labels: app.kubernetes.io/part-of: node-healthcheck-controller name: worker-0-0-5bl95 namespace: openshift-workload-availability ownerReferences: - apiVersion: remediation.medik8s.io/v1alpha1 controller: false kind: NodeHealthCheck name: nhc-far-worker uid: 157921c0-f33e-4131-a05a-9a397c42544c resourceVersion: "3806413" uid: 16ea6729-6e44-4703-887a-7a180c9d8942 spec: agent: fence_ipmilan nodeparameters: --ipport: master-0-0: "6230" master-0-1: "6231" master-0-2: "6232" worker-0-0: "6233" worker-0-1: "6234" worker-0-2: "6235" remediationStrategy: OutOfServiceTaint retrycount: 5 retryinterval: 10s sharedSecretName: test-far-shared sharedparameters: --action: reboot --ip: 192.168.123.1 --lanplus: "" timeout: 5m0s status: conditions: - lastTransitionTime: "2026-02-23T14:19:13Z" message: The unhealthy node was fully remediated (it was tainted, fenced using the fence agent and all the node resources have been deleted) reason: RemediationFinishedSuccessfully status: "False" type: Processing - lastTransitionTime: "2026-02-23T14:19:13Z" message: FAR taint was added and the fence agent command has been created and executed successfully reason: FenceAgentSucceeded status: "True" type: FenceAgentActionSucceeded - lastTransitionTime: "2026-02-23T14:19:13Z" message: The unhealthy node was fully remediated (it was tainted, fenced using the fence agent and all the node resources have been deleted) reason: RemediationFinishedSuccessfully status: "True" type: Succeeded lastUpdateTime: "2026-02-23T14:19:13Z" kind: List metadata: resourceVersion: "" selfLink: "" [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 7d7h v1.34.2 master-0-1 Ready control-plane,master 7d7h v1.34.2 master-0-2 Ready control-plane,master 7d7h v1.34.2 worker-0-0 Ready worker 7d6h v1.34.2 worker-0-1 Ready worker 7d6h v1.34.2 worker-0-2 Ready worker 7d6h v1.34.2 [root@cert-rhosp-02 ~]# oc get far No resources found in openshift-workload-availability namespace. [root@cert-rhosp-02 ~]# oc get fartemplate NAME AGE fenceagentsremediationtemplate-test 5m53s [root@cert-rhosp-02 ~]# oc logs pod/fence-agents-remediation-controller-manager-6647b79dbb-2xpt9 2026-02-23T14:08:56.351998408Z INFO setup Go Version: go1.25.3 (Red Hat 1.25.3-1.el9_7) X:strictfipsruntime 2026-02-23T14:08:56.352121664Z INFO setup Go OS/Arch: linux/amd64 2026-02-23T14:08:56.352124633Z INFO setup Operator Version: 10619ba 2026-02-23T14:08:56.352127073Z INFO setup Git Commit: 10619bab37f2c246d0044a467d7b220d177f4ba5 2026-02-23T14:08:56.352129066Z INFO setup Build Date: 2026-02-23T12:03:02+00:00 2026-02-23T14:08:56.352147951Z INFO setup HTTP/2 for webhooks disabled 2026-02-23T14:08:56.359062426Z INFO validation out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 34} 2026-02-23T14:08:56.359091133Z INFO setup out-of-service taint is supported on this cluster 2026-02-23T14:08:56.359151964Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation", "path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2026-02-23T14:08:56.359230085Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2026-02-23T14:08:56.359253099Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2026-02-23T14:08:56.359284935Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2026-02-23T14:08:56.359327524Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2026-02-23T14:08:56.359357277Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2026-02-23T14:08:56.359373627Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2026-02-23T14:08:56.359421806Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2026-02-23T14:08:56.359450515Z INFO setup starting manager 2026-02-23T14:08:56.359569152Z INFO controller-runtime.metrics Starting metrics server 2026-02-23T14:08:56.359621233Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-02-23T14:08:56.359682533Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-02-23T14:08:56.35972523Z INFO controller-runtime.webhook Starting webhook server I0223 14:08:56.359838 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/cb305759.medik8s.io... 2026-02-23T14:08:56.360089741Z INFO controller-runtime.certwatcher Updated current TLS certificate 2026-02-23T14:08:56.360169105Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-02-23T14:08:56.360765152Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} I0223 14:09:13.833549 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/cb305759.medik8s.io 2026-02-23T14:09:13.833607792Z DEBUG events fence-agents-remediation-controller-manager-6647b79dbb-2xpt9_ef4ea3cc-b6c4-4ea2-a548-c9c1d6d9e31c became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"cb305759.medik8s.io","uid":"ee55c474-b094-4e6c-8ae9-cefd48a17595","apiVersion":"coordination.k8s.io/v1","resourceVersion":"3802961"}, "reason": "LeaderElection"} 2026-02-23T14:09:13.833857613Z INFO Starting EventSource {"controller": "fenceagentsremediationtemplate", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediationTemplate", "source": "kind source: *v1alpha1.FenceAgentsRemediationTemplate"} 2026-02-23T14:09:13.833875484Z INFO Starting EventSource {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation", "source": "kind source: *v1alpha1.FenceAgentsRemediation"} 2026-02-23T14:09:13.94143304Z INFO Starting Controller {"controller": "fenceagentsremediationtemplate", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediationTemplate"} 2026-02-23T14:09:13.941502149Z INFO Starting workers {"controller": "fenceagentsremediationtemplate", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediationTemplate", "worker count": 1} 2026-02-23T14:09:13.94150523Z INFO Starting Controller {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation"} 2026-02-23T14:09:13.941523634Z INFO Starting workers {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation", "worker count": 1} 2026-02-23T14:19:07.784611118Z INFO fenceagentsremediationtemplate-resource default {"name": "fenceagentsremediationtemplate-test"} 2026-02-23T14:19:07.789079274Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-0"} 2026-02-23T14:19:07.789137325Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-0"} 2026-02-23T14:19:07.889413045Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--password"} 2026-02-23T14:19:07.889510182Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--username"} 2026-02-23T14:19:07.889544442Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-0"} 2026-02-23T14:19:07.889579863Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-0"} 2026-02-23T14:19:07.889601749Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-1"} 2026-02-23T14:19:07.88961291Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-1"} 2026-02-23T14:19:07.889634747Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--username"} 2026-02-23T14:19:07.889647283Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--password"} 2026-02-23T14:19:07.889681693Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-1"} 2026-02-23T14:19:07.88970391Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-1"} 2026-02-23T14:19:07.889715524Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-2"} 2026-02-23T14:19:07.889725727Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-2"} 2026-02-23T14:19:07.889740558Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--password"} 2026-02-23T14:19:07.889752144Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--username"} 2026-02-23T14:19:07.889767788Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-2"} 2026-02-23T14:19:07.889790169Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-2"} 2026-02-23T14:19:07.889801273Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-0"} 2026-02-23T14:19:07.889812051Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-0"} 2026-02-23T14:19:07.889826447Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--password"} 2026-02-23T14:19:07.889838213Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--username"} 2026-02-23T14:19:07.889855738Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-0"} 2026-02-23T14:19:07.889881227Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-0"} 2026-02-23T14:19:07.889896219Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-1"} 2026-02-23T14:19:07.889906759Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-1"} 2026-02-23T14:19:07.889919012Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--password"} 2026-02-23T14:19:07.889931863Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--username"} 2026-02-23T14:19:07.889950139Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-1"} 2026-02-23T14:19:07.889970402Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-1"} 2026-02-23T14:19:07.889981133Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "master-0-2"} 2026-02-23T14:19:07.889991242Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "master-0-2"} 2026-02-23T14:19:07.890003777Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--password"} 2026-02-23T14:19:07.890015435Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--username"} 2026-02-23T14:19:07.890029961Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "master-0-2"} 2026-02-23T14:19:07.890068207Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "master-0-2"} 2026-02-23T14:19:07.89370152Z INFO controllers.FenceAgentsRemediationTemplate Begin FenceAgentsRemediationTemplate Reconcile 2026-02-23T14:19:07.89376339Z INFO controllers.FenceAgentsRemediationTemplate status validation skipped, StatusValidationSample is zero or undefined 2026-02-23T14:19:07.898041012Z INFO controllers.FenceAgentsRemediationTemplate Finish FenceAgentsRemediationTemplate Reconcile 2026-02-23T14:19:08.148083786Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T14:19:08.148130477Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T14:19:08.258182443Z INFO controllers.FenceAgentsRemediation Finalizer was added {"CR Name": "worker-0-0-5bl95"} 2026-02-23T14:19:08.25821046Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "True", "fenceAgentActionSucceededConditionStatus": "Unknown", "succeededConditionStatus": "Unknown", "reason": "RemediationStarted", "LastUpdateTime": "2026-02-23T14:19:08.258209738Z"} 2026-02-23T14:19:08.258416998Z DEBUG events [remediation] Remediation started {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-5bl95","uid":"16ea6729-6e44-4703-887a-7a180c9d8942","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3806373"}, "reason": "RemediationStarted"} 2026-02-23T14:19:08.258446646Z DEBUG events [remediation] Finalizer was added {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-5bl95","uid":"16ea6729-6e44-4703-887a-7a180c9d8942","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3806373"}, "reason": "AddFinalizer"} 2026-02-23T14:19:08.264221232Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T14:19:08.26431162Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T14:19:08.264325039Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T14:19:08.272627511Z INFO taints Taint was added {"taint effect": "NoSchedule", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2026-02-23T14:12:48Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-23T14:12:48Z"},{"key":"remediation.medik8s.io/fence-agents-remediation","effect":"NoSchedule","timeAdded":"2026-02-23T14:19:08Z"}]} 2026-02-23T14:19:08.272705893Z INFO controllers.FenceAgentsRemediation FAR remediation taint was added {"Node Name": "worker-0-0"} 2026-02-23T14:19:08.272744948Z INFO controllers.FenceAgentsRemediation Build fence agent command line {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-0"} 2026-02-23T14:19:08.272750858Z INFO fenceagentsremediation-params BuildFenceAgentParams starting {"Node Name": "worker-0-0-5bl95"} 2026-02-23T14:19:08.272755138Z INFO fenceagentsremediation-params collectAllSecretParams start for node {"node": "worker-0-0"} 2026-02-23T14:19:08.272775211Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--username"} 2026-02-23T14:19:08.272778417Z INFO fenceagentsremediation-params found a value from secret {"secret name": "test-far-shared", "parameter name": "--password"} 2026-02-23T14:19:08.272793945Z INFO fenceagentsremediation-params collectAllSecretParams finish successfully for node {"node": "worker-0-0"} 2026-02-23T14:19:08.272806289Z INFO fenceagentsremediation-params BuildFenceAgentParams finished successfully {"Node Name": "worker-0-0-5bl95"} 2026-02-23T14:19:08.272814616Z INFO controllers.FenceAgentsRemediation Execute the fence agent {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-0", "FAR uid": "16ea6729-6e44-4703-887a-7a180c9d8942", "Parameters": ["--ip","--lanplus","--action","--ipport","--username","--password"]} 2026-02-23T14:19:08.272882249Z DEBUG events [remediation] Remediation taint was added {"type": "Normal", "object": {"kind":"Node","name":"worker-0-0","uid":"76213c99-accf-4836-856e-43602ebb0a4d","apiVersion":"v1","resourceVersion":"3805300"}, "reason": "AddRemediationTaint"} 2026-02-23T14:19:08.272968063Z DEBUG events [remediation] Fence agent was executed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-5bl95","uid":"16ea6729-6e44-4703-887a-7a180c9d8942","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3806375"}, "reason": "FenceAgentExecuted"} 2026-02-23T14:19:08.272999108Z INFO executer fence agent start {"uid": "16ea6729-6e44-4703-887a-7a180c9d8942", "fence_agent": "fence_ipmilan", "retryCount": 5, "retryInterval": "10s", "timeout": "5m0s"} 2026-02-23T14:19:08.294551863Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T14:19:08.294614571Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T14:19:08.294622049Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T14:19:08.294669706Z INFO controllers.FenceAgentsRemediation A Fence Agent is already running {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-0", "FAR uid": "16ea6729-6e44-4703-887a-7a180c9d8942"} 2026-02-23T14:19:08.30250326Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T14:19:13.244086987Z INFO executer command completed {"uid": "16ea6729-6e44-4703-887a-7a180c9d8942", "response": "Success: Rebooted\n", "errMessage": "", "err": null} 2026-02-23T14:19:13.244219167Z INFO executer fence agent done {"uid": "16ea6729-6e44-4703-887a-7a180c9d8942", "fence_agent": "fence_ipmilan", "stdout": "Success: Rebooted\n", "stderr": "", "err": null} 2026-02-23T14:19:13.244243234Z INFO executer updating status {"FAR uid": "16ea6729-6e44-4703-887a-7a180c9d8942"} 2026-02-23T14:19:13.244349385Z INFO executer Updating Status Condition {"processingConditionStatus": "", "fenceAgentActionSucceededConditionStatus": "True", "succeededConditionStatus": "", "reason": "FenceAgentSucceeded", "LastUpdateTime": "2026-02-23T14:19:13.244348431Z"} 2026-02-23T14:19:13.244691758Z DEBUG events [remediation] Fence agent was succeeded {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-5bl95","uid":"16ea6729-6e44-4703-887a-7a180c9d8942","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3806375"}, "reason": "FenceAgentSucceeded"} 2026-02-23T14:19:13.250314502Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T14:19:13.250346192Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T14:19:13.250311038Z INFO executer status updated {"FAR uid": "16ea6729-6e44-4703-887a-7a180c9d8942"} 2026-02-23T14:19:13.250539845Z INFO controllers.FenceAgentsRemediation Remediation strategy is OutOfServiceTaint which implicitly deletes resources - adding out-of-service taint {"Node Name": "worker-0-0-5bl95"} 2026-02-23T14:19:13.258041382Z INFO taints Taint was added {"taint effect": "NoExecute", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2026-02-23T14:12:48Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-23T14:12:48Z"},{"key":"remediation.medik8s.io/fence-agents-remediation","effect":"NoSchedule","timeAdded":"2026-02-23T14:19:08Z"},{"key":"node.kubernetes.io/out-of-service","value":"nodeshutdown","effect":"NoExecute","timeAdded":"2026-02-23T14:19:13Z"}]} 2026-02-23T14:19:13.258101653Z INFO controllers.FenceAgentsRemediation out-of-service taint was added {"Node Name": "worker-0-0"} 2026-02-23T14:19:13.258138787Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "False", "fenceAgentActionSucceededConditionStatus": "", "succeededConditionStatus": "True", "reason": "RemediationFinishedSuccessfully", "LastUpdateTime": "2026-02-23T14:19:13.258135215Z"} 2026-02-23T14:19:13.258151885Z INFO executer cancelling fence agent routine {"uid": "16ea6729-6e44-4703-887a-7a180c9d8942"} 2026-02-23T14:19:13.258161217Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR has completed to remediate the node {"Node Name": "worker-0-0"} 2026-02-23T14:19:13.258524828Z DEBUG events [remediation] The out-of-service taint was added {"type": "Normal", "object": {"kind":"Node","name":"worker-0-0","uid":"76213c99-accf-4836-856e-43602ebb0a4d","apiVersion":"v1","resourceVersion":"3806377"}, "reason": "AddOutOfServiceTaint"} 2026-02-23T14:19:13.258747378Z DEBUG events [remediation] Unhealthy node remediation was completed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-0","uid":"76213c99-accf-4836-856e-43602ebb0a4d","apiVersion":"v1","resourceVersion":"3806377"}, "reason": "NodeRemediationCompleted"} 2026-02-23T14:19:13.258756368Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-5bl95","uid":"16ea6729-6e44-4703-887a-7a180c9d8942","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3806410"}, "reason": "RemediationFinished"} 2026-02-23T14:19:13.265374099Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T14:19:13.266219654Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T14:19:13.266255552Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T14:19:13.272679001Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T14:20:27.729028667Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T14:20:27.729072589Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T14:20:27.729100752Z INFO controllers.FenceAgentsRemediation CR's deletion timestamp is not zero, and FAR finalizer exists {"CR Name": "worker-0-0-5bl95"} 2026-02-23T14:20:27.729104621Z INFO controllers.FenceAgentsRemediation Removing out-of-service taint {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-0"} 2026-02-23T14:20:27.735630951Z INFO taints Taint was removed {"taint effect": "NoExecute", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-23T14:12:48Z"},{"key":"remediation.medik8s.io/fence-agents-remediation","effect":"NoSchedule","timeAdded":"2026-02-23T14:19:08Z"}]} 2026-02-23T14:20:27.735677817Z INFO controllers.FenceAgentsRemediation out-of-service taint was removed {"Node Name": "worker-0-0"} 2026-02-23T14:20:27.735807358Z DEBUG events [remediation] The out-of-service taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-0","uid":"76213c99-accf-4836-856e-43602ebb0a4d","apiVersion":"v1","resourceVersion":"3807001"}, "reason": "RemoveOutOfServiceTaint"} 2026-02-23T14:20:27.742854819Z INFO controllers.FenceAgentsRemediation Failed to remove taint from node due to node update, retrying... ,{"node name": "worker-0-0", "taint key": "remediation.medik8s.io/fence-agents-remediation", "taint effect": "NoSchedule"} 2026-02-23T14:20:27.748130944Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T14:20:28.74946921Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T14:20:28.749548691Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2026-02-23T14:20:28.749606142Z INFO controllers.FenceAgentsRemediation CR's deletion timestamp is not zero, and FAR finalizer exists {"CR Name": "worker-0-0-5bl95"} 2026-02-23T14:20:28.749612697Z INFO controllers.FenceAgentsRemediation Removing out-of-service taint {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-0"} 2026-02-23T14:20:28.749636687Z INFO controllers.FenceAgentsRemediation out-of-service taint was removed {"Node Name": "worker-0-0"} 2026-02-23T14:20:28.749749522Z DEBUG events [remediation] The out-of-service taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-0","uid":"76213c99-accf-4836-856e-43602ebb0a4d","apiVersion":"v1","resourceVersion":"3807005"}, "reason": "RemoveOutOfServiceTaint"} 2026-02-23T14:20:28.757034203Z INFO taints Taint was removed {"taint effect": "NoSchedule", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2026-02-23T14:12:48Z"}]} 2026-02-23T14:20:28.757113677Z INFO controllers.FenceAgentsRemediation FAR remediation taint was removed {"Node Name": "worker-0-0"} 2026-02-23T14:20:28.757375652Z DEBUG events [remediation] Remediation taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-0","uid":"76213c99-accf-4836-856e-43602ebb0a4d","apiVersion":"v1","resourceVersion":"3807005"}, "reason": "RemoveRemediationTaint"} 2026-02-23T14:20:28.770686259Z INFO controllers.FenceAgentsRemediation Finalizer was removed {"CR Name": "worker-0-0-5bl95"} 2026-02-23T14:20:28.770748399Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T14:20:28.770820168Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2026-02-23T14:20:28.770839417Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR was not found {"CR Name": "worker-0-0-5bl95", "CR Namespace": "openshift-workload-availability"} 2026-02-23T14:20:28.770848103Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2026-02-23T14:20:28.770879543Z DEBUG events [remediation] Finalizer was removed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-5bl95","uid":"16ea6729-6e44-4703-887a-7a180c9d8942","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3807003"}, "reason": "RemoveFinalizer"}