[kni@titan45 ~]$ oc project openshift-workload-availability Already on project "openshift-workload-availability" on server "https://api.ocp-edge-cluster-0.qe.lab.redhat.com:6443". [kni@titan45 ~]$ oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.19.0-0.nightly-2025-09-11-184658 True False 4h31m Cluster version is 4.19.0-0.nightly-2025-09-11-184658 [kni@titan45 ~]$ oc get csv NAME DISPLAY VERSION REPLACES PHASE fence-agents-remediation.v0.6.0 Fence Agents Remediation Operator 0.6.0 fence-agents-remediation.v0.5.1 Succeeded node-healthcheck-operator.v0.10.0 Node Health Check Operator 0.10.0 node-healthcheck-operator.v0.9.1 Succeeded self-node-remediation.v0.10.2 Self Node Remediation Operator 0.10.2 self-node-remediation.v0.10.1 Succeeded [kni@titan45 ~]$ PODS=$(oc get pods -o name -n openshift-workload-availability | grep fence-agents-remediation-controller-manager) [kni@titan45 ~]$ echo $PODS pod/fence-agents-remediation-controller-manager-7bfcd47b77-k2dzj pod/fence-agents-remediation-controller-manager-7bfcd47b77-vs4c9 [kni@titan45 ~]$ for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/fence-agents-remediation-controller-manager-7bfcd47b77-k2dzj "worker-0-1" == pod/fence-agents-remediation-controller-manager-7bfcd47b77-vs4c9 "worker-0-0" [kni@titan45 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 4h43m v1.32.8 worker-0-1 Ready worker 4h43m v1.32.8 worker-0-2 Ready worker 4h43m v1.32.8 [kni@titan45 ~]$ oc get fartemplate -o yaml apiVersion: v1 items: [] kind: List metadata: resourceVersion: "" [kni@titan45 ~]$ oc get nhc -o yaml apiVersion: v1 items: [] kind: List metadata: resourceVersion: "" [kni@titan45 ~]$ oc patch bmh openshift-worker-0-2 -n openshift-machine-api --type='merge' -p '{"metadata":{"annotations":{"baremetalhost.metal3.io/detached":"far-ipmi-off"}}}' baremetalhost.metal3.io/openshift-worker-0-2 patched [kni@titan45 ~]$ oc get bmh openshift-worker-0-2 -o json -n openshift-machine-api | jq .metadata.annotations { "baremetalhost.metal3.io/detached": "far-ipmi-off" } [kni@titan45 ~]$ vi test.yaml [kni@titan45 ~]$ cat test.yaml --- apiVersion: v1 kind: Secret stringData: --password: password metadata: name: worker-0-cred namespace: openshift-workload-availability type: Opaque --- apiVersion: v1 kind: Secret stringData: --password: password metadata: name: worker-1-cred namespace: openshift-workload-availability type: Opaque --- apiVersion: v1 kind: Secret stringData: --password: password metadata: name: worker-2-cred namespace: openshift-workload-availability type: Opaque --- apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate metadata: name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability spec: template: spec: agent: fence_ipmilan retrycount: 5 retryinterval: 10s timeout: 300s nodeparameters: '--ipport': master-0-0: '6230' master-0-1: '6231' master-0-2: '6232' worker-0-0: '6233' worker-0-1: '6234' worker-0-2: '6235' sharedparameters: '--action': "off" '--lanplus': '' '--ip': 192.168.123.1 '--username': admin nodeSecretNames: worker-0-0: worker-0-cred worker-0-1: worker-1-cred worker-0-2: worker-2-cred [kni@titan45 ~]$ oc apply -f test.yaml secret/worker-0-cred configured secret/worker-1-cred created secret/worker-2-cred created fenceagentsremediationtemplate.fence-agents-remediation.medik8s.io/fenceagentsremediationtemplate-test created [kni@titan45 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 5h13m v1.32.8 worker-0-1 Ready worker 5h13m v1.32.8 worker-0-2 Ready worker 5h12m v1.32.8 [kni@titan45 ~]$ oc debug node/worker-0-2 -- chroot /host bash -c "date & uptime -s" Temporary namespace openshift-debug-dg878 is created for debugging node... Starting pod/worker-0-2-debug-xtv2d ... To use host binaries, run `chroot /host` Fri Sep 12 14:21:04 UTC 2025 2025-09-12 13:10:32 Removing debug pod ... Temporary namespace openshift-debug-dg878 was removed. [kni@titan45 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 5h16m v1.32.8 worker-0-1 Ready worker 5h16m v1.32.8 worker-0-2 NotReady worker 5h16m v1.32.8 [kni@titan45 ~]$ oc get nhc -o yaml apiVersion: v1 items: [] kind: List metadata: resourceVersion: "" [kni@titan45 ~]$ vi test.yaml [kni@titan45 ~]$ cat test.yaml apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: name: nhc-far-worker spec: maxUnhealthy: 1 remediationTemplate: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/control-plane operator: DoesNotExist values: [] - key: node-role.kubernetes.io/master operator: DoesNotExist values: [] unhealthyConditions: - duration: 30s status: 'False' type: Ready - duration: 30s status: Unknown type: Ready [kni@titan45 ~]$ oc apply -f test.yaml nodehealthcheck.remediation.medik8s.io/nhc-far-worker created [kni@titan45 ~]$ oc get far NAME AGE worker-0-2-qkt5p 10s [kni@titan45 ~]$ oc get far -o yaml apiVersion: v1 items: - apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediation metadata: annotations: remediation.medik8s.io/node-name: worker-0-2 remediation.medik8s.io/template-name: fenceagentsremediationtemplate-test creationTimestamp: "2025-09-12T14:26:05Z" finalizers: - fence-agents-remediation.medik8s.io/far-finalizer generateName: worker-0-2- generation: 1 labels: app.kubernetes.io/part-of: node-healthcheck-controller name: worker-0-2-qkt5p namespace: openshift-workload-availability ownerReferences: - apiVersion: remediation.medik8s.io/v1alpha1 controller: false kind: NodeHealthCheck name: nhc-far-worker uid: 8f4fe4d7-b762-4226-ab9f-eb4caf806de1 resourceVersion: "142279" uid: 162c4468-cfcc-4aa3-becf-51535abb7d4a spec: agent: fence_ipmilan nodeSecretNames: worker-0-0: worker-0-cred worker-0-1: worker-1-cred worker-0-2: worker-2-cred nodeparameters: --ipport: master-0-0: "6230" master-0-1: "6231" master-0-2: "6232" worker-0-0: "6233" worker-0-1: "6234" worker-0-2: "6235" remediationStrategy: ResourceDeletion retrycount: 5 retryinterval: 10s sharedSecretName: fence-agents-credentials-shared sharedparameters: --action: "off" --ip: 192.168.123.1 --lanplus: "" --username: admin timeout: 5m0s status: conditions: - lastTransitionTime: "2025-09-12T14:26:11Z" message: The unhealthy node was fully remediated (it was tainted, fenced using the fence agent and all the node resources have been deleted) reason: RemediationFinishedSuccessfully status: "False" type: Processing - lastTransitionTime: "2025-09-12T14:26:09Z" message: FAR taint was added and the fence agent command has been created and executed successfully reason: FenceAgentSucceeded status: "True" type: FenceAgentActionSucceeded - lastTransitionTime: "2025-09-12T14:26:11Z" message: The unhealthy node was fully remediated (it was tainted, fenced using the fence agent and all the node resources have been deleted) reason: RemediationFinishedSuccessfully status: "True" type: Succeeded lastUpdateTime: "2025-09-12T14:26:11Z" kind: List metadata: resourceVersion: "" [kni@titan45 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 5h30m v1.32.8 worker-0-1 Ready worker 5h31m v1.32.8 worker-0-2 NotReady worker 5h30m v1.32.8 [kni@titan45 ~]$ oc get bmh openshift-worker-0-2 -o json -n openshift-machine-api | jq .metadata.annotations { "baremetalhost.metal3.io/detached": "far-ipmi-off" } ------------------------Removing the BMH Annotation------------------------ [kni@titan45 ~]$ oc annotate bmh openshift-worker-0-2 -n openshift-machine-api baremetalhost.metal3.io/detached- baremetalhost.metal3.io/openshift-worker-0-2 annotated [kni@titan45 ~]$ oc get bmh openshift-worker-0-2 -o json -n openshift-machine-api | jq .metadata.annotations null [kni@titan45 ~]$ oc debug node/worker-0-2 -- chroot /host bash -c "date & uptime -s" Temporary namespace openshift-debug-726p6 is created for debugging node... Starting pod/worker-0-2-debug-mtjcn ... To use host binaries, run `chroot /host` Fri Sep 12 15:42:49 UTC 2025 2025-09-12 15:41:03 Removing debug pod ... Temporary namespace openshift-debug-726p6 was removed. [kni@titan45 ~]$ oc logs pod/fence-agents-remediation-controller-manager-7bfcd47b77-k2dzj 2025-09-12T12:53:17.788012498Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-09-12T12:53:17.78842785Z INFO setup Go OS/Arch: linux/amd64 2025-09-12T12:53:17.788439436Z INFO setup Operator Version: bd73055e 2025-09-12T12:53:17.788445776Z INFO setup Git Commit: bd73055ef2c68bfdc865d2c54179f4448bd454da 2025-09-12T12:53:17.78845107Z INFO setup Build Date: 2025-09-08T09:09:10+00:00 2025-09-12T12:53:17.78855798Z INFO setup HTTP/2 for webhooks disabled 2025-09-12T12:53:17.814858402Z INFO validation out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 32} 2025-09-12T12:53:17.814994014Z INFO setup out-of-service taint is supported on this cluster 2025-09-12T12:53:17.815168867Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation"} 2025-09-12T12:53:17.815387928Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2025-09-12T12:53:17.815800709Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2025-09-12T12:53:17.816033511Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-12T12:53:17.816235507Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-12T12:53:17.816377522Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-12T12:53:17.816578509Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-12T12:53:17.816677702Z INFO setup starting manager 2025-09-12T12:53:17.817381301Z INFO controller-runtime.metrics Starting metrics server 2025-09-12T12:53:17.817581706Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-09-12T12:53:17.817754054Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2025-09-12T12:53:17.818993993Z INFO controller-runtime.webhook Starting webhook server I0912 12:53:17.819875 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/cb305759.medik8s.io... 2025-09-12T12:53:17.820009895Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-09-12T12:53:17.820473088Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-09-12T12:53:17.821355913Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} 2025-09-12T13:04:44.614456273Z INFO fenceagentsremediationtemplate-resource default {"name": "fenceagentsremediationtemplate-test"} 2025-09-12T13:04:44.629741498Z INFO fenceagentsremediationtemplate-resource validate create {"name": "fenceagentsremediationtemplate-test"} 2025-09-12T13:04:46.467876889Z INFO fenceagentsremediation-resource validate create {"name": "worker-0-2-j9q9l"} 2025-09-12T13:04:46.608391464Z INFO fenceagentsremediation-resource validate update {"name": "worker-0-2-j9q9l"} 2025-09-12T13:09:16.315720311Z INFO fenceagentsremediationtemplate-resource validate update {"name": "fenceagentsremediationtemplate-test"} 2025-09-12T14:19:28.946839528Z INFO fenceagentsremediationtemplate-resource validate create {"name": "fenceagentsremediationtemplate-test"} [kni@titan45 ~]$ oc logs [kni@titan45 ~]$ oc logs pod/fence-agents-remediation-controller-manager-7bfcd47b77-vs4c9 2025-09-12T12:53:08.524544618Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-09-12T12:53:08.524858082Z INFO setup Go OS/Arch: linux/amd64 2025-09-12T12:53:08.524867645Z INFO setup Operator Version: bd73055e 2025-09-12T12:53:08.524899355Z INFO setup Git Commit: bd73055ef2c68bfdc865d2c54179f4448bd454da 2025-09-12T12:53:08.524908832Z INFO setup Build Date: 2025-09-08T09:09:10+00:00 2025-09-12T12:53:08.524964073Z INFO setup HTTP/2 for webhooks disabled 2025-09-12T12:53:08.59661479Z INFO validation out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 32} 2025-09-12T12:53:08.596918062Z INFO setup out-of-service taint is supported on this cluster 2025-09-12T12:53:08.59714752Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation"} 2025-09-12T12:53:08.59731168Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2025-09-12T12:53:08.597674534Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2025-09-12T12:53:08.597853421Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-12T12:53:08.598187196Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-12T12:53:08.598294914Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-12T12:53:08.598448954Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-12T12:53:08.598594664Z INFO setup starting manager 2025-09-12T12:53:08.599056239Z INFO controller-runtime.metrics Starting metrics server 2025-09-12T12:53:08.599266017Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-09-12T12:53:08.599524825Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2025-09-12T12:53:08.599483554Z INFO controller-runtime.webhook Starting webhook server I0912 12:53:08.599866 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/cb305759.medik8s.io... 2025-09-12T12:53:08.600499824Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-09-12T12:53:08.600706999Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-09-12T12:53:08.600948661Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} I0912 12:53:31.430420 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/cb305759.medik8s.io 2025-09-12T12:53:31.431268869Z DEBUG events fence-agents-remediation-controller-manager-7bfcd47b77-vs4c9_718e85aa-6178-494b-a22b-4c07a7110f26 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"cb305759.medik8s.io","uid":"abd01c01-e4b5-4cf6-92c8-9d02571ac460","apiVersion":"coordination.k8s.io/v1","resourceVersion":"110668"}, "reason": "LeaderElection"} 2025-09-12T12:53:31.432312416Z INFO Starting EventSource {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation", "source": "kind source: *v1alpha1.FenceAgentsRemediation"} 2025-09-12T12:53:31.432375753Z INFO Starting Controller {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation"} 2025-09-12T12:53:31.539140902Z INFO Starting workers {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation", "worker count": 1} 2025-09-12T13:10:15.576426467Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T13:10:15.576848386Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-12T13:10:15.577076116Z INFO controllers.FenceAgentsRemediation CR's deletion timestamp is not zero, and FAR finalizer exists {"CR Name": "worker-0-2-j9q9l"} 2025-09-12T13:10:15.577126869Z INFO controllers.FenceAgentsRemediation FAR didn't finish remediate the node {"CR Name": "worker-0-2-j9q9l", "processing condition": "True", "fenceAgentActionSucceeded condition": "Unknown", "succeeded condition": "Unknown"} 2025-09-12T13:10:15.612967899Z INFO taints Taint was removed {"taint effect": "NoExecute", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-09-12T12:41:04Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-09-12T12:41:11Z"}]} 2025-09-12T13:10:15.613346967Z INFO controllers.FenceAgentsRemediation FAR remediation taint was removed {"Node Name": "worker-0-2"} 2025-09-12T13:10:15.614067346Z DEBUG events [remediation] Remediation taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"33b1fdb5-b748-44b3-a34a-efe8e0fb4cc6","apiVersion":"v1","resourceVersion":"115897"}, "reason": "RemoveRemediationTaint"} 2025-09-12T13:10:15.668152427Z INFO fenceagentsremediation-resource validate update {"name": "worker-0-2-j9q9l"} 2025-09-12T13:10:15.693366558Z INFO controllers.FenceAgentsRemediation Finalizer was removed {"CR Name": "worker-0-2-j9q9l"} 2025-09-12T13:10:15.693813783Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T13:10:15.694028288Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T13:10:15.694222844Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR was not found{"CR Name": "worker-0-2-j9q9l", "CR Namespace": "openshift-workload-availability"} 2025-09-12T13:10:15.694259164Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T13:10:15.694216901Z DEBUG events [remediation] Finalizer was removed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-j9q9l","uid":"ddc8fd9f-c217-4ade-9c0e-6ff69585cca6","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"116619"}, "reason": "RemoveFinalizer"} 2025-09-12T13:10:15.787477431Z INFO fenceagentsremediation-resource validate create {"name": "worker-0-2-t66k4"} 2025-09-12T13:10:15.809122558Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T13:10:15.809245392Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-12T13:10:15.828296264Z INFO fenceagentsremediation-resource validate update {"name": "worker-0-2-t66k4"} 2025-09-12T13:10:15.856531022Z INFO controllers.FenceAgentsRemediation Finalizer was added {"CR Name": "worker-0-2-t66k4"} 2025-09-12T13:10:15.856818926Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "True", "fenceAgentActionSucceededConditionStatus": "Unknown", "succeededConditionStatus": "Unknown", "reason": "RemediationStarted", "LastUpdateTime": "2025-09-12T13:10:15.8568139Z"} 2025-09-12T13:10:15.85689691Z DEBUG events [remediation] Remediation started {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-t66k4","uid":"306b530d-e6a9-4c55-8274-c44a8e5681bd","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"116628"}, "reason": "RemediationStarted"} 2025-09-12T13:10:15.857166937Z DEBUG events [remediation] Finalizer was added {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-t66k4","uid":"306b530d-e6a9-4c55-8274-c44a8e5681bd","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"116628"}, "reason": "AddFinalizer"} 2025-09-12T13:10:15.880587342Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T13:10:15.881055423Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T13:10:15.881136274Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-12T13:10:15.917381174Z INFO taints Taint was added {"taint effect": "NoExecute", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-09-12T12:41:04Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-09-12T12:41:11Z"},{"key":"medik8s.io/fence-agents-remediation","effect":"NoExecute","timeAdded":"2025-09-12T13:10:15Z"}]} 2025-09-12T13:10:15.917542732Z INFO controllers.FenceAgentsRemediation FAR remediation taint was added {"Node Name": "worker-0-2"} 2025-09-12T13:10:15.917775573Z INFO controllers.FenceAgentsRemediation Build fence agent command line {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2"} 2025-09-12T13:10:15.918075181Z INFO controllers.FenceAgentsRemediation found a value from secret {"secret name": "test-far-shared", "parameter name": "--password"} 2025-09-12T13:10:15.918106601Z INFO controllers.FenceAgentsRemediation found a value from secret {"secret name": "test-far-shared", "parameter name": "--username"} 2025-09-12T13:10:15.918116887Z DEBUG events [remediation] Remediation taint was added {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"33b1fdb5-b748-44b3-a34a-efe8e0fb4cc6","apiVersion":"v1","resourceVersion":"116620"}, "reason": "AddRemediationTaint"} 2025-09-12T13:10:15.918273015Z INFO controllers.FenceAgentsRemediation Execute the fence agent {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "306b530d-e6a9-4c55-8274-c44a8e5681bd", "Parameters": ["--action","--ip","--lanplus","--ipport","--username","--password"]} 2025-09-12T13:10:15.919247301Z DEBUG events [remediation] Fence agent was executed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-t66k4","uid":"306b530d-e6a9-4c55-8274-c44a8e5681bd","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"116632"}, "reason": "FenceAgentExecuted"} 2025-09-12T13:10:15.918758943Z INFO executer fence agent start {"uid": "306b530d-e6a9-4c55-8274-c44a8e5681bd", "fence_agent": "fence_ipmilan", "retryCount": 5, "retryInterval": "10s", "timeout": "5m0s"} 2025-09-12T13:10:15.951930755Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T13:10:15.952144973Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T13:10:15.952192493Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-12T13:10:15.952345267Z INFO controllers.FenceAgentsRemediation A Fence Agent is already running {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "306b530d-e6a9-4c55-8274-c44a8e5681bd"} 2025-09-12T13:10:15.982234211Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T13:10:22.492073836Z INFO executer command completed {"uid": "306b530d-e6a9-4c55-8274-c44a8e5681bd", "response": "Success: Rebooted\n", "errMessage": "", "err": null} 2025-09-12T13:10:22.492440674Z INFO executer fence agent done {"uid": "306b530d-e6a9-4c55-8274-c44a8e5681bd", "fence_agent": "fence_ipmilan", "stdout": "Success: Rebooted\n", "stderr": "", "err": null} 2025-09-12T13:10:22.492570358Z INFO executer updating status {"FAR uid": "306b530d-e6a9-4c55-8274-c44a8e5681bd"} 2025-09-12T13:10:22.493113203Z INFO executer Updating Status Condition {"processingConditionStatus": "", "fenceAgentActionSucceededConditionStatus": "True", "succeededConditionStatus": "", "reason": "FenceAgentSucceeded", "LastUpdateTime": "2025-09-12T13:10:22.493106664Z"} 2025-09-12T13:10:22.493220191Z DEBUG events [remediation] Fence agent was succeeded {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-t66k4","uid":"306b530d-e6a9-4c55-8274-c44a8e5681bd","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"116632"}, "reason": "FenceAgentSucceeded"} 2025-09-12T13:10:22.514025745Z INFO executer status updated {"FAR uid": "306b530d-e6a9-4c55-8274-c44a8e5681bd"} 2025-09-12T13:10:22.514100801Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T13:10:22.514492303Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-12T13:10:22.514789234Z INFO controllers.FenceAgentsRemediation Remediation strategy is ResourceDeletion which explicitly deletes resources - manually deleting workload {"Node Name": "worker-0-2-t66k4"} 2025-09-12T13:10:22.515353293Z DEBUG events [remediation] Manually delete pods from the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"33b1fdb5-b748-44b3-a34a-efe8e0fb4cc6","apiVersion":"v1","resourceVersion":"116634"}, "reason": "DeleteResources"} 2025-09-12T13:10:22.61739323Z INFO commons-resource starting to delete pods {"node name": "worker-0-2"} 2025-09-12T13:10:25.180021467Z INFO commons-resource done deleting pods {"node name": "worker-0-2"} 2025-09-12T13:10:25.180077956Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "False", "fenceAgentActionSucceededConditionStatus": "", "succeededConditionStatus": "True", "reason": "RemediationFinishedSuccessfully", "LastUpdateTime": "2025-09-12T13:10:25.18007606Z"} 2025-09-12T13:10:25.18010578Z INFO executer cancelling fence agent routine {"uid": "306b530d-e6a9-4c55-8274-c44a8e5681bd"} 2025-09-12T13:10:25.180118975Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR has completed to remediate the node {"Node Name": "worker-0-2"} 2025-09-12T13:10:25.180486827Z DEBUG events [remediation] Unhealthy node remediation was completed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"33b1fdb5-b748-44b3-a34a-efe8e0fb4cc6","apiVersion":"v1","resourceVersion":"116634"}, "reason": "NodeRemediationCompleted"} 2025-09-12T13:10:25.180553113Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-t66k4","uid":"306b530d-e6a9-4c55-8274-c44a8e5681bd","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"116692"}, "reason": "RemediationFinished"} 2025-09-12T13:10:25.199722291Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T13:10:25.200094225Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T13:10:25.20022978Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-12T13:10:25.213959666Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T13:12:05.350404481Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T13:12:05.350557812Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-12T13:12:05.350689692Z INFO controllers.FenceAgentsRemediation CR's deletion timestamp is not zero, and FAR finalizer exists {"CR Name": "worker-0-2-t66k4"} 2025-09-12T13:12:05.418982797Z INFO taints Taint was removed {"taint effect": "NoExecute", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-09-12T12:41:11Z"}]} 2025-09-12T13:12:05.419142166Z INFO controllers.FenceAgentsRemediation FAR remediation taint was removed {"Node Name": "worker-0-2"} 2025-09-12T13:12:05.419455924Z DEBUG events [remediation] Remediation taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"33b1fdb5-b748-44b3-a34a-efe8e0fb4cc6","apiVersion":"v1","resourceVersion":"117492"}, "reason": "RemoveRemediationTaint"} 2025-09-12T13:12:05.465068817Z INFO fenceagentsremediation-resource validate update {"name": "worker-0-2-t66k4"} 2025-09-12T13:12:05.501857425Z INFO controllers.FenceAgentsRemediation Finalizer was removed {"CR Name": "worker-0-2-t66k4"} 2025-09-12T13:12:05.502071344Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T13:12:05.502148837Z DEBUG events [remediation] Finalizer was removed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-t66k4","uid":"306b530d-e6a9-4c55-8274-c44a8e5681bd","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"117503"}, "reason": "RemoveFinalizer"} 2025-09-12T13:12:05.502939927Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T13:12:05.503184958Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR was not found{"CR Name": "worker-0-2-t66k4", "CR Namespace": "openshift-workload-availability"} 2025-09-12T13:12:05.503315709Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T14:19:28.928218552Z INFO fenceagentsremediationtemplate-resource default {"name": "fenceagentsremediationtemplate-test"} 2025-09-12T14:26:05.781898592Z INFO fenceagentsremediation-resource validate create {"name": "worker-0-2-qkt5p"} 2025-09-12T14:26:05.795767873Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T14:26:05.796156631Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-12T14:26:05.821754335Z INFO fenceagentsremediation-resource validate update {"name": "worker-0-2-qkt5p"} 2025-09-12T14:26:05.834150639Z INFO controllers.FenceAgentsRemediation Finalizer was added {"CR Name": "worker-0-2-qkt5p"} 2025-09-12T14:26:05.834319605Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "True", "fenceAgentActionSucceededConditionStatus": "Unknown", "succeededConditionStatus": "Unknown", "reason": "RemediationStarted", "LastUpdateTime": "2025-09-12T14:26:05.834314682Z"} 2025-09-12T14:26:05.834342655Z DEBUG events [remediation] Remediation started {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-qkt5p","uid":"162c4468-cfcc-4aa3-becf-51535abb7d4a","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"142088"}, "reason": "RemediationStarted"} 2025-09-12T14:26:05.83462162Z DEBUG events [remediation] Finalizer was added {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-qkt5p","uid":"162c4468-cfcc-4aa3-becf-51535abb7d4a","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"142088"}, "reason": "AddFinalizer"} 2025-09-12T14:26:06.055177577Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T14:26:06.055496763Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T14:26:06.055547212Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-12T14:26:06.091822595Z INFO taints Taint was added {"taint effect": "NoExecute", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-09-12T14:23:08Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-09-12T14:23:14Z"},{"key":"medik8s.io/fence-agents-remediation","effect":"NoExecute","timeAdded":"2025-09-12T14:26:06Z"}]} 2025-09-12T14:26:06.092029711Z INFO controllers.FenceAgentsRemediation FAR remediation taint was added {"Node Name": "worker-0-2"} 2025-09-12T14:26:06.09214673Z INFO controllers.FenceAgentsRemediation Build fence agent command line {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2"} 2025-09-12T14:26:06.092286549Z INFO controllers.FenceAgentsRemediation found a value from secret {"secret name": "worker-2-cred", "parameter name": "--password"} 2025-09-12T14:26:06.092411805Z INFO controllers.FenceAgentsRemediation Execute the fence agent {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "162c4468-cfcc-4aa3-becf-51535abb7d4a", "Parameters": ["--lanplus","--username","--ipport","--password","--action","--ip"]} 2025-09-12T14:26:06.092850511Z DEBUG events [remediation] Remediation taint was added {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"33b1fdb5-b748-44b3-a34a-efe8e0fb4cc6","apiVersion":"v1","resourceVersion":"141550"}, "reason": "AddRemediationTaint"} 2025-09-12T14:26:06.092933575Z DEBUG events [remediation] Fence agent was executed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-qkt5p","uid":"162c4468-cfcc-4aa3-becf-51535abb7d4a","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"142089"}, "reason": "FenceAgentExecuted"} 2025-09-12T14:26:06.093260806Z INFO executer fence agent start {"uid": "162c4468-cfcc-4aa3-becf-51535abb7d4a", "fence_agent": "fence_ipmilan", "retryCount": 5, "retryInterval": "10s", "timeout": "5m0s"} 2025-09-12T14:26:06.15032652Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T14:26:06.1505169Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T14:26:06.15056738Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-12T14:26:06.175958392Z INFO controllers.FenceAgentsRemediation A Fence Agent is already running {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "162c4468-cfcc-4aa3-becf-51535abb7d4a"} 2025-09-12T14:26:06.292237973Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T14:26:09.137624019Z INFO executer command completed {"uid": "162c4468-cfcc-4aa3-becf-51535abb7d4a", "response": "Success: Powered OFF\n", "errMessage": "", "err": null} 2025-09-12T14:26:09.138139984Z INFO executer fence agent done {"uid": "162c4468-cfcc-4aa3-becf-51535abb7d4a", "fence_agent": "fence_ipmilan", "stdout": "Success: Powered OFF\n", "stderr": "", "err": null} 2025-09-12T14:26:09.138234197Z INFO executer updating status {"FAR uid": "162c4468-cfcc-4aa3-becf-51535abb7d4a"} 2025-09-12T14:26:09.138520718Z INFO executer Updating Status Condition {"processingConditionStatus": "", "fenceAgentActionSucceededConditionStatus": "True", "succeededConditionStatus": "", "reason": "FenceAgentSucceeded", "LastUpdateTime": "2025-09-12T14:26:09.138517766Z"} 2025-09-12T14:26:09.138671477Z DEBUG events [remediation] Fence agent was succeeded {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-qkt5p","uid":"162c4468-cfcc-4aa3-becf-51535abb7d4a","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"142089"}, "reason": "FenceAgentSucceeded"} 2025-09-12T14:26:09.158578533Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T14:26:09.158713513Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-12T14:26:09.158851097Z INFO controllers.FenceAgentsRemediation Remediation strategy is ResourceDeletion which explicitly deletes resources - manually deleting workload {"Node Name": "worker-0-2-qkt5p"} 2025-09-12T14:26:09.158896253Z INFO executer status updated {"FAR uid": "162c4468-cfcc-4aa3-becf-51535abb7d4a"} 2025-09-12T14:26:09.159970398Z INFO commons-resource starting to delete pods {"node name": "worker-0-2"} 2025-09-12T14:26:09.16146904Z DEBUG events [remediation] Manually delete pods from the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"33b1fdb5-b748-44b3-a34a-efe8e0fb4cc6","apiVersion":"v1","resourceVersion":"142094"}, "reason": "DeleteResources"} 2025-09-12T14:26:11.595782285Z INFO commons-resource done deleting pods {"node name": "worker-0-2"} 2025-09-12T14:26:11.595856062Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "False", "fenceAgentActionSucceededConditionStatus": "", "succeededConditionStatus": "True", "reason": "RemediationFinishedSuccessfully", "LastUpdateTime": "2025-09-12T14:26:11.595853325Z"} 2025-09-12T14:26:11.595890878Z INFO executer cancelling fence agent routine {"uid": "162c4468-cfcc-4aa3-becf-51535abb7d4a"} 2025-09-12T14:26:11.595909565Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR has completed to remediate the node {"Node Name": "worker-0-2"} 2025-09-12T14:26:11.596261836Z DEBUG events [remediation] Unhealthy node remediation was completed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"33b1fdb5-b748-44b3-a34a-efe8e0fb4cc6","apiVersion":"v1","resourceVersion":"142094"}, "reason": "NodeRemediationCompleted"} 2025-09-12T14:26:11.596358504Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-qkt5p","uid":"162c4468-cfcc-4aa3-becf-51535abb7d4a","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"142150"}, "reason": "RemediationFinished"} 2025-09-12T14:26:11.618800936Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-12T14:26:11.61904067Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-12T14:26:11.619094124Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-12T14:26:11.638243921Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile