========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.16.0-0.nightly-2025-09-06-014223 True False 10h Cluster version is 4.16.0-0.nightly-2025-09-06-014223 ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get csv NAME DISPLAY VERSION REPLACES PHASE fence-agents-remediation.v0.6.0 Fence Agents Remediation Operator 0.6.0 fence-agents-remediation.v0.5.1 Succeeded node-healthcheck-operator.v0.10.0 Node Health Check Operator 0.10.0 node-healthcheck-operator.v0.9.1 Succeeded ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ PODS=$(oc get pods -o name | grep fence-agents-remediation-controller-manager) ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ for p in $PODS; do > echo "== $p" > oc get "$p" -o json | jq .spec.nodeName > done == pod/fence-agents-remediation-controller-manager-6577bdfc69-ml4cm "worker-0-0" == pod/fence-agents-remediation-controller-manager-6577bdfc69-zz7r4 "worker-0-1" ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get fartemplate -o yaml apiVersion: v1 items: [] kind: List metadata: resourceVersion: "" ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get secret | grep worker ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get nhc -o yaml apiVersion: v1 items: [] kind: List metadata: resourceVersion: "" ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get far No resources found in openshift-workload-availability namespace. ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get pods | grep fence fence-agents-remediation-controller-manager-6577bdfc69-ml4cm 2/2 Running 0 9h fence-agents-remediation-controller-manager-6577bdfc69-zz7r4 2/2 Running 0 9h ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 10h v1.29.14+c68a663 worker-0-1 Ready worker 10h v1.29.14+c68a663 worker-0-2 Ready worker 10h v1.29.14+c68a663 ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc debug node/worker-0-2 -- chroot /host bash -c "uptime -s" Temporary namespace openshift-debug-2wgqf is created for debugging node... Starting pod/worker-0-2-debug-hlkhk ... To use host binaries, run `chroot /host` 2025-09-07 21:44:09 Removing debug pod ... Temporary namespace openshift-debug-2wgqf was removed. ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ vi secret_far_standalone.yaml ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ cat secret_far_standalone.yaml apiVersion: v1 kind: Secret stringData: --password: password metadata: name: worker-0-cred namespace: openshift-workload-availability --- apiVersion: v1 kind: Secret stringData: --password: password metadata: name: worker-1-cred namespace: openshift-workload-availability type: Opaque --- apiVersion: v1 kind: Secret stringData: --password: password metadata: name: worker-2-cred namespace: openshift-workload-availability type: Opaque --- apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediation metadata: name: worker-0-2 namespace: openshift-workload-availability spec: agent: fence_ipmilan retrycount: 5 retryinterval: 10s timeout: 300s nodeparameters: '--ipport': master-0-0: '6230' master-0-1: '6231' master-0-2: '6232' worker-0-0: '6233' worker-0-1: '6234' worker-0-2: '6235' sharedparameters: '--action': reboot '--ip': 192.168.123.1 '--lanplus': '' '--username': admin nodeSecretNames: worker-0-0: worker-0-cred worker-0-1: worker-1-cred worker-0-2: worker-2-cred ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc apply -f secret_far_standalone.yaml secret/worker-0-cred created secret/worker-1-cred created secret/worker-2-cred created fenceagentsremediation.fence-agents-remediation.medik8s.io/worker-0-2 created ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 10h v1.29.14+c68a663 worker-0-1 Ready worker 10h v1.29.14+c68a663 worker-0-2 NotReady worker 10h v1.29.14+c68a663 ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get far -o yaml apiVersion: v1 items: [] kind: List metadata: resourceVersion: "" ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc apply -f secret_far_standalone.yaml secret/worker-0-cred configured secret/worker-1-cred configured secret/worker-2-cred configured fenceagentsremediation.fence-agents-remediation.medik8s.io/worker-0-2 created ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc get far -o yaml apiVersion: v1 items: - apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediation metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","kind":"FenceAgentsRemediation","metadata":{"annotations":{},"name":"worker-0-2","namespace":"openshift-workload-availability"},"spec":{"agent":"fence_ipmilan","nodeSecretNames":{"worker-0-0":"worker-0-cred","worker-0-1":"worker-1-cred","worker-0-2":"worker-2-cred"},"nodeparameters":{"--ipport":{"master-0-0":"6230","master-0-1":"6231","master-0-2":"6232","worker-0-0":"6233","worker-0-1":"6234","worker-0-2":"6235"}},"retrycount":5,"retryinterval":"10s","sharedparameters":{"--action":"reboot","--ip":"192.168.123.1","--lanplus":"","--username":"admin"},"timeout":"300s"}} creationTimestamp: "2025-09-08T07:11:44Z" finalizers: - fence-agents-remediation.medik8s.io/far-finalizer generation: 2 name: worker-0-2 namespace: openshift-workload-availability resourceVersion: "269862" uid: e2e9aecf-8819-4fe1-879e-f14a097a4804 spec: agent: fence_ipmilan nodeSecretNames: worker-0-0: worker-0-cred worker-0-1: worker-1-cred worker-0-2: worker-2-cred nodeparameters: --ipport: master-0-0: "6230" master-0-1: "6231" master-0-2: "6232" worker-0-0: "6233" worker-0-1: "6234" worker-0-2: "6235" remediationStrategy: ResourceDeletion retrycount: 5 retryinterval: 10s sharedSecretName: fence-agents-credentials-shared sharedparameters: --action: reboot --ip: 192.168.123.1 --lanplus: "" --username: admin timeout: 5m0s status: conditions: - lastTransitionTime: "2025-09-08T07:11:51Z" message: The unhealthy node was fully remediated (it was tainted, fenced using the fence agent and all the node resources have been deleted) reason: RemediationFinishedSuccessfully status: "False" type: Processing - lastTransitionTime: "2025-09-08T07:11:49Z" message: FAR taint was added and the fence agent command has been created and executed successfully reason: FenceAgentSucceeded status: "True" type: FenceAgentActionSucceeded - lastTransitionTime: "2025-09-08T07:11:51Z" message: The unhealthy node was fully remediated (it was tainted, fenced using the fence agent and all the node resources have been deleted) reason: RemediationFinishedSuccessfully status: "True" type: Succeeded lastUpdateTime: "2025-09-08T07:11:51Z" kind: List metadata: resourceVersion: "" ========================================================================================================================================================================== [kni@cert-rhosp-02 ~]$ oc logs pod/fence-agents-remediation-controller-manager-6577bdfc69-zz7r4 2025-09-07T21:04:59.701924373Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-09-07T21:04:59.702052966Z INFO setup Go OS/Arch: linux/amd64 2025-09-07T21:04:59.702056838Z INFO setup Operator Version: 6cd59c9a 2025-09-07T21:04:59.702059663Z INFO setup Git Commit: 6cd59c9a8b6d88630f8d80303ac285e5334e0b41 2025-09-07T21:04:59.7020621Z INFO setup Build Date: 2025-09-01T19:15:36+00:00 2025-09-07T21:04:59.702083288Z INFO setup HTTP/2 for webhooks disabled 2025-09-07T21:04:59.710750513Z INFO validation out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 29} 2025-09-07T21:04:59.710775129Z INFO setup out-of-service taint is supported on this cluster 2025-09-07T21:04:59.71080999Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation"} 2025-09-07T21:04:59.710838549Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2025-09-07T21:04:59.710909663Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2025-09-07T21:04:59.710953514Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-07T21:04:59.71098751Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-07T21:04:59.711004447Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-07T21:04:59.711033897Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-07T21:04:59.711051888Z INFO setup starting manager 2025-09-07T21:04:59.711193467Z INFO controller-runtime.metrics Starting metrics server 2025-09-07T21:04:59.711221002Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-09-07T21:04:59.711276862Z INFO controller-runtime.webhook Starting webhook server 2025-09-07T21:04:59.711314706Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} I0907 21:04:59.711367 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/cb305759.medik8s.io... 2025-09-07T21:04:59.711476038Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-09-07T21:04:59.711545932Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-09-07T21:04:59.711563342Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} I0907 21:04:59.716328 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/cb305759.medik8s.io 2025-09-07T21:04:59.716388117Z DEBUG events fence-agents-remediation-controller-manager-6577bdfc69-zz7r4_159d8166-544a-437b-87ca-46dc5a6de601 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"cb305759.medik8s.io","uid":"9bcd0029-2dec-4ea9-a827-779095b67b9d","apiVersion":"coordination.k8s.io/v1","resourceVersion":"41040"}, "reason": "LeaderElection"} 2025-09-07T21:04:59.716522773Z INFO Starting EventSource {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation", "source": "kind source: *v1alpha1.FenceAgentsRemediation"} 2025-09-07T21:04:59.716540318Z INFO Starting Controller {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation"} 2025-09-07T21:04:59.817703339Z INFO Starting workers {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation", "worker count": 1} 2025-09-07T21:08:32.434107162Z INFO fenceagentsremediationtemplate-resource default {"name": "fenceagentsremediationtemplate-test"} 2025-09-07T21:08:32.438214848Z INFO fenceagentsremediationtemplate-resource validate create {"name": "fenceagentsremediationtemplate-test"} ... ... ... 2025-09-07T21:43:25.665754829Z INFO fenceagentsremediation-resource validate create {"name": "worker-0-2"} 2025-09-07T21:43:25.670438113Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-07T21:43:25.670479252Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-07T21:43:25.676400718Z INFO fenceagentsremediation-resource validate update {"name": "worker-0-2"} 2025-09-07T21:43:25.679324012Z INFO controllers.FenceAgentsRemediation Finalizer was added {"CR Name": "worker-0-2"} 2025-09-07T21:43:25.679342125Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "True", "fenceAgentActionSucceededConditionStatus": "Unknown", "succeededConditionStatus": "Unknown", "reason": "RemediationStarted", "LastUpdateTime": "2025-09-07T21:43:25.679341162Z"} 2025-09-07T21:43:25.67939631Z DEBUG events [remediation] Remediation started {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"cf4eda2d-1ef2-43f0-ac07-2a88117e60d2","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"56353"}, "reason": "RemediationStarted"} 2025-09-07T21:43:25.679436185Z DEBUG events [remediation] Finalizer was added {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"cf4eda2d-1ef2-43f0-ac07-2a88117e60d2","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"56353"}, "reason": "AddFinalizer"} 2025-09-07T21:43:25.885287234Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-07T21:43:25.885371169Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-07T21:43:25.885383295Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-07T21:43:25.891070542Z INFO taints Taint was added {"taint effect": "NoExecute", "taint list": [{"key":"medik8s.io/fence-agents-remediation","effect":"NoExecute","timeAdded":"2025-09-07T21:43:25Z"}]} 2025-09-07T21:43:25.891129707Z INFO controllers.FenceAgentsRemediation FAR remediation taint was added {"Node Name": "worker-0-2"} 2025-09-07T21:43:25.891195222Z INFO controllers.FenceAgentsRemediation Build fence agent command line {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2"} 2025-09-07T21:43:25.891268826Z INFO controllers.FenceAgentsRemediation Execute the fence agent {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "cf4eda2d-1ef2-43f0-ac07-2a88117e60d2", "ParametersError": "json: unsupported type: iter.Seq[github.com/medik8s/fence-agents-remediation/api/v1alpha1.ParameterName]"} 2025-09-07T21:43:25.891448392Z DEBUG events [remediation] Remediation taint was added {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"56185"}, "reason": "AddRemediationTaint"} 2025-09-07T21:43:25.89148774Z DEBUG events [remediation] Fence agent was executed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"cf4eda2d-1ef2-43f0-ac07-2a88117e60d2","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"56354"}, "reason": "FenceAgentExecuted"} 2025-09-07T21:43:25.891500969Z INFO executer fence agent start {"uid": "cf4eda2d-1ef2-43f0-ac07-2a88117e60d2", "fence_agent": "fence_ipmilan", "retryCount": 5, "retryInterval": "10s", "timeout": "5m0s"} 2025-09-07T21:43:25.899412865Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-07T21:43:25.899477083Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-07T21:43:25.899494308Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-07T21:43:25.899525559Z INFO controllers.FenceAgentsRemediation A Fence Agent is already running {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "cf4eda2d-1ef2-43f0-ac07-2a88117e60d2"} 2025-09-07T21:43:25.905304949Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-07T21:43:28.453979816Z INFO executer command completed {"uid": "cf4eda2d-1ef2-43f0-ac07-2a88117e60d2", "response": "Success: Powered OFF\n", "errMessage": "", "err": null} 2025-09-07T21:43:28.454019268Z INFO executer fence agent done {"uid": "cf4eda2d-1ef2-43f0-ac07-2a88117e60d2", "fence_agent": "fence_ipmilan", "stdout": "Success: Powered OFF\n", "stderr": "", "err": null} 2025-09-07T21:43:28.454025334Z INFO executer updating status {"FAR uid": "cf4eda2d-1ef2-43f0-ac07-2a88117e60d2"} 2025-09-07T21:43:28.454081244Z INFO executer Updating Status Condition {"processingConditionStatus": "", "fenceAgentActionSucceededConditionStatus": "True", "succeededConditionStatus": "", "reason": "FenceAgentSucceeded", "LastUpdateTime": "2025-09-07T21:43:28.454080567Z"} 2025-09-07T21:43:28.454263494Z DEBUG events [remediation] Fence agent was succeeded {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"cf4eda2d-1ef2-43f0-ac07-2a88117e60d2","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"56354"}, "reason": "FenceAgentSucceeded"} 2025-09-07T21:43:28.458759054Z INFO executer status updated {"FAR uid": "cf4eda2d-1ef2-43f0-ac07-2a88117e60d2"} 2025-09-07T21:43:28.459165355Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-07T21:43:28.459188525Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-07T21:43:28.45922655Z INFO controllers.FenceAgentsRemediation Remediation strategy is ResourceDeletion which explicitly deletes resources - manually deleting workload {"Node Name": "worker-0-2"} 2025-09-07T21:43:28.459313025Z DEBUG events [remediation] Manually delete pods from the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"56359"}, "reason": "DeleteResources"} 2025-09-07T21:43:28.459473467Z INFO commons-resource starting to delete pods {"node name": "worker-0-2"} 2025-09-07T21:43:30.562675998Z INFO commons-resource done deleting pods {"node name": "worker-0-2"} 2025-09-07T21:43:30.56270249Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "False", "fenceAgentActionSucceededConditionStatus": "", "succeededConditionStatus": "True", "reason": "RemediationFinishedSuccessfully", "LastUpdateTime": "2025-09-07T21:43:30.562701464Z"} 2025-09-07T21:43:30.562727096Z INFO executer cancelling fence agent routine {"uid": "cf4eda2d-1ef2-43f0-ac07-2a88117e60d2"} 2025-09-07T21:43:30.562733547Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR has completed to remediate the node {"Node Name": "worker-0-2"} 2025-09-07T21:43:30.562836343Z DEBUG events [remediation] Unhealthy node remediation was completed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"56359"}, "reason": "NodeRemediationCompleted"} 2025-09-07T21:43:30.562861273Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"cf4eda2d-1ef2-43f0-ac07-2a88117e60d2","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"56399"}, "reason": "RemediationFinished"} 2025-09-07T21:43:30.769233247Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-07T21:43:30.769335469Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-07T21:43:30.769343583Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-07T21:43:30.775289766Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T06:52:01.51126686Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T06:52:01.511449948Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-08T06:52:01.511494997Z INFO controllers.FenceAgentsRemediation CR's deletion timestamp is not zero, and FAR finalizer exists {"CR Name": "worker-0-2"} 2025-09-08T06:52:01.517110233Z INFO taints Taint was removed {"taint effect": "NoExecute", "taint list": null} 2025-09-08T06:52:01.517201154Z INFO controllers.FenceAgentsRemediation FAR remediation taint was removed {"Node Name": "worker-0-2"} 2025-09-08T06:52:01.517385901Z DEBUG events [remediation] Remediation taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"261231"}, "reason": "RemoveRemediationTaint"} 2025-09-08T06:52:01.527784676Z INFO fenceagentsremediation-resource validate update {"name": "worker-0-2"} 2025-09-08T06:52:01.533374419Z INFO controllers.FenceAgentsRemediation Finalizer was removed {"CR Name": "worker-0-2"} 2025-09-08T06:52:01.533481916Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T06:52:01.533676805Z DEBUG events [remediation] Finalizer was removed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"cf4eda2d-1ef2-43f0-ac07-2a88117e60d2","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"261362"}, "reason": "RemoveFinalizer"} 2025-09-08T06:52:01.534659303Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T06:52:01.53468572Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR was not found {"CR Name": "worker-0-2", "CR Namespace": "openshift-workload-availability"} 2025-09-08T06:52:01.534690159Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T06:54:40.138175099Z INFO fenceagentsremediation-resource validate create {"name": "worker-0-2"} 2025-09-08T06:54:40.14171472Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T06:54:40.141737723Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-08T06:54:40.150353921Z INFO controllers.FenceAgentsRemediation Finalizer was added {"CR Name": "worker-0-2"} 2025-09-08T06:54:40.150429147Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "True", "fenceAgentActionSucceededConditionStatus": "Unknown", "succeededConditionStatus": "Unknown", "reason": "RemediationStarted", "LastUpdateTime": "2025-09-08T06:54:40.150428177Z"} 2025-09-08T06:54:40.15043157Z DEBUG events [remediation] Remediation started {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"1e7527f1-12ad-4353-8d21-f533d1689b0b","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"262580"}, "reason": "RemediationStarted"} 2025-09-08T06:54:40.150531544Z DEBUG events [remediation] Finalizer was added {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"1e7527f1-12ad-4353-8d21-f533d1689b0b","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"262580"}, "reason": "AddFinalizer"} 2025-09-08T06:54:40.356118518Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T06:54:40.356246625Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T06:54:40.356261753Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-08T06:54:40.362736818Z INFO taints Taint was added {"taint effect": "NoExecute", "taint list": [{"key":"medik8s.io/fence-agents-remediation","effect":"NoExecute","timeAdded":"2025-09-08T06:54:40Z"}]} 2025-09-08T06:54:40.362774559Z INFO controllers.FenceAgentsRemediation FAR remediation taint was added {"Node Name": "worker-0-2"} 2025-09-08T06:54:40.362817823Z INFO controllers.FenceAgentsRemediation Build fence agent command line {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2"} 2025-09-08T06:54:40.36289588Z INFO controllers.FenceAgentsRemediation Execute the fence agent {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "1e7527f1-12ad-4353-8d21-f533d1689b0b", "ParametersError": "json: unsupported type: iter.Seq[github.com/medik8s/fence-agents-remediation/api/v1alpha1.ParameterName]"} 2025-09-08T06:54:40.363221248Z DEBUG events [remediation] Remediation taint was added {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"261953"}, "reason": "AddRemediationTaint"} 2025-09-08T06:54:40.363250629Z DEBUG events [remediation] Fence agent was executed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"1e7527f1-12ad-4353-8d21-f533d1689b0b","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"262581"}, "reason": "FenceAgentExecuted"} 2025-09-08T06:54:40.363260547Z INFO executer fence agent start {"uid": "1e7527f1-12ad-4353-8d21-f533d1689b0b", "fence_agent": "fence_ipmilan", "retryCount": 5, "retryInterval": "10s", "timeout": "5m0s"} 2025-09-08T06:54:40.371806268Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T06:54:40.37187228Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T06:54:40.371890831Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-08T06:54:40.371923874Z INFO controllers.FenceAgentsRemediation A Fence Agent is already running {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "1e7527f1-12ad-4353-8d21-f533d1689b0b"} 2025-09-08T06:54:40.380707388Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T06:54:40.437615771Z INFO executer command failed {"uid": "1e7527f1-12ad-4353-8d21-f533d1689b0b", "response": "", "errMessage": "2025-09-08 06:54:40,430 ERROR: Failed: Unable to obtain correct plug status or plug is not available\n\n\n", "err": "exit status 1"} 2025-09-08T06:54:50.502823732Z INFO executer command failed {"uid": "1e7527f1-12ad-4353-8d21-f533d1689b0b", "response": "", "errMessage": "2025-09-08 06:54:50,494 ERROR: Failed: Unable to obtain correct plug status or plug is not available\n\n\n", "err": "exit status 1"} 2025-09-08T06:55:00.598445602Z INFO executer command failed {"uid": "1e7527f1-12ad-4353-8d21-f533d1689b0b", "response": "", "errMessage": "2025-09-08 06:55:00,559 ERROR: Failed: Unable to obtain correct plug status or plug is not available\n\n\n", "err": "exit status 1"} 2025-09-08T06:55:10.693910613Z INFO executer command failed {"uid": "1e7527f1-12ad-4353-8d21-f533d1689b0b", "response": "", "errMessage": "2025-09-08 06:55:10,646 ERROR: Failed: Unable to obtain correct plug status or plug is not available\n\n\n", "err": "exit status 1"} 2025-09-08T06:55:20.797586304Z INFO executer command failed {"uid": "1e7527f1-12ad-4353-8d21-f533d1689b0b", "response": "", "errMessage": "2025-09-08 06:55:20,745 ERROR: Failed: Unable to obtain correct plug status or plug is not available\n\n\n", "err": "exit status 1"} 2025-09-08T06:55:20.797680263Z INFO executer fence agent done {"uid": "1e7527f1-12ad-4353-8d21-f533d1689b0b", "fence_agent": "fence_ipmilan", "stdout": "", "stderr": "2025-09-08 06:55:20,745 ERROR: Failed: Unable to obtain correct plug status or plug is not available\n\n\n", "err": "exit status 1"} 2025-09-08T06:55:20.797702233Z INFO executer fence agent context timed out 2025-09-08T06:55:20.797712645Z INFO executer updating status {"FAR uid": "1e7527f1-12ad-4353-8d21-f533d1689b0b"} 2025-09-08T06:55:20.797759394Z INFO executer Updating Status Condition {"processingConditionStatus": "False", "fenceAgentActionSucceededConditionStatus": "False", "succeededConditionStatus": "False", "reason": "FenceAgentFailed", "LastUpdateTime": "2025-09-08T06:55:20.797758573Z"} 2025-09-08T06:55:20.802949363Z INFO executer status updated {"FAR uid": "1e7527f1-12ad-4353-8d21-f533d1689b0b"} 2025-09-08T06:55:20.803216625Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T06:55:20.803238169Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-08T06:55:20.80818952Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T06:59:07.125316189Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T06:59:07.125369799Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-08T06:59:07.131605062Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T07:09:47.97293327Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T07:09:47.972986179Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-08T07:09:47.973032315Z INFO controllers.FenceAgentsRemediation CR's deletion timestamp is not zero, and FAR finalizer exists {"CR Name": "worker-0-2"} 2025-09-08T07:09:47.973036847Z INFO controllers.FenceAgentsRemediation FAR didn't finish remediate the node {"CR Name": "worker-0-2", "processing condition": "False", "fenceAgentActionSucceeded condition": "False", "succeeded condition": "False"} 2025-09-08T07:09:47.973042646Z INFO executer cancelling fence agent routine {"uid": "1e7527f1-12ad-4353-8d21-f533d1689b0b"} 2025-09-08T07:09:47.979258717Z INFO taints Taint was removed {"taint effect": "NoExecute", "taint list": null} 2025-09-08T07:09:47.979342116Z INFO controllers.FenceAgentsRemediation FAR remediation taint was removed {"Node Name": "worker-0-2"} 2025-09-08T07:09:47.979601022Z DEBUG events [remediation] Remediation taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"268590"}, "reason": "RemoveRemediationTaint"} 2025-09-08T07:09:47.99588336Z INFO fenceagentsremediation-resource validate update {"name": "worker-0-2"} 2025-09-08T07:09:48.000623227Z INFO controllers.FenceAgentsRemediation Finalizer was removed {"CR Name": "worker-0-2"} 2025-09-08T07:09:48.000684582Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T07:09:48.000704494Z DEBUG events [remediation] Finalizer was removed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"1e7527f1-12ad-4353-8d21-f533d1689b0b","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"268615"}, "reason": "RemoveFinalizer"} 2025-09-08T07:09:48.000886333Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T07:09:48.000917398Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR was not found {"CR Name": "worker-0-2", "CR Namespace": "openshift-workload-availability"} 2025-09-08T07:09:48.000923852Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T07:11:44.506340899Z INFO fenceagentsremediation-resource validate create {"name": "worker-0-2"} 2025-09-08T07:11:44.510696077Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T07:11:44.510724227Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-08T07:11:44.51904703Z INFO controllers.FenceAgentsRemediation Finalizer was added {"CR Name": "worker-0-2"} 2025-09-08T07:11:44.519072868Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "True", "fenceAgentActionSucceededConditionStatus": "Unknown", "succeededConditionStatus": "Unknown", "reason": "RemediationStarted", "LastUpdateTime": "2025-09-08T07:11:44.519071801Z"} 2025-09-08T07:11:44.519108619Z DEBUG events [remediation] Remediation started {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"e2e9aecf-8819-4fe1-879e-f14a097a4804","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"269663"}, "reason": "RemediationStarted"} 2025-09-08T07:11:44.51914225Z DEBUG events [remediation] Finalizer was added {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"e2e9aecf-8819-4fe1-879e-f14a097a4804","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"269663"}, "reason": "AddFinalizer"} 2025-09-08T07:11:44.726221918Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T07:11:44.726325556Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T07:11:44.72634564Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-08T07:11:44.732969686Z INFO taints Taint was added {"taint effect": "NoExecute", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-09-08T07:11:14Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-09-08T07:11:19Z"},{"key":"medik8s.io/fence-agents-remediation","effect":"NoExecute","timeAdded":"2025-09-08T07:11:44Z"}]} 2025-09-08T07:11:44.733009472Z INFO controllers.FenceAgentsRemediation FAR remediation taint was added {"Node Name": "worker-0-2"} 2025-09-08T07:11:44.733032079Z INFO controllers.FenceAgentsRemediation Build fence agent command line {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2"} 2025-09-08T07:11:44.733062292Z INFO controllers.FenceAgentsRemediation found a value from secret {"secret name": "worker-2-cred", "parameter name": "--password"} 2025-09-08T07:11:44.733091043Z INFO controllers.FenceAgentsRemediation Execute the fence agent {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "e2e9aecf-8819-4fe1-879e-f14a097a4804", "ParametersError": "json: unsupported type: iter.Seq[github.com/medik8s/fence-agents-remediation/api/v1alpha1.ParameterName]"} 2025-09-08T07:11:44.733323671Z DEBUG events [remediation] Remediation taint was added {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"269466"}, "reason": "AddRemediationTaint"} 2025-09-08T07:11:44.733348745Z DEBUG events [remediation] Fence agent was executed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"e2e9aecf-8819-4fe1-879e-f14a097a4804","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"269665"}, "reason": "FenceAgentExecuted"} 2025-09-08T07:11:44.733336193Z INFO executer fence agent start {"uid": "e2e9aecf-8819-4fe1-879e-f14a097a4804", "fence_agent": "fence_ipmilan", "retryCount": 5, "retryInterval": "10s", "timeout": "5m0s"} 2025-09-08T07:11:44.739890161Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T07:11:44.73995471Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T07:11:44.739969507Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-08T07:11:44.740012859Z INFO controllers.FenceAgentsRemediation A Fence Agent is already running {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "e2e9aecf-8819-4fe1-879e-f14a097a4804"} 2025-09-08T07:11:44.748068122Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T07:11:49.682536327Z INFO executer command completed {"uid": "e2e9aecf-8819-4fe1-879e-f14a097a4804", "response": "Success: Rebooted\n", "errMessage": "", "err": null} 2025-09-08T07:11:49.682584768Z INFO executer fence agent done {"uid": "e2e9aecf-8819-4fe1-879e-f14a097a4804", "fence_agent": "fence_ipmilan", "stdout": "Success: Rebooted\n", "stderr": "", "err": null} 2025-09-08T07:11:49.682589591Z INFO executer updating status {"FAR uid": "e2e9aecf-8819-4fe1-879e-f14a097a4804"} 2025-09-08T07:11:49.682654789Z INFO executer Updating Status Condition {"processingConditionStatus": "", "fenceAgentActionSucceededConditionStatus": "True", "succeededConditionStatus": "", "reason": "FenceAgentSucceeded", "LastUpdateTime": "2025-09-08T07:11:49.682653984Z"} 2025-09-08T07:11:49.68296967Z DEBUG events [remediation] Fence agent was succeeded {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"e2e9aecf-8819-4fe1-879e-f14a097a4804","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"269665"}, "reason": "FenceAgentSucceeded"} 2025-09-08T07:11:49.68796394Z INFO executer status updated {"FAR uid": "e2e9aecf-8819-4fe1-879e-f14a097a4804"} 2025-09-08T07:11:49.688382289Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T07:11:49.688404882Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-08T07:11:49.688440638Z INFO controllers.FenceAgentsRemediation Remediation strategy is ResourceDeletion which explicitly deletes resources - manually deleting workload {"Node Name": "worker-0-2"} 2025-09-08T07:11:49.688781865Z INFO commons-resource starting to delete pods {"node name": "worker-0-2"} 2025-09-08T07:11:49.688920373Z DEBUG events [remediation] Manually delete pods from the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"269669"}, "reason": "DeleteResources"} 2025-09-08T07:11:51.841510974Z INFO commons-resource done deleting pods {"node name": "worker-0-2"} 2025-09-08T07:11:51.841536911Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "False", "fenceAgentActionSucceededConditionStatus": "", "succeededConditionStatus": "True", "reason": "RemediationFinishedSuccessfully", "LastUpdateTime": "2025-09-08T07:11:51.841536122Z"} 2025-09-08T07:11:51.841549024Z INFO executer cancelling fence agent routine {"uid": "e2e9aecf-8819-4fe1-879e-f14a097a4804"} 2025-09-08T07:11:51.84155516Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR has completed to remediate the node {"Node Name": "worker-0-2"} 2025-09-08T07:11:51.841685869Z DEBUG events [remediation] Unhealthy node remediation was completed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"dd6cacd3-cb5f-4f66-b1f8-144000228ef0","apiVersion":"v1","resourceVersion":"269669"}, "reason": "NodeRemediationCompleted"} 2025-09-08T07:11:51.841719138Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2","uid":"e2e9aecf-8819-4fe1-879e-f14a097a4804","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"269724"}, "reason": "RemediationFinished"} 2025-09-08T07:11:52.047625252Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-08T07:11:52.047689463Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-08T07:11:52.047704228Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-08T07:11:52.053368151Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile