[kni@cert-rhosp-02 ~]$ oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.20.0-0.nightly-2025-09-14-115638 True False 3h56m Cluster version is 4.20.0-0.nightly-2025-09-14-115638 [kni@cert-rhosp-02 ~]$ oc get csv NAME DISPLAY VERSION REPLACES PHASE fence-agents-remediation.v0.6.0 Fence Agents Remediation Operator 0.6.0 fence-agents-remediation.v0.5.1 Succeeded node-healthcheck-operator.v0.10.0 Node Health Check Operator 0.10.0 node-healthcheck-operator.v0.9.1 Succeeded [kni@cert-rhosp-02 ~]$ PODS=$(oc get pods -o name -n openshift-workload-availability | grep fence-agents-remediation-controller-manager) [kni@cert-rhosp-02 ~]$ echo $PODS pod/fence-agents-remediation-controller-manager-569dff989d-rsf6m pod/fence-agents-remediation-controller-manager-569dff989d-wxphl [kni@cert-rhosp-02 ~]$ for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/fence-agents-remediation-controller-manager-569dff989d-rsf6m "worker-0-0" == pod/fence-agents-remediation-controller-manager-569dff989d-wxphl "worker-0-2" [kni@cert-rhosp-02 ~]$ PODS=$(oc get pods -o name -n openshift-workload-availability | grep node-healthcheck-controller-manager) [kni@cert-rhosp-02 ~]$ echo $PODS pod/node-healthcheck-controller-manager-7548bcd566-4j9bc pod/node-healthcheck-controller-manager-7548bcd566-w42wm [kni@cert-rhosp-02 ~]$ for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/node-healthcheck-controller-manager-7548bcd566-4j9bc "master-0-1" == pod/node-healthcheck-controller-manager-7548bcd566-w42wm "master-0-2" [kni@cert-rhosp-02 ~]$ oc delete pod fence-agents-remediation-controller-manager-569dff989d-rsf6m pod "fence-agents-remediation-controller-manager-569dff989d-rsf6m" deleted [kni@cert-rhosp-02 ~]$ oc delete pod node-healthcheck-controller-manager-7548bcd566-4j9bc pod "node-healthcheck-controller-manager-7548bcd566-4j9bc" deleted [kni@cert-rhosp-02 ~]$ [kni@cert-rhosp-02 ~]$ PODS=$(oc get pods -o name -n openshift-workload-availability | grep fence-agents-remediation-controller-manager) [kni@cert-rhosp-02 ~]$ echo $PODS pod/fence-agents-remediation-controller-manager-569dff989d-wxphl pod/fence-agents-remediation-controller-manager-569dff989d-xg92x [kni@cert-rhosp-02 ~]$ for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/fence-agents-remediation-controller-manager-569dff989d-wxphl "worker-0-2" == pod/fence-agents-remediation-controller-manager-569dff989d-xg92x "worker-0-1" [kni@cert-rhosp-02 ~]$ PODS=$(oc get pods -o name -n openshift-workload-availability | grep node-healthcheck-controller-manager) [kni@cert-rhosp-02 ~]$ echo $PODS pod/node-healthcheck-controller-manager-7548bcd566-kfr4h pod/node-healthcheck-controller-manager-7548bcd566-w42wm [kni@cert-rhosp-02 ~]$ for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/node-healthcheck-controller-manager-7548bcd566-kfr4h "master-0-1" == pod/node-healthcheck-controller-manager-7548bcd566-w42wm "master-0-2" [kni@cert-rhosp-02 ~]$ oc get pod node-healthcheck-controller-manager-7548bcd566-kfr4h -o json | jq .spec.affinity { "nodeAffinity": { "preferredDuringSchedulingIgnoredDuringExecution": [ { "preference": { "matchExpressions": [ { "key": "node-role.kubernetes.io/infra", "operator": "Exists" } ] }, "weight": 3 }, { "preference": { "matchExpressions": [ { "key": "node-role.kubernetes.io/master", "operator": "Exists" } ] }, "weight": 1 }, { "preference": { "matchExpressions": [ { "key": "node-role.kubernetes.io/control-plane", "operator": "Exists" } ] }, "weight": 1 } ] } } [kni@cert-rhosp-02 ~]$ vi test.yaml [kni@cert-rhosp-02 ~]$ cat test.yaml apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate metadata: name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability spec: template: spec: agent: fence_ipmilan retrycount: 5 retryinterval: 10s timeout: 300s nodeparameters: '--ipport': master-0-0: '6230' master-0-1: '6231' master-0-2: '6232' worker-0-0: '6233' worker-0-1: '6234' worker-0-2: '6235' sharedparameters: '--action': reboot '--lanplus': '' '--ip': 192.168.123.1 '--password': password '--username': admin --- apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: name: nhc-far-worker spec: maxUnhealthy: 2 healthyDelay: 500s remediationTemplate: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/control-plane operator: DoesNotExist values: [] - key: node-role.kubernetes.io/master operator: DoesNotExist values: [] unhealthyConditions: - duration: 30s status: 'False' type: Ready - duration: 30s status: Unknown type: Ready [kni@cert-rhosp-02 ~]$ oc apply -f test.yaml fenceagentsremediationtemplate.fence-agents-remediation.medik8s.io/fenceagentsremediationtemplate-test created nodehealthcheck.remediation.medik8s.io/nhc-far-worker created [kni@cert-rhosp-02 ~]$ oc debug node/worker-0-2 -- chroot /host bash -c "date & uptime -s" Temporary namespace openshift-debug-pr7lc is created for debugging node... Starting pod/worker-0-2-debug-zg74j ... To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`. Mon Sep 15 15:35:49 UTC 2025 2025-09-15 11:01:54 Removing debug pod ... Temporary namespace openshift-debug-pr7lc was removed. [kni@cert-rhosp-02 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 4h33m v1.33.4 worker-0-1 Ready worker 4h33m v1.33.4 worker-0-2 Ready worker 4h33m v1.33.4 [kni@cert-rhosp-02 ~]$ oc debug node/worker-0-2 -- chroot /host bash -c "systemctl stop kubelet" Temporary namespace openshift-debug-2l2q6 is created for debugging node... Starting pod/worker-0-2-debug-wsfzw ... To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`. [kni@cert-rhosp-02 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 4h34m v1.33.4 worker-0-1 Ready worker 4h33m v1.33.4 worker-0-2 NotReady worker 4h34m v1.33.4 [kni@cert-rhosp-02 ~]$ oc get far No resources found in openshift-workload-availability namespace. [kni@cert-rhosp-02 ~]$ oc get nhc -o yaml apiVersion: v1 items: - apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","metadata":{"annotations":{},"name":"nhc-far-worker"},"spec":{"healthyDelay":"500s","maxUnhealthy":2,"remediationTemplate":{"apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","kind":"FenceAgentsRemediationTemplate","name":"fenceagentsremediationtemplate-test","namespace":"openshift-workload-availability"},"selector":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"DoesNotExist","values":[]},{"key":"node-role.kubernetes.io/master","operator":"DoesNotExist","values":[]}]},"unhealthyConditions":[{"duration":"30s","status":"False","type":"Ready"},{"duration":"30s","status":"Unknown","type":"Ready"}]}} creationTimestamp: "2025-09-15T15:35:14Z" generation: 1 name: nhc-far-worker resourceVersion: "110885" uid: c83c3e72-a0c2-46e9-8267-11ccedab5357 spec: healthyDelay: 500s maxUnhealthy: 2 remediationTemplate: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/control-plane operator: DoesNotExist values: [] - key: node-role.kubernetes.io/master operator: DoesNotExist values: [] unhealthyConditions: - duration: 30s status: "False" type: Ready - duration: 30s status: Unknown type: Ready status: conditions: - lastTransitionTime: "2025-09-15T15:35:14Z" message: No issues found, NodeHealthCheck is enabled. reason: NodeHealthCheckEnabled status: "False" type: Disabled healthyNodes: 2 lastUpdateTime: "2025-09-15T15:38:02Z" observedNodes: 3 phase: Remediating reason: NHC is remediating 1 nodes unhealthyNodes: - name: worker-0-2 remediations: - resource: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediation name: worker-0-2-lbzgm namespace: openshift-workload-availability uid: ae5c04e0-f5eb-4ec1-888f-efda126568ee started: "2025-09-15T15:38:02Z" templateName: fenceagentsremediationtemplate-test kind: List metadata: resourceVersion: "" [kni@cert-rhosp-02 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 4h36m v1.33.4 worker-0-1 Ready worker 4h36m v1.33.4 worker-0-2 Ready worker 4h36m v1.33.4 [kni@cert-rhosp-02 ~]$ oc get far NAME AGE worker-0-2-lbzgm 115s [kni@cert-rhosp-02 ~]$ oc get far NAME AGE worker-0-2-lbzgm 9m31s [kni@cert-rhosp-02 ~]$ oc get far No resources found in openshift-workload-availability namespace. NHC Logs: 2025-09-15T14:40:56.320436521Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-09-15T14:40:56.320544223Z INFO setup Go OS/Arch: linux/amd64 2025-09-15T14:40:56.320547669Z INFO setup Operator Version: 2ef588d 2025-09-15T14:40:56.320549536Z INFO setup Git Commit: 2ef588d65de4087c46447401cd6e757459d79210 2025-09-15T14:40:56.320551097Z INFO setup Build Date: 2025-09-01T17:55:11+00:00 2025-09-15T14:40:56.320552767Z INFO setup HTTP/2 for metrics and webhook server disabled 2025-09-15T14:40:56.34124507Z INFO setup supported control plane topology {"topology": "HighlyAvailable"} 2025-09-15T14:40:56.341276329Z INFO setup Cluster capabilities {"IsOnOpenshift": true, "HasMachineAPI": true} 2025-09-15T14:40:56.341508126Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "remediation.medik8s.io/v1alpha1, Kind=NodeHealthCheck", "path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2025-09-15T14:40:56.34158068Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2025-09-15T14:40:56.341617021Z INFO setup starting manager 2025-09-15T14:40:56.341663286Z INFO controller-runtime.metrics Starting metrics server 2025-09-15T14:40:56.341702815Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-09-15T14:40:56.341732569Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": "127.0.0.1:8080", "secure": false} 2025-09-15T14:40:56.341764013Z INFO controller-runtime.webhook Starting webhook server 2025-09-15T14:40:56.341960708Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-09-15T14:40:56.34202505Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-09-15T14:40:56.342100254Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} I0915 14:40:56.442900 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/e1f13584.medik8s.io... I0915 15:30:54.791458 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/e1f13584.medik8s.io 2025-09-15T15:30:54.791523093Z DEBUG events node-healthcheck-controller-manager-7548bcd566-w42wm_d1d62f61-acb8-48c2-b219-c8e972c8fa1a became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"e1f13584.medik8s.io","uid":"9b90af02-d3d0-4553-bc44-07f67b459346","apiVersion":"coordination.k8s.io/v1","resourceVersion":"107919"}, "reason": "LeaderElection"} 2025-09-15T15:30:54.791749954Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "channel source: 0xc000742070"} 2025-09-15T15:30:54.791793026Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "channel source: 0xc000742380"} 2025-09-15T15:30:54.791798222Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.Machine"} 2025-09-15T15:30:54.791788396Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1alpha1.NodeHealthCheck"} 2025-09-15T15:30:54.791813968Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.MachineHealthCheck"} 2025-09-15T15:30:54.79179586Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1.Node"} 2025-09-15T15:30:54.791789443Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1.Node"} I0915 15:30:54.792504 1 shared_informer.go:313] Waiting for caches to sync for feature gate accessor I0915 15:30:54.892597 1 shared_informer.go:320] Caches are synced for feature gate accessor I0915 15:30:54.892635 1 simple_featuregate_reader.go:171] Starting feature-gate-detector 2025-09-15T15:30:54.892599799Z INFO MHCChecker found termination handler MHC, will ignore Nodes with Terminating condition 2025-09-15T15:30:54.892651459Z INFO MHCChecker MHC Checker status changed, notifying NHC controller 2025-09-15T15:30:54.892647647Z INFO Starting Controller {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck"} 2025-09-15T15:30:54.892670046Z INFO Starting workers {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "worker count": 1} 2025-09-15T15:30:54.892607926Z INFO Starting Controller {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck"} 2025-09-15T15:30:54.89268715Z INFO Starting workers {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "worker count": 1} 2025-09-15T15:30:54.892750812Z INFO FeatureGateAccessor FeatureGates initialized 2025-09-15T15:30:54.892777955Z INFO adding all NHCs to reconcile queue for handling MHC event I0915 15:30:54.892844 1 recorder_logging.go:49] &Event{ObjectMeta:{dummy.18657e715bb05c38.d3b4e2d2 dummy 0 0001-01-01 00:00:00 +0000 UTC map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Pod,Namespace:dummy,Name:dummy,UID:,APIVersion:v1,ResourceVersion:,FieldPath:,},Reason:FeatureGatesInitialized,Message:FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "NewOLM", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}},Source:EventSource{Component:,Host:,},FirstTimestamp:2025-09-15 15:30:54.892735544 +0000 UTC m=+2998.593846172,LastTimestamp:2025-09-15 15:30:54.892735544 +0000 UTC m=+2998.593846172,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:,ReportingInstance:,} 2025-09-15T15:30:55.20514154Z INFO console-plugin successfully created / updated console plugin resources 2025-09-15T15:35:14.561596048Z INFO nodehealthcheck-resource validate create {"name": "nhc-far-worker"} 2025-09-15T15:35:14.564466249Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:35:14.572621055Z INFO controllers.NodeHealthCheck enabling NHC, valid config, no conflicting MHC configured in the cluster {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:35:14.572749464Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-09-15T15:35:14.572842688Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation template CRs {"kind": "FenceAgentsRemediationTemplate"} 2025-09-15T15:35:14.572867502Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-09-15T15:35:14.572891688Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation CRs {"kind": "FenceAgentsRemediation"} 2025-09-15T15:35:14.572794731Z DEBUG events [remediation] No issues found, NodeHealthCheck is enabled. {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"109537"}, "reason": "Enabled"} 2025-09-15T15:35:14.579102297Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:35:14Z" level=info msg="invalidating lease" time="2025-09-15T15:35:14Z" level=info msg="getting lease" 2025-09-15T15:35:14.681855706Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:35:14Z" level=info msg="invalidating lease" time="2025-09-15T15:35:14Z" level=info msg="getting lease" 2025-09-15T15:35:14.684234877Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-15T15:35:14Z" level=info msg="invalidating lease" time="2025-09-15T15:35:14Z" level=info msg="getting lease" 2025-09-15T15:35:14.685938083Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-far-worker", "new status": {"observedNodes":3,"healthyNodes":3,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-09-15T15:35:14Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation"}, "patch": "{\"status\":{\"conditions\":[{\"lastTransitionTime\":\"2025-09-15T15:35:14Z\",\"message\":\"No issues found, NodeHealthCheck is enabled.\",\"reason\":\"NodeHealthCheckEnabled\",\"status\":\"False\",\"type\":\"Disabled\"}],\"healthyNodes\":3,\"observedNodes\":3,\"phase\":\"Enabled\",\"reason\":\"NHC is enabled, no ongoing remediation\"}}"} 2025-09-15T15:35:14.891419197Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-09-15T15:37:31.906082072Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-far-worker"} 2025-09-15T15:37:31.906176154Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-far-worker"} 2025-09-15T15:37:31.906230126Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:37:31.915487086Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown", "duration left": "29.084514696s"} 2025-09-15T15:37:31.928600902Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:37:32Z" level=info msg="invalidating lease" time="2025-09-15T15:37:32Z" level=info msg="getting lease" 2025-09-15T15:37:32.031766013Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:37:32Z" level=info msg="invalidating lease" time="2025-09-15T15:37:32Z" level=info msg="getting lease" 2025-09-15T15:37:32.132274953Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-far-worker", "new status": {"observedNodes":3,"healthyNodes":2,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-09-15T15:35:14Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-09-15T15:35:14Z"}, "patch": "{\"status\":{\"healthyNodes\":2}}"} 2025-09-15T15:37:32.344597011Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "30.084514696s"} 2025-09-15T15:38:02.429765528Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:38:02.43434264Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2025-09-15T15:38:02.434503221Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-2", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"110487"}, "reason": "DetectedUnhealthy"} 2025-09-15T15:38:02.438940074Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:38:02Z" level=info msg="invalidating lease" time="2025-09-15T15:38:02Z" level=info msg="getting lease" 2025-09-15T15:38:02.440416941Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:38:02Z" level=info msg="invalidating lease" time="2025-09-15T15:38:02Z" level=info msg="getting lease" 2025-09-15T15:38:02.441783443Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:38:02.445438163Z INFO controllers.NodeHealthCheck.resource manager Attempting to obtain Node Lease {"Node name": "worker-0-2"} time="2025-09-15T15:38:02Z" level=info msg="request lease" time="2025-09-15T15:38:02Z" level=info msg="getting lease" time="2025-09-15T15:38:02Z" level=info msg="create lease" 2025-09-15T15:38:02.448129526Z INFO controllers.NodeHealthCheck.resource manager Creating a remediation CR {"CR name": "", "CR kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} 2025-09-15T15:38:02.456445148Z DEBUG events [remediation] Created remediation object for node worker-0-2 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"110487"}, "reason": "RemediationCreated"} 2025-09-15T15:38:02.458598381Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-far-worker", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-2","remediations":[{"resource":{"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-lbzgm","uid":"ae5c04e0-f5eb-4ec1-888f-efda126568ee","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1"},"started":"2025-09-15T15:38:02Z","templateName":"fenceagentsremediationtemplate-test"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-09-15T15:35:14Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-09-15T15:37:32Z"}, "patch": "{\"status\":{\"phase\":\"Remediating\",\"reason\":\"NHC is remediating 1 nodes\",\"unhealthyNodes\":[{\"name\":\"worker-0-2\",\"remediations\":[{\"resource\":{\"apiVersion\":\"fence-agents-remediation.medik8s.io/v1alpha1\",\"kind\":\"FenceAgentsRemediation\",\"name\":\"worker-0-2-lbzgm\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"ae5c04e0-f5eb-4ec1-888f-efda126568ee\"},\"started\":\"2025-09-15T15:38:02Z\",\"templateName\":\"fenceagentsremediationtemplate-test\"}]}]}}"} 2025-09-15T15:38:02.569625831Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:38:02.569718551Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-15T15:38:02.569734044Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:38:02.569739652Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-15T15:38:02.574366059Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:38:02.57439859Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-15T15:38:02.574407681Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:38:02.574412112Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-15T15:38:02.66432404Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-09-15T15:38:02.664405858Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:38:02.668861161Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2025-09-15T15:38:02.672783434Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:02Z"} 2025-09-15T15:38:02.672819116Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:38:02Z" level=info msg="invalidating lease" time="2025-09-15T15:38:02Z" level=info msg="getting lease" 2025-09-15T15:38:02.674699929Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:38:02Z" level=info msg="invalidating lease" time="2025-09-15T15:38:02Z" level=info msg="getting lease" 2025-09-15T15:38:02.676540449Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:38:02.680051674Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-2-lbzgm", "kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} time="2025-09-15T15:38:02Z" level=info msg="getting lease" 2025-09-15T15:38:02.680118405Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-far-worker", "lease name": "node-worker-0-2", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2025-09-15T15:38:02.683546348Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-09-15T15:38:24.168838282Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:38:24.168903149Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-15T15:38:24.16892001Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:38:24.168927039Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-15T15:38:24.168965613Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:38:24.174131878Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2025-09-15T15:38:24.178250481Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:02Z"} 2025-09-15T15:38:24.178273642Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:38:24Z" level=info msg="invalidating lease" time="2025-09-15T15:38:24Z" level=info msg="getting lease" 2025-09-15T15:38:24.180120403Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:38:24Z" level=info msg="invalidating lease" time="2025-09-15T15:38:24Z" level=info msg="getting lease" 2025-09-15T15:38:24.181845653Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:38:24.185220603Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-2-lbzgm", "kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} time="2025-09-15T15:38:24Z" level=info msg="getting lease" 2025-09-15T15:38:24.185271762Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-far-worker", "lease name": "node-worker-0-2", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2025-09-15T15:38:24.18715903Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-09-15T15:38:26.629755256Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:38:26.629800655Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-15T15:38:26.629821337Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:38:26.629830524Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-15T15:38:26.629876965Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:38:26.63737271Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2025-09-15T15:38:26.642255293Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:38:26.642285813Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:38:26Z" level=info msg="invalidating lease" time="2025-09-15T15:38:26Z" level=info msg="getting lease" 2025-09-15T15:38:26.644231904Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:38:26Z" level=info msg="invalidating lease" time="2025-09-15T15:38:26Z" level=info msg="getting lease" 2025-09-15T15:38:26.645943392Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:38:26.649505474Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-2-lbzgm", "kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} time="2025-09-15T15:38:26Z" level=info msg="getting lease" 2025-09-15T15:38:26.64956224Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-far-worker", "lease name": "node-worker-0-2", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2025-09-15T15:38:26.651577297Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-09-15T15:39:37.128319313Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-far-worker"} 2025-09-15T15:39:37.12847132Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:39:37.129124494Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-far-worker"} 2025-09-15T15:39:37.135183552Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "False", "duration left": "29.864818245s"} 2025-09-15T15:39:37.145312211Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:39:37.14535141Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:39:37Z" level=info msg="invalidating lease" time="2025-09-15T15:39:37Z" level=info msg="getting lease" 2025-09-15T15:39:37.148794065Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:39:37Z" level=info msg="invalidating lease" time="2025-09-15T15:39:37Z" level=info msg="getting lease" 2025-09-15T15:39:37.248926195Z INFO controllers.NodeHealthCheck Ignoring node, because it was unhealthy, and is likely to be unhealthy again. {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:39:37.249208085Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "30.864818245s"} 2025-09-15T15:39:39.652759686Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-far-worker"} 2025-09-15T15:39:39.652834604Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-far-worker"} 2025-09-15T15:39:39.652883018Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:39:39.672513003Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:39:39.672547078Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:39:39Z" level=info msg="invalidating lease" time="2025-09-15T15:39:39Z" level=info msg="getting lease" 2025-09-15T15:39:39.675042017Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:39:39Z" level=info msg="invalidating lease" time="2025-09-15T15:39:39Z" level=info msg="getting lease" 2025-09-15T15:39:39.677509586Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:39:39.679901751Z INFO controllers.NodeHealthCheck.resource manager setting a delay for node getting healthy {"node name": "worker-0-2", "delay in seconds": 500} 2025-09-15T15:39:39.692046902Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:39:39.692111598Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-15T15:39:39.692122101Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:39:39.69212612Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-15T15:39:39.692370913Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-far-worker", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-2","remediations":[{"resource":{"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-lbzgm","uid":"ae5c04e0-f5eb-4ec1-888f-efda126568ee","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1"},"started":"2025-09-15T15:38:02Z","templateName":"fenceagentsremediationtemplate-test"}],"conditionsHealthyTimestamp":"2025-09-15T15:39:39Z","healthyDelayed":true}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-09-15T15:35:14Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-09-15T15:38:02Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"conditionsHealthyTimestamp\":\"2025-09-15T15:39:39Z\",\"healthyDelayed\":true,\"name\":\"worker-0-2\",\"remediations\":[{\"resource\":{\"apiVersion\":\"fence-agents-remediation.medik8s.io/v1alpha1\",\"kind\":\"FenceAgentsRemediation\",\"name\":\"worker-0-2-lbzgm\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"ae5c04e0-f5eb-4ec1-888f-efda126568ee\"},\"started\":\"2025-09-15T15:38:02Z\",\"templateName\":\"fenceagentsremediationtemplate-test\"}]}]}}"} 2025-09-15T15:39:39.899227773Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:39:39.899341537Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:39:39.907819684Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:39:39.907849372Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:39:39Z" level=info msg="invalidating lease" time="2025-09-15T15:39:39Z" level=info msg="getting lease" 2025-09-15T15:39:39.909556841Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:39:39Z" level=info msg="invalidating lease" time="2025-09-15T15:39:39Z" level=info msg="getting lease" 2025-09-15T15:39:39.911455988Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:39:39.913497382Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 499.086503921} 2025-09-15T15:39:39.91367964Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:39:50.900006776Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:39:50.908758469Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:39:50.90879266Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:39:50Z" level=info msg="invalidating lease" time="2025-09-15T15:39:50Z" level=info msg="getting lease" 2025-09-15T15:39:50.910701717Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:39:50Z" level=info msg="invalidating lease" time="2025-09-15T15:39:50Z" level=info msg="getting lease" 2025-09-15T15:39:50.912929324Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:39:50.914717946Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 488.085283649} 2025-09-15T15:39:50.914926523Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:40:01.915585779Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:40:01.925238226Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:40:01.925266025Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:40:01.927262575Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 477.072738881} 2025-09-15T15:40:01.927281878Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:40:01Z" level=info msg="invalidating lease" time="2025-09-15T15:40:01Z" level=info msg="getting lease" 2025-09-15T15:40:01.929018033Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:40:01Z" level=info msg="invalidating lease" time="2025-09-15T15:40:01Z" level=info msg="getting lease" 2025-09-15T15:40:01.930839488Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:40:12.9313907Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:40:12.940878748Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:40:12.940915037Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:40:12Z" level=info msg="invalidating lease" time="2025-09-15T15:40:12Z" level=info msg="getting lease" 2025-09-15T15:40:12.943455872Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:40:12Z" level=info msg="invalidating lease" time="2025-09-15T15:40:12Z" level=info msg="getting lease" 2025-09-15T15:40:12.945287853Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:40:12.947181772Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 466.052819675} 2025-09-15T15:40:12.947423931Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:40:23.948120829Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:40:23.957219873Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:40:23.957249224Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:40:23Z" level=info msg="invalidating lease" time="2025-09-15T15:40:23Z" level=info msg="getting lease" 2025-09-15T15:40:23.959445834Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:40:23Z" level=info msg="invalidating lease" time="2025-09-15T15:40:23Z" level=info msg="getting lease" 2025-09-15T15:40:23.961411212Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:40:23.963167764Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 455.036833869} 2025-09-15T15:40:23.963457364Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:40:34.96440363Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:40:34.974532438Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:40:34.974571487Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:40:34Z" level=info msg="invalidating lease" time="2025-09-15T15:40:34Z" level=info msg="getting lease" 2025-09-15T15:40:34.976807638Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:40:34Z" level=info msg="invalidating lease" time="2025-09-15T15:40:34Z" level=info msg="getting lease" 2025-09-15T15:40:34.979151786Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:40:34.98100338Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 444.018998811} 2025-09-15T15:40:34.981216558Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:40:45.982093726Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:40:45.992520893Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:40:45.992612258Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:40:45Z" level=info msg="invalidating lease" time="2025-09-15T15:40:45Z" level=info msg="getting lease" 2025-09-15T15:40:45.994957276Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:40:45Z" level=info msg="invalidating lease" time="2025-09-15T15:40:45Z" level=info msg="getting lease" 2025-09-15T15:40:45.996782342Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:40:45.998445507Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 433.001555807} 2025-09-15T15:40:45.998630345Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:40:56.999322714Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:40:57.008533523Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:40:57.008563643Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:40:57Z" level=info msg="invalidating lease" time="2025-09-15T15:40:57Z" level=info msg="getting lease" 2025-09-15T15:40:57.010569328Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:40:57Z" level=info msg="invalidating lease" time="2025-09-15T15:40:57Z" level=info msg="getting lease" 2025-09-15T15:40:57.012484691Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:40:57.014458425Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 421.985542968} 2025-09-15T15:40:57.014662261Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:41:08.015241565Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:41:08.024691803Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:41:08.024720786Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:41:08Z" level=info msg="invalidating lease" time="2025-09-15T15:41:08Z" level=info msg="getting lease" 2025-09-15T15:41:08.026856122Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:41:08Z" level=info msg="invalidating lease" time="2025-09-15T15:41:08Z" level=info msg="getting lease" 2025-09-15T15:41:08.028648934Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:41:08.030419899Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 410.969581715} 2025-09-15T15:41:08.030613341Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:41:19.031412058Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:41:19.039181508Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:41:19.039251205Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:41:19.041248147Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 399.958753585} 2025-09-15T15:41:19.041299321Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:41:19Z" level=info msg="invalidating lease" time="2025-09-15T15:41:19Z" level=info msg="getting lease" 2025-09-15T15:41:19.042914621Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:41:19Z" level=info msg="invalidating lease" time="2025-09-15T15:41:19Z" level=info msg="getting lease" 2025-09-15T15:41:19.044710892Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:41:30.045404392Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:41:30.05431785Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:41:30.054348144Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:41:30Z" level=info msg="invalidating lease" time="2025-09-15T15:41:30Z" level=info msg="getting lease" 2025-09-15T15:41:30.056008678Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:41:30Z" level=info msg="invalidating lease" time="2025-09-15T15:41:30Z" level=info msg="getting lease" 2025-09-15T15:41:30.057669453Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:41:30.059583376Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 388.940417893} 2025-09-15T15:41:30.059804172Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:41:41.0605207Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:41:41.069230985Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:41:41.069265662Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:41:41.071031862Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 377.928969922} 2025-09-15T15:41:41.071071875Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:41:41Z" level=info msg="invalidating lease" time="2025-09-15T15:41:41Z" level=info msg="getting lease" 2025-09-15T15:41:41.073094751Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:41:41Z" level=info msg="invalidating lease" time="2025-09-15T15:41:41Z" level=info msg="getting lease" 2025-09-15T15:41:41.075363699Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:41:52.075916071Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:41:52.084031766Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:41:52.08406673Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:41:52.085658865Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 366.914343172} 2025-09-15T15:41:52.085678432Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:41:52Z" level=info msg="invalidating lease" time="2025-09-15T15:41:52Z" level=info msg="getting lease" 2025-09-15T15:41:52.08749432Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:41:52Z" level=info msg="invalidating lease" time="2025-09-15T15:41:52Z" level=info msg="getting lease" 2025-09-15T15:41:52.089583073Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:42:03.089723144Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:42:03.098921339Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:42:03.098950603Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:42:03Z" level=info msg="invalidating lease" time="2025-09-15T15:42:03Z" level=info msg="getting lease" 2025-09-15T15:42:03.100523179Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:42:03Z" level=info msg="invalidating lease" time="2025-09-15T15:42:03Z" level=info msg="getting lease" 2025-09-15T15:42:03.102369796Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:42:03.104312134Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 355.895690386} 2025-09-15T15:42:03.104510765Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:42:14.104716391Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:42:14.113753416Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:42:14.113895539Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:42:14Z" level=info msg="invalidating lease" time="2025-09-15T15:42:14Z" level=info msg="getting lease" 2025-09-15T15:42:14.115988106Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:42:14Z" level=info msg="invalidating lease" time="2025-09-15T15:42:14Z" level=info msg="getting lease" 2025-09-15T15:42:14.118004352Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:42:14.11998772Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 344.880013877} 2025-09-15T15:42:14.120191163Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:42:25.120814523Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:42:25.129832402Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:42:25.129872376Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:42:25Z" level=info msg="invalidating lease" time="2025-09-15T15:42:25Z" level=info msg="getting lease" 2025-09-15T15:42:25.131585245Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:42:25Z" level=info msg="invalidating lease" time="2025-09-15T15:42:25Z" level=info msg="getting lease" 2025-09-15T15:42:25.133527032Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:42:25.135504153Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 333.864497347} 2025-09-15T15:42:25.135732383Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:42:36.136531544Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:42:36.147087915Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:42:36.147130426Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:42:36Z" level=info msg="invalidating lease" time="2025-09-15T15:42:36Z" level=info msg="getting lease" 2025-09-15T15:42:36.148923727Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:42:36Z" level=info msg="invalidating lease" time="2025-09-15T15:42:36Z" level=info msg="getting lease" 2025-09-15T15:42:36.151036822Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:42:36.153237997Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 322.846763425} 2025-09-15T15:42:36.153464247Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:42:47.153643392Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:42:47.163528685Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:42:47.163557019Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:42:47Z" level=info msg="invalidating lease" time="2025-09-15T15:42:47Z" level=info msg="getting lease" 2025-09-15T15:42:47.165496306Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:42:47Z" level=info msg="invalidating lease" time="2025-09-15T15:42:47Z" level=info msg="getting lease" 2025-09-15T15:42:47.167735401Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:42:47.16984428Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 311.830159965} 2025-09-15T15:42:47.170066111Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:42:58.171225003Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:42:58.180287517Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:42:58.1803208Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:42:58Z" level=info msg="invalidating lease" time="2025-09-15T15:42:58Z" level=info msg="getting lease" 2025-09-15T15:42:58.182213359Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:42:58Z" level=info msg="invalidating lease" time="2025-09-15T15:42:58Z" level=info msg="getting lease" 2025-09-15T15:42:58.184282682Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:42:58.186283211Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 300.813718455} 2025-09-15T15:42:58.186467069Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:43:09.186726394Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:43:09.195987674Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:43:09.196014537Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:43:09Z" level=info msg="invalidating lease" time="2025-09-15T15:43:09Z" level=info msg="getting lease" 2025-09-15T15:43:09.197748974Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:43:09Z" level=info msg="invalidating lease" time="2025-09-15T15:43:09Z" level=info msg="getting lease" 2025-09-15T15:43:09.199701212Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:43:09.201631665Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 289.798369872} 2025-09-15T15:43:09.201826261Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:43:20.20262594Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:43:20.210715237Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:43:20.210802362Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:43:20Z" level=info msg="invalidating lease" time="2025-09-15T15:43:20Z" level=info msg="getting lease" 2025-09-15T15:43:20.212713455Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:43:20.214477031Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 278.785524414} 2025-09-15T15:43:20.214498304Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:43:20Z" level=info msg="invalidating lease" time="2025-09-15T15:43:20Z" level=info msg="getting lease" 2025-09-15T15:43:20.216273102Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:43:31.216802607Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:43:31.225849274Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:43:31.225921416Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:43:31Z" level=info msg="invalidating lease" time="2025-09-15T15:43:31Z" level=info msg="getting lease" 2025-09-15T15:43:31.227962Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:43:31.229547606Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 267.770453858} 2025-09-15T15:43:31.229588359Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:43:31Z" level=info msg="invalidating lease" time="2025-09-15T15:43:31Z" level=info msg="getting lease" 2025-09-15T15:43:31.231376824Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:43:42.232404404Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:43:42.2423261Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:43:42.242359183Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:43:42Z" level=info msg="invalidating lease" time="2025-09-15T15:43:42Z" level=info msg="getting lease" 2025-09-15T15:43:42.244639965Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:43:42Z" level=info msg="invalidating lease" time="2025-09-15T15:43:42Z" level=info msg="getting lease" 2025-09-15T15:43:42.246741362Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:43:42.248501151Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 256.751500544} 2025-09-15T15:43:42.248696961Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:43:53.249765018Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:43:53.25829286Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:43:53.258355682Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:43:53Z" level=info msg="invalidating lease" time="2025-09-15T15:43:53Z" level=info msg="getting lease" 2025-09-15T15:43:53.26032702Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:43:53Z" level=info msg="invalidating lease" time="2025-09-15T15:43:53Z" level=info msg="getting lease" 2025-09-15T15:43:53.262073013Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:43:53.263624779Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 245.73637667} 2025-09-15T15:43:53.263810812Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:44:04.264183049Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:44:04.273174749Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:44:04.273211086Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:44:04Z" level=info msg="invalidating lease" time="2025-09-15T15:44:04Z" level=info msg="getting lease" 2025-09-15T15:44:04.275738723Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:44:04Z" level=info msg="invalidating lease" time="2025-09-15T15:44:04Z" level=info msg="getting lease" 2025-09-15T15:44:04.277934303Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:44:04.279809356Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 234.720192447} 2025-09-15T15:44:04.280016217Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:44:15.280265162Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:44:15.289384382Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:44:15.289414421Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:44:15Z" level=info msg="invalidating lease" time="2025-09-15T15:44:15Z" level=info msg="getting lease" 2025-09-15T15:44:15.291454946Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:44:15Z" level=info msg="invalidating lease" time="2025-09-15T15:44:15Z" level=info msg="getting lease" 2025-09-15T15:44:15.293275517Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:44:15.294912095Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 223.705089407} 2025-09-15T15:44:15.295112282Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:44:26.295758423Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:44:26.305158196Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:44:26.305189545Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:44:26Z" level=info msg="invalidating lease" time="2025-09-15T15:44:26Z" level=info msg="getting lease" 2025-09-15T15:44:26.307035475Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:44:26Z" level=info msg="invalidating lease" time="2025-09-15T15:44:26Z" level=info msg="getting lease" 2025-09-15T15:44:26.309036433Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:44:26.310698302Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 212.689303037} 2025-09-15T15:44:26.310903814Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:44:37.311749141Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:44:37.320704621Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:44:37.320728547Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:44:37Z" level=info msg="invalidating lease" time="2025-09-15T15:44:37Z" level=info msg="getting lease" 2025-09-15T15:44:37.322843159Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:44:37.324574378Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 201.675428074} 2025-09-15T15:44:37.324596409Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:44:37Z" level=info msg="invalidating lease" time="2025-09-15T15:44:37Z" level=info msg="getting lease" 2025-09-15T15:44:37.326919638Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:44:48.327750123Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:44:48.337374075Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:44:48.337457579Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:44:48Z" level=info msg="invalidating lease" time="2025-09-15T15:44:48Z" level=info msg="getting lease" 2025-09-15T15:44:48.33920145Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:44:48Z" level=info msg="invalidating lease" time="2025-09-15T15:44:48Z" level=info msg="getting lease" 2025-09-15T15:44:48.340851386Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:44:48.342801697Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 190.657199702} 2025-09-15T15:44:48.342986664Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:44:59.344038633Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:44:59.35385242Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:44:59.353882937Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:44:59Z" level=info msg="invalidating lease" time="2025-09-15T15:44:59Z" level=info msg="getting lease" 2025-09-15T15:44:59.355508837Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:44:59Z" level=info msg="invalidating lease" time="2025-09-15T15:44:59Z" level=info msg="getting lease" 2025-09-15T15:44:59.357221577Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:44:59.35909574Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 179.640906047} 2025-09-15T15:44:59.3592775Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:45:10.360043325Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:45:10.369528086Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:45:10.369559086Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:45:10.371155481Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 168.628846223} 2025-09-15T15:45:10.37117763Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:45:10Z" level=info msg="invalidating lease" time="2025-09-15T15:45:10Z" level=info msg="getting lease" 2025-09-15T15:45:10.372747302Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:45:10Z" level=info msg="invalidating lease" time="2025-09-15T15:45:10Z" level=info msg="getting lease" 2025-09-15T15:45:10.374713643Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:45:21.374892001Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:45:21.383556021Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:45:21.383583044Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:45:21Z" level=info msg="invalidating lease" time="2025-09-15T15:45:21Z" level=info msg="getting lease" 2025-09-15T15:45:21.385311598Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:45:21Z" level=info msg="invalidating lease" time="2025-09-15T15:45:21Z" level=info msg="getting lease" 2025-09-15T15:45:21.386912847Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:45:21.388605967Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 157.611395357} 2025-09-15T15:45:21.388804225Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:45:32.389188962Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:45:32.39769698Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:45:32.397731725Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:45:32Z" level=info msg="invalidating lease" time="2025-09-15T15:45:32Z" level=info msg="getting lease" 2025-09-15T15:45:32.39956424Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:45:32Z" level=info msg="invalidating lease" time="2025-09-15T15:45:32Z" level=info msg="getting lease" 2025-09-15T15:45:32.401113517Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:45:32.40296819Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 146.597033788} 2025-09-15T15:45:32.403235846Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:45:43.40374146Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:45:43.412662389Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:45:43.412690329Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:45:43Z" level=info msg="invalidating lease" time="2025-09-15T15:45:43Z" level=info msg="getting lease" 2025-09-15T15:45:43.414627788Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:45:43Z" level=info msg="invalidating lease" time="2025-09-15T15:45:43Z" level=info msg="getting lease" 2025-09-15T15:45:43.416414961Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:45:43.418542602Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 135.581458962} 2025-09-15T15:45:43.418739619Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:45:54.419884418Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:45:54.42889426Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:45:54.428921033Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:45:54.43056068Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 124.569441368} 2025-09-15T15:45:54.430583829Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:45:54Z" level=info msg="invalidating lease" time="2025-09-15T15:45:54Z" level=info msg="getting lease" 2025-09-15T15:45:54.432153736Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:45:54Z" level=info msg="invalidating lease" time="2025-09-15T15:45:54Z" level=info msg="getting lease" 2025-09-15T15:45:54.43417335Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:46:05.43492018Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:46:05.443862499Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:46:05.44388637Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:46:05.445554874Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 113.554446615} 2025-09-15T15:46:05.445575669Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:46:05Z" level=info msg="invalidating lease" time="2025-09-15T15:46:05Z" level=info msg="getting lease" 2025-09-15T15:46:05.447432163Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:46:05Z" level=info msg="invalidating lease" time="2025-09-15T15:46:05Z" level=info msg="getting lease" 2025-09-15T15:46:05.450207087Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:46:16.451093768Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:46:16.459144702Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:46:16.459173721Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:46:16Z" level=info msg="invalidating lease" time="2025-09-15T15:46:16Z" level=info msg="getting lease" 2025-09-15T15:46:16.460694463Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:46:16Z" level=info msg="invalidating lease" time="2025-09-15T15:46:16Z" level=info msg="getting lease" 2025-09-15T15:46:16.462514334Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:46:16.465240735Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 102.534761439} 2025-09-15T15:46:16.465429929Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:46:27.466223062Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:46:27.475251376Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:46:27.475279086Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:46:27Z" level=info msg="invalidating lease" time="2025-09-15T15:46:27Z" level=info msg="getting lease" 2025-09-15T15:46:27.476770447Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:46:27Z" level=info msg="invalidating lease" time="2025-09-15T15:46:27Z" level=info msg="getting lease" 2025-09-15T15:46:27.47861917Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:46:27.480438091Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 91.519563273} 2025-09-15T15:46:27.480616351Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:46:38.480805052Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:46:38.489247842Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:46:38.489276561Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:46:38Z" level=info msg="invalidating lease" time="2025-09-15T15:46:38Z" level=info msg="getting lease" 2025-09-15T15:46:38.490925107Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:46:38.492459004Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 80.507542367} 2025-09-15T15:46:38.492478766Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:46:38Z" level=info msg="invalidating lease" time="2025-09-15T15:46:38Z" level=info msg="getting lease" 2025-09-15T15:46:38.494476705Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:46:49.494704423Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:46:49.504419855Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:46:49.504450106Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:46:49Z" level=info msg="invalidating lease" time="2025-09-15T15:46:49Z" level=info msg="getting lease" 2025-09-15T15:46:49.506052405Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:46:49Z" level=info msg="invalidating lease" time="2025-09-15T15:46:49Z" level=info msg="getting lease" 2025-09-15T15:46:49.507974618Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:46:49.509930656Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 69.490073831} 2025-09-15T15:46:49.510133546Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:47:00.511278908Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:47:00.520118913Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:47:00.520152287Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:47:00Z" level=info msg="invalidating lease" time="2025-09-15T15:47:00Z" level=info msg="getting lease" 2025-09-15T15:47:00.521788416Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:47:00Z" level=info msg="invalidating lease" time="2025-09-15T15:47:00Z" level=info msg="getting lease" 2025-09-15T15:47:00.52351537Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:47:00.525322497Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 58.474679042} 2025-09-15T15:47:00.525542295Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:47:11.525804952Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:47:11.534339483Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:47:11.534365736Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:47:11Z" level=info msg="invalidating lease" time="2025-09-15T15:47:11Z" level=info msg="getting lease" 2025-09-15T15:47:11.536336406Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:47:11Z" level=info msg="invalidating lease" time="2025-09-15T15:47:11Z" level=info msg="getting lease" 2025-09-15T15:47:11.538511Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:47:11.540209043Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 47.459792441} 2025-09-15T15:47:11.540402201Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:47:22.541230941Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:47:22.550374255Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:47:22.550413527Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:47:22Z" level=info msg="invalidating lease" time="2025-09-15T15:47:22Z" level=info msg="getting lease" 2025-09-15T15:47:22.552461215Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:47:22.554689273Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 36.445312173} 2025-09-15T15:47:22.554711683Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:47:22Z" level=info msg="invalidating lease" time="2025-09-15T15:47:22Z" level=info msg="getting lease" 2025-09-15T15:47:22.556599726Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:47:33.557274297Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:47:33.565380639Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:47:33.56546108Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:47:33Z" level=info msg="invalidating lease" time="2025-09-15T15:47:33Z" level=info msg="getting lease" 2025-09-15T15:47:33.567250833Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:47:33Z" level=info msg="invalidating lease" time="2025-09-15T15:47:33Z" level=info msg="getting lease" 2025-09-15T15:47:33.569088652Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:47:33.570583702Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 25.429417812} 2025-09-15T15:47:33.570786971Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:47:44.571780734Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:47:44.580038748Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:47:44.580079267Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:47:44.582088441Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 14.417913474} 2025-09-15T15:47:44.582112052Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:47:44Z" level=info msg="invalidating lease" time="2025-09-15T15:47:44Z" level=info msg="getting lease" 2025-09-15T15:47:44.58517093Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:47:44Z" level=info msg="invalidating lease" time="2025-09-15T15:47:44Z" level=info msg="getting lease" 2025-09-15T15:47:44.587215196Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:47:55.587828465Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:47:55.597328071Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:47:55.59735667Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:47:55Z" level=info msg="invalidating lease" time="2025-09-15T15:47:55Z" level=info msg="getting lease" 2025-09-15T15:47:55.599423993Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:47:55Z" level=info msg="invalidating lease" time="2025-09-15T15:47:55Z" level=info msg="getting lease" 2025-09-15T15:47:55.601514003Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:47:55.603295138Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 3.396706349} 2025-09-15T15:47:55.603828561Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "4.396706349s"} 2025-09-15T15:48:00.001232741Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:48:00.009724965Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-15T15:38:26Z"} 2025-09-15T15:48:00.009768475Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:48:00Z" level=info msg="invalidating lease" time="2025-09-15T15:48:00Z" level=info msg="getting lease" 2025-09-15T15:48:00.011682708Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} 2025-09-15T15:48:00.013474755Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-2"} 2025-09-15T15:48:00.020234676Z INFO controllers.NodeHealthCheck.resource manager deleted remediation CR {"name": "worker-0-2-lbzgm"} 2025-09-15T15:48:00.020260501Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-15T15:48:00.020216331Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:48:00.020337925Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-15T15:48:00.020350467Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:48:00.020355108Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-15T15:48:00.020276548Z DEBUG events [remediation] Deleted remediation CR of kind FenceAgentsRemediation with name worker-0-2-lbzgm {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"112079"}, "reason": "RemediationRemoved"} 2025-09-15T15:48:00.043631074Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"c83c3e72-a0c2-46e9-8267-11ccedab5357","controller":false}} 2025-09-15T15:48:00.043667122Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-2-lbzgm", "Remediation CR Kind": "FenceAgentsRemediation"} time="2025-09-15T15:48:00Z" level=info msg="invalidating lease" time="2025-09-15T15:48:00Z" level=info msg="getting lease" 2025-09-15T15:48:00.044262685Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-15T15:48:00.044348999Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:48:00.114232245Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:48:00Z" level=info msg="invalidating lease" time="2025-09-15T15:48:00Z" level=info msg="getting lease" 2025-09-15T15:48:00.115664244Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:48:00Z" level=info msg="invalidating lease" time="2025-09-15T15:48:00Z" level=info msg="getting lease" 2025-09-15T15:48:00.117266435Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-15T15:48:00Z" level=info msg="invalidating lease" time="2025-09-15T15:48:00Z" level=info msg="getting lease" 2025-09-15T15:48:00.122094328Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-far-worker", "new status": {"observedNodes":3,"healthyNodes":3,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-09-15T15:35:14Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-09-15T15:39:39Z"}, "patch": "{\"status\":{\"healthyNodes\":3,\"phase\":\"Enabled\",\"reason\":\"NHC is enabled, no ongoing remediation\",\"unhealthyNodes\":null}}"} 2025-09-15T15:48:00.327749256Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-09-15T15:48:11.044413478Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-15T15:48:11.052324908Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-15T15:48:11Z" level=info msg="invalidating lease" time="2025-09-15T15:48:11Z" level=info msg="getting lease" 2025-09-15T15:48:11.053746118Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-15T15:48:11Z" level=info msg="invalidating lease" time="2025-09-15T15:48:11Z" level=info msg="getting lease" 2025-09-15T15:48:11.055261531Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-15T15:48:11Z" level=info msg="invalidating lease" time="2025-09-15T15:48:11Z" level=info msg="getting lease" 2025-09-15T15:48:11.057115295Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "0s"} FAR Logs: 2025-09-15T15:30:40.84807986Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-09-15T15:30:40.848561732Z INFO setup Go OS/Arch: linux/amd64 2025-09-15T15:30:40.848566835Z INFO setup Operator Version: bd73055e 2025-09-15T15:30:40.848570443Z INFO setup Git Commit: bd73055ef2c68bfdc865d2c54179f4448bd454da 2025-09-15T15:30:40.848573026Z INFO setup Build Date: 2025-09-08T09:09:10+00:00 2025-09-15T15:30:40.84859781Z INFO setup HTTP/2 for webhooks disabled 2025-09-15T15:30:40.855456391Z INFO validation out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 33} 2025-09-15T15:30:40.855495245Z INFO setup out-of-service taint is supported on this cluster 2025-09-15T15:30:40.855537756Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation"} 2025-09-15T15:30:40.855581512Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2025-09-15T15:30:40.855663053Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2025-09-15T15:30:40.85573322Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-15T15:30:40.855776984Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-15T15:30:40.855809733Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-15T15:30:40.855851849Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-09-15T15:30:40.855886957Z INFO setup starting manager 2025-09-15T15:30:40.856071964Z INFO controller-runtime.metrics Starting metrics server 2025-09-15T15:30:40.856201173Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-09-15T15:30:40.856311014Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2025-09-15T15:30:40.856306386Z INFO controller-runtime.webhook Starting webhook server I0915 15:30:40.856421 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/cb305759.medik8s.io... 2025-09-15T15:30:40.856660747Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-09-15T15:30:40.856802659Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-09-15T15:30:40.856818116Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} 2025-09-15T15:35:14.547811427Z INFO fenceagentsremediationtemplate-resource default {"name": "fenceagentsremediationtemplate-test"} 2025-09-15T15:35:14.552156753Z INFO fenceagentsremediationtemplate-resource validate create {"name": "fenceagentsremediationtemplate-test"} 2025-09-15T15:38:02.453385489Z INFO fenceagentsremediation-resource validate create {"name": "worker-0-2-lbzgm"} 2025-09-15T15:38:02.566348718Z INFO fenceagentsremediation-resource validate update {"name": "worker-0-2-lbzgm"} I0915 15:38:21.255267 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/cb305759.medik8s.io 2025-09-15T15:38:21.255375237Z DEBUG events fence-agents-remediation-controller-manager-569dff989d-xg92x_e9a93dc6-e1e6-4927-9493-c6fbaa0fe1ed became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"cb305759.medik8s.io","uid":"0c18e7f2-24ca-41f0-b21e-0b28bf11657a","apiVersion":"coordination.k8s.io/v1","resourceVersion":"111436"}, "reason": "LeaderElection"} 2025-09-15T15:38:21.25590744Z INFO Starting EventSource {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation", "source": "kind source: *v1alpha1.FenceAgentsRemediation"} 2025-09-15T15:38:21.256174989Z INFO Starting Controller {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation"} 2025-09-15T15:38:21.359548214Z INFO Starting workers {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation", "worker count": 1} 2025-09-15T15:38:21.359767139Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-15T15:38:21.359848699Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-15T15:38:21.462738053Z INFO controllers.FenceAgentsRemediation Build fence agent command line {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2"} 2025-09-15T15:38:21.663020971Z INFO controllers.FenceAgentsRemediation Execute the fence agent {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-2", "FAR uid": "ae5c04e0-f5eb-4ec1-888f-efda126568ee", "Parameters": ["--action","--ip","--lanplus","--password","--ipport","--username"]} 2025-09-15T15:38:21.663288323Z INFO executer fence agent start {"uid": "ae5c04e0-f5eb-4ec1-888f-efda126568ee", "fence_agent": "fence_ipmilan", "retryCount": 5, "retryInterval": "10s", "timeout": "5m0s"} 2025-09-15T15:38:21.663343737Z DEBUG events [remediation] Fence agent was executed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-lbzgm","uid":"ae5c04e0-f5eb-4ec1-888f-efda126568ee","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"110888"}, "reason": "FenceAgentExecuted"} 2025-09-15T15:38:21.668489002Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-15T15:38:24.162984255Z INFO executer command completed {"uid": "ae5c04e0-f5eb-4ec1-888f-efda126568ee", "response": "Success: Rebooted\n", "errMessage": "", "err": null} 2025-09-15T15:38:24.16312255Z INFO executer fence agent done {"uid": "ae5c04e0-f5eb-4ec1-888f-efda126568ee", "fence_agent": "fence_ipmilan", "stdout": "Success: Rebooted\n", "stderr": "", "err": null} 2025-09-15T15:38:24.163142394Z INFO executer updating status {"FAR uid": "ae5c04e0-f5eb-4ec1-888f-efda126568ee"} 2025-09-15T15:38:24.163241887Z INFO executer Updating Status Condition {"processingConditionStatus": "", "fenceAgentActionSucceededConditionStatus": "True", "succeededConditionStatus": "", "reason": "FenceAgentSucceeded", "LastUpdateTime": "2025-09-15T15:38:24.163240864Z"} 2025-09-15T15:38:24.163303361Z DEBUG events [remediation] Fence agent was succeeded {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-lbzgm","uid":"ae5c04e0-f5eb-4ec1-888f-efda126568ee","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"110888"}, "reason": "FenceAgentSucceeded"} 2025-09-15T15:38:24.168267914Z INFO executer status updated {"FAR uid": "ae5c04e0-f5eb-4ec1-888f-efda126568ee"} 2025-09-15T15:38:24.168637861Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-15T15:38:24.168659995Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-15T15:38:24.168735148Z INFO controllers.FenceAgentsRemediation Remediation strategy is ResourceDeletion which explicitly deletes resources - manually deleting workload {"Node Name": "worker-0-2-lbzgm"} 2025-09-15T15:38:24.168816813Z DEBUG events [remediation] Manually delete pods from the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"f46f7f46-454f-4bd8-b813-1f15642c2857","apiVersion":"v1","resourceVersion":"110891"}, "reason": "DeleteResources"} 2025-09-15T15:38:24.269321063Z INFO commons-resource starting to delete pods {"node name": "worker-0-2"} 2025-09-15T15:38:26.623622954Z INFO commons-resource done deleting pods {"node name": "worker-0-2"} 2025-09-15T15:38:26.623652193Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "False", "fenceAgentActionSucceededConditionStatus": "", "succeededConditionStatus": "True", "reason": "RemediationFinishedSuccessfully", "LastUpdateTime": "2025-09-15T15:38:26.623651267Z"} 2025-09-15T15:38:26.623667865Z INFO executer cancelling fence agent routine {"uid": "ae5c04e0-f5eb-4ec1-888f-efda126568ee"} 2025-09-15T15:38:26.623675386Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR has completed to remediate the node {"Node Name": "worker-0-2"} 2025-09-15T15:38:26.623881856Z DEBUG events [remediation] Unhealthy node remediation was completed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"f46f7f46-454f-4bd8-b813-1f15642c2857","apiVersion":"v1","resourceVersion":"110891"}, "reason": "NodeRemediationCompleted"} 2025-09-15T15:38:26.623920538Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-lbzgm","uid":"ae5c04e0-f5eb-4ec1-888f-efda126568ee","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"111479"}, "reason": "RemediationFinished"} 2025-09-15T15:38:26.829881813Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-15T15:38:26.830040094Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-15T15:38:26.830072175Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-15T15:38:26.835400396Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-15T15:39:39.692410806Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-15T15:39:39.692436418Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-15T15:39:39.698352886Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-15T15:48:00.020503395Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-15T15:48:00.020562062Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-09-15T15:48:00.020604016Z INFO controllers.FenceAgentsRemediation CR's deletion timestamp is not zero, and FAR finalizer exists {"CR Name": "worker-0-2-lbzgm"} 2025-09-15T15:48:00.027366421Z INFO taints Taint was removed {"taint effect": "NoExecute", "taint list": null} 2025-09-15T15:48:00.027506303Z INFO controllers.FenceAgentsRemediation FAR remediation taint was removed {"Node Name": "worker-0-2"} 2025-09-15T15:48:00.027883537Z DEBUG events [remediation] Remediation taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-2","uid":"f46f7f46-454f-4bd8-b813-1f15642c2857","apiVersion":"v1","resourceVersion":"114506"}, "reason": "RemoveRemediationTaint"} 2025-09-15T15:48:00.035870282Z INFO fenceagentsremediation-resource validate update {"name": "worker-0-2-lbzgm"} 2025-09-15T15:48:00.042411417Z INFO controllers.FenceAgentsRemediation Finalizer was removed {"CR Name": "worker-0-2-lbzgm"} 2025-09-15T15:48:00.042475211Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-09-15T15:48:00.042570703Z DEBUG events [remediation] Finalizer was removed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-lbzgm","uid":"ae5c04e0-f5eb-4ec1-888f-efda126568ee","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"114810"}, "reason": "RemoveFinalizer"} 2025-09-15T15:48:00.043245928Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-09-15T15:48:00.043274439Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR was not found {"CR Name": "worker-0-2-lbzgm", "CR Namespace": "openshift-workload-availability"} 2025-09-15T15:48:00.043278501Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile