[kni@titan45 ~]$ oc project openshift-workload-availability Already on project "openshift-workload-availability" on server "https://api.ocp-edge-cluster-0.qe.lab.redhat.com:6443". [kni@titan45 ~]$ oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.19.0-0.nightly-2025-09-11-184658 True False 12h Cluster version is 4.19.0-0.nightly-2025-09-11-184658 [kni@titan45 ~]$ oc get csv NAME DISPLAY VERSION REPLACES PHASE fence-agents-remediation.v0.6.0 Fence Agents Remediation Operator 0.6.0 fence-agents-remediation.v0.5.1 Succeeded node-healthcheck-operator.v0.10.0 Node Health Check Operator 0.10.0 node-healthcheck-operator.v0.9.1 Succeeded self-node-remediation.v0.10.2 Self Node Remediation Operator 0.10.2 self-node-remediation.v0.10.1 Succeeded [kni@titan45 ~]$ PODS=$(oc get pods -o name -n openshift-workload-availability | grep fence-agents-remediation-controller-manager) [kni@titan45 ~]$ echo $PODS pod/fence-agents-remediation-controller-manager-7bfcd47b77-89hsp pod/fence-agents-remediation-controller-manager-7bfcd47b77-t9nw9 [kni@titan45 ~]$ for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/fence-agents-remediation-controller-manager-7bfcd47b77-89hsp "worker-0-1" == pod/fence-agents-remediation-controller-manager-7bfcd47b77-t9nw9 "worker-0-2" [kni@titan45 ~]$ oc get PODS=$(oc get pods -o name -n openshift-workload-availability | grep node-healthcheck-controller-manager) error: the server doesn't have a resource type "PODS=pod" [kni@titan45 ~]$ echo $PODS pod/fence-agents-remediation-controller-manager-7bfcd47b77-89hsp pod/fence-agents-remediation-controller-manager-7bfcd47b77-t9nw9 [kni@titan45 ~]$ for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/fence-agents-remediation-controller-manager-7bfcd47b77-89hsp "worker-0-1" == pod/fence-agents-remediation-controller-manager-7bfcd47b77-t9nw9 "worker-0-2" [kni@titan45 ~]$ vi test.yaml [kni@titan45 ~]$ cat test.yaml apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate metadata: name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability spec: template: spec: agent: fence_ipmilan retrycount: 5 retryinterval: 10s timeout: 300s nodeparameters: '--ipport': master-0-0: '6230' master-0-1: '6231' master-0-2: '6232' worker-0-0: '6233' worker-0-1: '6234' worker-0-2: '6235' sharedparameters: '--action': reboot '--lanplus': '' '--ip': 192.168.123.1 '--password': password '--username': admin --- apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: name: nhc-far-worker spec: maxUnhealthy: 1 healthyDelay: 300s remediationTemplate: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/control-plane operator: DoesNotExist values: [] - key: node-role.kubernetes.io/master operator: DoesNotExist values: [] unhealthyConditions: - duration: 30s status: 'False' type: Ready - duration: 30s status: Unknown type: Ready [kni@titan45 ~]$ oc apply -f test.yaml fenceagentsremediationtemplate.fence-agents-remediation.medik8s.io/fenceagentsremediationtemplate-test created nodehealthcheck.remediation.medik8s.io/nhc-far-worker created [kni@titan45 ~]$ oc debug node/worker-0-0 -- chroot /host bash -c "date & uptime -s" Temporary namespace openshift-debug-966ts is created for debugging node... Starting pod/worker-0-0-debug-xdvsw ... To use host binaries, run `chroot /host` Fri Sep 12 22:20:16 UTC 2025 2025-09-12 09:04:56 Removing debug pod ... Temporary namespace openshift-debug-966ts was removed. [kni@titan45 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 13h v1.32.8 worker-0-1 Ready worker 13h v1.32.8 worker-0-2 Ready worker 13h v1.32.8 [kni@titan45 ~]$ oc debug node/worker-0-0 -- chroot /host bash -c "systemctl stop kubelet" Temporary namespace openshift-debug-nnhkh is created for debugging node... Starting pod/worker-0-0-debug-fqlkq ... To use host binaries, run `chroot /host` [kni@titan45 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 NotReady worker 13h v1.32.8 worker-0-1 Ready worker 13h v1.32.8 worker-0-2 Ready worker 13h v1.32.8 [kni@titan45 ~]$ oc get far NAME AGE worker-0-0-wrm5v 4s [kni@titan45 ~]$ oc get far -o yaml apiVersion: v1 items: - apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediation metadata: annotations: remediation.medik8s.io/node-name: worker-0-0 remediation.medik8s.io/template-name: fenceagentsremediationtemplate-test creationTimestamp: "2025-09-12T22:24:32Z" finalizers: - fence-agents-remediation.medik8s.io/far-finalizer generateName: worker-0-0- generation: 1 labels: app.kubernetes.io/part-of: node-healthcheck-controller name: worker-0-0-wrm5v namespace: openshift-workload-availability ownerReferences: - apiVersion: remediation.medik8s.io/v1alpha1 controller: false kind: NodeHealthCheck name: nhc-far-worker uid: d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4 resourceVersion: "301020" uid: 9f455ee6-8d40-42de-8a37-f8abf2f55efd spec: agent: fence_ipmilan nodeparameters: --ipport: master-0-0: "6230" master-0-1: "6231" master-0-2: "6232" worker-0-0: "6233" worker-0-1: "6234" worker-0-2: "6235" remediationStrategy: ResourceDeletion retrycount: 5 retryinterval: 10s sharedSecretName: fence-agents-credentials-shared sharedparameters: --action: reboot --ip: 192.168.123.1 --lanplus: "" --password: password --username: admin timeout: 5m0s status: conditions: - lastTransitionTime: "2025-09-12T22:24:32Z" message: FAR CR was found, its name matches one of the cluster nodes, and a finalizer was set to the CR reason: RemediationStarted status: "True" type: Processing - lastTransitionTime: "2025-09-12T22:24:39Z" message: FAR taint was added and the fence agent command has been created and executed successfully reason: FenceAgentSucceeded status: "True" type: FenceAgentActionSucceeded - lastTransitionTime: "2025-09-12T22:24:32Z" message: FAR CR was found, its name matches one of the cluster nodes, and a finalizer was set to the CR reason: RemediationStarted status: Unknown type: Succeeded lastUpdateTime: "2025-09-12T22:24:39Z" kind: List metadata: resourceVersion: "" [kni@titan45 ~]$ oc get far NAME AGE worker-0-0-wrm5v 2m22s [kni@titan45 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 13h v1.32.8 worker-0-1 Ready worker 13h v1.32.8 worker-0-2 Ready worker 13h v1.32.8 [kni@titan45 ~]$ oc get far No resources found in openshift-workload-availability namespace. [kni@titan45 ~]$ oc debug node/worker-0-0 -- chroot /host bash -c "date & uptime -s" Temporary namespace openshift-debug-d8x96 is created for debugging node... Starting pod/worker-0-0-debug-xkkzv ... To use host binaries, run `chroot /host` Fri Sep 12 22:38:49 UTC 2025 2025-09-12 22:24:49 Removing debug pod ... Temporary namespace openshift-debug-d8x96 was removed. [kni@titan45 ~]$ oc get nhc NAME AGE nhc-far-worker 20m [kni@titan45 ~]$ oc get nhc -o yaml apiVersion: v1 items: - apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","metadata":{"annotations":{},"name":"nhc-far-worker"},"spec":{"healthyDelay":"300s","maxUnhealthy":1,"remediationTemplate":{"apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","kind":"FenceAgentsRemediationTemplate","name":"fenceagentsremediationtemplate-test","namespace":"openshift-workload-availability"},"selector":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"DoesNotExist","values":[]},{"key":"node-role.kubernetes.io/master","operator":"DoesNotExist","values":[]}]},"unhealthyConditions":[{"duration":"30s","status":"False","type":"Ready"},{"duration":"30s","status":"Unknown","type":"Ready"}]}} creationTimestamp: "2025-09-12T22:19:21Z" generation: 2 name: nhc-far-worker resourceVersion: "307153" uid: d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4 spec: healthyDelay: -1s maxUnhealthy: 1 remediationTemplate: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/control-plane operator: DoesNotExist values: [] - key: node-role.kubernetes.io/master operator: DoesNotExist values: [] unhealthyConditions: - duration: 30s status: "False" type: Ready - duration: 30s status: Unknown type: Ready status: conditions: - lastTransitionTime: "2025-09-12T22:24:31Z" message: No issues found, NodeHealthCheck is enabled. reason: NodeHealthCheckEnabled status: "False" type: Disabled healthyNodes: 3 lastUpdateTime: "2025-09-12T22:31:26Z" observedNodes: 3 phase: Enabled reason: NHC is enabled, no ongoing remediation kind: List metadata: resourceVersion: "" [kni@titan45 ~]$ oc debug node/worker-0-0 -- chroot /host bash -c "systemctl stop kubelet" Temporary namespace openshift-debug-65rcg is created for debugging node... Starting pod/worker-0-0-debug-2b4kq ... To use host binaries, run `chroot /host` [kni@titan45 ~]$ oc get far No resources found in openshift-workload-availability namespace. [kni@titan45 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 NotReady worker 13h v1.32.8 worker-0-1 Ready worker 13h v1.32.8 worker-0-2 Ready worker 13h v1.32.8 [kni@titan45 ~]$ oc get far -o yaml apiVersion: v1 items: - apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediation metadata: annotations: remediation.medik8s.io/node-name: worker-0-0 remediation.medik8s.io/template-name: fenceagentsremediationtemplate-test creationTimestamp: "2025-09-12T22:41:57Z" finalizers: - fence-agents-remediation.medik8s.io/far-finalizer generateName: worker-0-0- generation: 1 labels: app.kubernetes.io/part-of: node-healthcheck-controller name: worker-0-0-nsmhb namespace: openshift-workload-availability ownerReferences: - apiVersion: remediation.medik8s.io/v1alpha1 controller: false kind: NodeHealthCheck name: nhc-far-worker uid: d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4 resourceVersion: "308128" uid: 627403c2-464d-49c7-ac5c-59ccfc267bf9 spec: agent: fence_ipmilan nodeparameters: --ipport: master-0-0: "6230" master-0-1: "6231" master-0-2: "6232" worker-0-0: "6233" worker-0-1: "6234" worker-0-2: "6235" remediationStrategy: ResourceDeletion retrycount: 5 retryinterval: 10s sharedSecretName: fence-agents-credentials-shared sharedparameters: --action: reboot --ip: 192.168.123.1 --lanplus: "" --password: password --username: admin timeout: 5m0s status: conditions: - lastTransitionTime: "2025-09-12T22:42:07Z" message: The unhealthy node was fully remediated (it was tainted, fenced using the fence agent and all the node resources have been deleted) reason: RemediationFinishedSuccessfully status: "False" type: Processing - lastTransitionTime: "2025-09-12T22:42:04Z" message: FAR taint was added and the fence agent command has been created and executed successfully reason: FenceAgentSucceeded status: "True" type: FenceAgentActionSucceeded - lastTransitionTime: "2025-09-12T22:42:07Z" message: The unhealthy node was fully remediated (it was tainted, fenced using the fence agent and all the node resources have been deleted) reason: RemediationFinishedSuccessfully status: "True" type: Succeeded lastUpdateTime: "2025-09-12T22:42:07Z" kind: List metadata: resourceVersion: "" [kni@titan45 ~]$ oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 13h v1.32.8 worker-0-1 Ready worker 13h v1.32.8 worker-0-2 Ready worker 13h v1.32.8 [kni@titan45 ~]$ oc get far NAME AGE worker-0-0-nsmhb 112s [kni@titan45 ~]$ oc get nhc -o yaml apiVersion: v1 items: - apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","metadata":{"annotations":{},"name":"nhc-far-worker"},"spec":{"healthyDelay":"300s","maxUnhealthy":1,"remediationTemplate":{"apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","kind":"FenceAgentsRemediationTemplate","name":"fenceagentsremediationtemplate-test","namespace":"openshift-workload-availability"},"selector":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"DoesNotExist","values":[]},{"key":"node-role.kubernetes.io/master","operator":"DoesNotExist","values":[]}]},"unhealthyConditions":[{"duration":"30s","status":"False","type":"Ready"},{"duration":"30s","status":"Unknown","type":"Ready"}]}} creationTimestamp: "2025-09-12T22:19:21Z" generation: 2 name: nhc-far-worker resourceVersion: "308662" uid: d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4 spec: healthyDelay: -1s maxUnhealthy: 1 remediationTemplate: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate name: fenceagentsremediationtemplate-test namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/control-plane operator: DoesNotExist values: [] - key: node-role.kubernetes.io/master operator: DoesNotExist values: [] unhealthyConditions: - duration: 30s status: "False" type: Ready - duration: 30s status: Unknown type: Ready status: conditions: - lastTransitionTime: "2025-09-12T22:24:31Z" message: No issues found, NodeHealthCheck is enabled. reason: NodeHealthCheckEnabled status: "False" type: Disabled healthyNodes: 2 lastUpdateTime: "2025-09-12T22:43:44Z" observedNodes: 3 phase: Remediating reason: NHC is remediating 1 nodes unhealthyNodes: - conditionsHealthyTimestamp: "2025-09-12T22:43:44Z" name: worker-0-0 remediations: - resource: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediation name: worker-0-0-nsmhb namespace: openshift-workload-availability uid: 627403c2-464d-49c7-ac5c-59ccfc267bf9 started: "2025-09-12T22:41:57Z" templateName: fenceagentsremediationtemplate-test kind: List metadata: resourceVersion: "" [kni@titan45 ~]$ oc get far NAME AGE worker-0-0-nsmhb 10m NHC Logs: 2025-09-12T22:10:43.831645974Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-09-12T22:10:43.831821548Z INFO setup Go OS/Arch: linux/amd64 2025-09-12T22:10:43.831827874Z INFO setup Operator Version: 2ef588d 2025-09-12T22:10:43.831831456Z INFO setup Git Commit: 2ef588d65de4087c46447401cd6e757459d79210 2025-09-12T22:10:43.831834743Z INFO setup Build Date: 2025-09-01T17:55:11+00:00 2025-09-12T22:10:43.831838028Z INFO setup HTTP/2 for metrics and webhook server disabled 2025-09-12T22:10:43.882184266Z INFO setup supported control plane topology {"topology": "HighlyAvailable"} 2025-09-12T22:10:43.882251406Z INFO setup Cluster capabilities {"IsOnOpenshift": true, "HasMachineAPI": true} 2025-09-12T22:10:43.882699282Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "remediation.medik8s.io/v1alpha1, Kind=NodeHealthCheck", "path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2025-09-12T22:10:43.882823633Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2025-09-12T22:10:43.882893549Z INFO setup starting manager 2025-09-12T22:10:43.88301123Z INFO controller-runtime.metrics Starting metrics server 2025-09-12T22:10:43.883065217Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-09-12T22:10:43.883121307Z INFO controller-runtime.webhook Starting webhook server 2025-09-12T22:10:43.88316044Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": "127.0.0.1:8080", "secure": false} 2025-09-12T22:10:43.883504465Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-09-12T22:10:43.883614052Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-09-12T22:10:43.883719148Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} I0912 22:10:43.984161 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/e1f13584.medik8s.io... I0912 22:11:10.937612 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/e1f13584.medik8s.io 2025-09-12T22:11:10.937684525Z DEBUG events node-healthcheck-controller-manager-7d56844489-cl2bj_f3fcaa9c-3daf-4ae7-ac4e-bc8837ee4d16 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"e1f13584.medik8s.io","uid":"f1abf5da-d2e6-4954-a649-cb707fce355d","apiVersion":"coordination.k8s.io/v1","resourceVersion":"295786"}, "reason": "LeaderElection"} 2025-09-12T22:11:10.939816193Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1alpha1.NodeHealthCheck"} 2025-09-12T22:11:10.939831607Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.MachineHealthCheck"} 2025-09-12T22:11:10.939911653Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1.Node"} 2025-09-12T22:11:10.939877827Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1.Node"} 2025-09-12T22:11:10.939834077Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "channel source: 0xc0002aa070"} 2025-09-12T22:11:10.939966263Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.Machine"} I0912 22:11:10.941918 1 shared_informer.go:313] Waiting for caches to sync for feature gate accessor 2025-09-12T22:11:10.944236433Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "channel source: 0xc0002aa2a0"} 2025-09-12T22:11:11.038921814Z INFO MHCChecker found custom MHC, will disable NHC 2025-09-12T22:11:11.039010021Z INFO MHCChecker MHC Checker status changed, notifying NHC controller 2025-09-12T22:11:11.039182762Z INFO adding all NHCs to reconcile queue for handling MHC event I0912 22:11:11.042056 1 shared_informer.go:320] Caches are synced for feature gate accessor I0912 22:11:11.042170 1 simple_featuregate_reader.go:171] Starting feature-gate-detector 2025-09-12T22:11:11.042530823Z INFO FeatureGateAccessor FeatureGates initialized I0912 22:11:11.043060 1 recorder_logging.go:49] &Event{ObjectMeta:{dummy.1864a88b59d7e26b.2f847e17 dummy 0 0001-01-01 00:00:00 +0000 UTC map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Pod,Namespace:dummy,Name:dummy,UID:,APIVersion:v1,ResourceVersion:,FieldPath:,},Reason:FeatureGatesInitialized,Message:FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AWSEFSDriverVolumeMetrics", "AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BareMetalLoadBalancer", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ChunkSizeMiB", "CloudDualStackNodeIPs", "ConsolePluginContentSecurityPolicy", "DisableKubeletCloudCredentialProviders", "GCPLabelsTags", "GatewayAPI", "GatewayAPIController", "HardwareSpeed", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "MultiArchInstallAWS", "MultiArchInstallGCP", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "NewOLM", "NodeDisruptionPolicy", "OnClusterBuild", "PersistentIPsForVirtualization", "PinnedImages", "PrivateHostedZoneAWS", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "VSphereDriverConfiguration", "VSphereMultiVCenters", "ValidatingAdmissionPolicy"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AutomatedEtcdBackup", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "HighlyAvailableArbiter", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "InsightsRuntimeExtractor", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "MachineAPIProviderOpenStack", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PlatformOperators", "ProcMountType", "SELinuxChangePolicy", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass", "VolumeGroupSnapshot"}},Source:EventSource{Component:,Host:,},FirstTimestamp:2025-09-12 22:11:11.042577003 +0000 UTC m=+27.265702588,LastTimestamp:2025-09-12 22:11:11.042577003 +0000 UTC m=+27.265702588,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:,ReportingInstance:,} 2025-09-12T22:11:11.142683552Z INFO Starting Controller {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck"} 2025-09-12T22:11:11.142807352Z INFO Starting workers {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "worker count": 1} 2025-09-12T22:11:11.142660448Z INFO Starting Controller {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck"} 2025-09-12T22:11:11.143025335Z INFO Starting workers {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "worker count": 1} 2025-09-12T22:11:11.392422583Z INFO console-plugin successfully created / updated console plugin resources 2025-09-12T22:19:21.924339711Z INFO nodehealthcheck-resource validate create {"name": "nhc-far-worker"} 2025-09-12T22:19:21.936283417Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:19:21.943258678Z INFO controllers.NodeHealthCheck disabling NHC in order to avoid conflict with custom MHCs configured in the cluster {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:19:21.943658729Z DEBUG events [remediation] Custom MachineHealthCheck(s) detected, disabling NodeHealthCheck to avoid conflicts {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"298629"}, "reason": "Disabled"} 2025-09-12T22:19:21.944745532Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-far-worker", "new status": {"observedNodes":0,"healthyNodes":0,"conditions":[{"type":"Disabled","status":"True","lastTransitionTime":"2025-09-12T22:19:21Z","reason":"ConflictingMachineHealthCheckDetected","message":"Custom MachineHealthCheck(s) detected, disabling NodeHealthCheck to avoid conflicts"}],"phase":"Disabled","reason":"NHC is disabled: ConflictingMachineHealthCheckDetected: Custom MachineHealthCheck(s) detected, disabling NodeHealthCheck to avoid conflicts"}, "patch": "{\"status\":{\"conditions\":[{\"lastTransitionTime\":\"2025-09-12T22:19:21Z\",\"message\":\"Custom MachineHealthCheck(s) detected, disabling NodeHealthCheck to avoid conflicts\",\"reason\":\"ConflictingMachineHealthCheckDetected\",\"status\":\"True\",\"type\":\"Disabled\"}],\"healthyNodes\":0,\"observedNodes\":0,\"phase\":\"Disabled\",\"reason\":\"NHC is disabled: ConflictingMachineHealthCheckDetected: Custom MachineHealthCheck(s) detected, disabling NodeHealthCheck to avoid conflicts\"}}"} 2025-09-12T22:19:22.163366266Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-09-12T22:22:09.881205459Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-far-worker"} 2025-09-12T22:22:09.881476017Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-far-worker"} 2025-09-12T22:22:09.88160782Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:22:09.897749026Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-09-12T22:24:31.825337583Z INFO MHCChecker found termination handler MHC, will ignore Nodes with Terminating condition 2025-09-12T22:24:31.825422533Z INFO MHCChecker MHC Checker status changed, notifying NHC controller 2025-09-12T22:24:31.825650834Z INFO adding all NHCs to reconcile queue for handling MHC event 2025-09-12T22:24:31.825787187Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:24:31.847552855Z INFO controllers.NodeHealthCheck enabling NHC, valid config, no conflicting MHC configured in the cluster {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:24:31.847706284Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-09-12T22:24:31.847767024Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation template CRs {"kind": "FenceAgentsRemediationTemplate"} 2025-09-12T22:24:31.847795506Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-09-12T22:24:31.847808916Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation CRs {"kind": "FenceAgentsRemediation"} 2025-09-12T22:24:31.848212248Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-09-12T22:24:31.848177965Z DEBUG events [remediation] No issues found, NodeHealthCheck is enabled. {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"298631"}, "reason": "Enabled"} 2025-09-12T22:24:31.848459076Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-0", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"298631"}, "reason": "DetectedUnhealthy"} 2025-09-12T22:24:31.86563667Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:24:31Z" level=info msg="invalidating lease" time="2025-09-12T22:24:31Z" level=info msg="getting lease" 2025-09-12T22:24:32.069833421Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:24:32Z" level=info msg="invalidating lease" time="2025-09-12T22:24:32Z" level=info msg="getting lease" 2025-09-12T22:24:32.080608713Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:24:32.097078149Z INFO controllers.NodeHealthCheck.resource manager Attempting to obtain Node Lease {"Node name": "worker-0-0"} time="2025-09-12T22:24:32Z" level=info msg="request lease" time="2025-09-12T22:24:32Z" level=info msg="getting lease" time="2025-09-12T22:24:32Z" level=info msg="create lease" 2025-09-12T22:24:32.109977719Z INFO controllers.NodeHealthCheck.resource manager Creating a remediation CR {"CR name": "", "CR kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} 2025-09-12T22:24:32.144925894Z DEBUG events [remediation] Created remediation object for node worker-0-0 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"298631"}, "reason": "RemediationCreated"} 2025-09-12T22:24:32.155227044Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-far-worker", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-wrm5v","uid":"9f455ee6-8d40-42de-8a37-f8abf2f55efd","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1"},"started":"2025-09-12T22:24:32Z","templateName":"fenceagentsremediationtemplate-test"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-09-12T22:24:31Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-09-12T22:19:21Z"}, "patch": "{\"status\":{\"conditions\":[{\"lastTransitionTime\":\"2025-09-12T22:24:31Z\",\"message\":\"No issues found, NodeHealthCheck is enabled.\",\"reason\":\"NodeHealthCheckEnabled\",\"status\":\"False\",\"type\":\"Disabled\"}],\"healthyNodes\":2,\"observedNodes\":3,\"phase\":\"Remediating\",\"reason\":\"NHC is remediating 1 nodes\",\"unhealthyNodes\":[{\"name\":\"worker-0-0\",\"remediations\":[{\"resource\":{\"apiVersion\":\"fence-agents-remediation.medik8s.io/v1alpha1\",\"kind\":\"FenceAgentsRemediation\",\"name\":\"worker-0-0-wrm5v\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"9f455ee6-8d40-42de-8a37-f8abf2f55efd\"},\"started\":\"2025-09-12T22:24:32Z\",\"templateName\":\"fenceagentsremediationtemplate-test\"}]}]}}"} 2025-09-12T22:24:32.298122358Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:24:32.29830527Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:24:32.298335437Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:24:32.29835125Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:24:32.319802266Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:24:32.319882751Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:24:32.319909837Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:24:32.319958001Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:24:32.376629337Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-09-12T22:24:32.376968831Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:24:32.437927486Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-09-12T22:24:32.525377632Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:32Z"} 2025-09-12T22:24:32.525491986Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:24:32Z" level=info msg="invalidating lease" time="2025-09-12T22:24:32Z" level=info msg="getting lease" 2025-09-12T22:24:32.641351667Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:24:32Z" level=info msg="invalidating lease" time="2025-09-12T22:24:32Z" level=info msg="getting lease" 2025-09-12T22:24:32.766865786Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:24:32.901444433Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-0-wrm5v", "kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} time="2025-09-12T22:24:32Z" level=info msg="getting lease" 2025-09-12T22:24:32.901740199Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-far-worker", "lease name": "node-worker-0-0", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2025-09-12T22:24:33.006966239Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-09-12T22:24:39.377592515Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:24:39.37771064Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:24:39.377778918Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:24:39.377805195Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:24:39.377936976Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:24:39.396574252Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-09-12T22:24:39.41302455Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:32Z"} 2025-09-12T22:24:39.413151191Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:24:39Z" level=info msg="invalidating lease" time="2025-09-12T22:24:39Z" level=info msg="getting lease" 2025-09-12T22:24:39.513481573Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:24:39Z" level=info msg="invalidating lease" time="2025-09-12T22:24:39Z" level=info msg="getting lease" 2025-09-12T22:24:39.521183087Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:24:39.614778841Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-0-wrm5v", "kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} time="2025-09-12T22:24:39Z" level=info msg="getting lease" 2025-09-12T22:24:39.615039767Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-far-worker", "lease name": "node-worker-0-0", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2025-09-12T22:24:39.721977754Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-09-12T22:24:42.02281427Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:24:42.022923956Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:24:42.022975487Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:24:42.023019838Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:24:42.023118609Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:24:42.031943189Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-09-12T22:24:42.042138413Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:24:42.042183543Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:24:42Z" level=info msg="invalidating lease" time="2025-09-12T22:24:42Z" level=info msg="getting lease" 2025-09-12T22:24:42.046866116Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:24:42Z" level=info msg="invalidating lease" time="2025-09-12T22:24:42Z" level=info msg="getting lease" 2025-09-12T22:24:42.051810906Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:24:42.062820006Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-0-wrm5v", "kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} time="2025-09-12T22:24:42Z" level=info msg="getting lease" 2025-09-12T22:24:42.062944828Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-far-worker", "lease name": "node-worker-0-0", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2025-09-12T22:24:42.067706698Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-09-12T22:26:25.112111392Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-far-worker"} 2025-09-12T22:26:25.112638817Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-far-worker"} 2025-09-12T22:26:25.112790191Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:26:25.238291405Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:26:25.238396555Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:26:25.249576548Z INFO controllers.NodeHealthCheck.resource manager setting a delay for node getting healthy {"node name": "worker-0-0", "delay in seconds": 300} 2025-09-12T22:26:25.291971243Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} 2025-09-12T22:26:25.29384517Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:26:25.2940408Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:26:25.294083508Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:26:25.294104063Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} time="2025-09-12T22:26:25Z" level=info msg="invalidating lease" time="2025-09-12T22:26:25Z" level=info msg="getting lease" 2025-09-12T22:26:25.302391779Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:26:25Z" level=info msg="invalidating lease" time="2025-09-12T22:26:25Z" level=info msg="getting lease" 2025-09-12T22:26:25.312577634Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-far-worker", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-wrm5v","uid":"9f455ee6-8d40-42de-8a37-f8abf2f55efd","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1"},"started":"2025-09-12T22:24:32Z","templateName":"fenceagentsremediationtemplate-test"}],"conditionsHealthyTimestamp":"2025-09-12T22:26:25Z","healthyDelayed":true}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-09-12T22:24:31Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-09-12T22:24:32Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"conditionsHealthyTimestamp\":\"2025-09-12T22:26:25Z\",\"healthyDelayed\":true,\"name\":\"worker-0-0\",\"remediations\":[{\"resource\":{\"apiVersion\":\"fence-agents-remediation.medik8s.io/v1alpha1\",\"kind\":\"FenceAgentsRemediation\",\"name\":\"worker-0-0-wrm5v\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"9f455ee6-8d40-42de-8a37-f8abf2f55efd\"},\"started\":\"2025-09-12T22:24:32Z\",\"templateName\":\"fenceagentsremediationtemplate-test\"}]}]}}"} 2025-09-12T22:26:25.539956438Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:26:25.540219753Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:26:25.570408715Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:26:25.570491445Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:26:25.673973123Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 299.326038426} 2025-09-12T22:26:25.674147526Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:26:25Z" level=info msg="invalidating lease" time="2025-09-12T22:26:25Z" level=info msg="getting lease" 2025-09-12T22:26:25.771582354Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:26:25Z" level=info msg="invalidating lease" time="2025-09-12T22:26:25Z" level=info msg="getting lease" 2025-09-12T22:26:25.872430808Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:26:36.541103558Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:26:36.573863682Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:26:36.573954295Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:26:36Z" level=info msg="invalidating lease" time="2025-09-12T22:26:36Z" level=info msg="getting lease" 2025-09-12T22:26:36.584214435Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:26:36Z" level=info msg="invalidating lease" time="2025-09-12T22:26:36Z" level=info msg="getting lease" 2025-09-12T22:26:36.591365787Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:26:36.599167815Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 288.400842005} 2025-09-12T22:26:36.599828595Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:26:47.601163844Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:26:47.638486921Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:26:47.638580658Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:26:47.646430449Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 277.353575174} 2025-09-12T22:26:47.646536209Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:26:47Z" level=info msg="invalidating lease" time="2025-09-12T22:26:47Z" level=info msg="getting lease" 2025-09-12T22:26:47.654488168Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:26:47Z" level=info msg="invalidating lease" time="2025-09-12T22:26:47Z" level=info msg="getting lease" 2025-09-12T22:26:47.664623371Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:26:58.665648267Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:26:58.711665692Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:26:58.71176745Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:26:58.722451181Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 266.277557669} 2025-09-12T22:26:58.722555922Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:26:58Z" level=info msg="invalidating lease" time="2025-09-12T22:26:58Z" level=info msg="getting lease" 2025-09-12T22:26:58.732394473Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:26:58Z" level=info msg="invalidating lease" time="2025-09-12T22:26:58Z" level=info msg="getting lease" 2025-09-12T22:26:58.744401806Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:27:09.744925721Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:27:09.772643977Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:27:09.772727424Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:27:09Z" level=info msg="invalidating lease" time="2025-09-12T22:27:09Z" level=info msg="getting lease" 2025-09-12T22:27:09.779592868Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:27:09.785901039Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 255.214104254} 2025-09-12T22:27:09.785958119Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:27:09Z" level=info msg="invalidating lease" time="2025-09-12T22:27:09Z" level=info msg="getting lease" 2025-09-12T22:27:09.795070155Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:27:20.796339703Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:27:20.831089337Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:27:20.8311802Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:27:20.839975722Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 244.160031088} 2025-09-12T22:27:20.84011207Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:27:20Z" level=info msg="invalidating lease" time="2025-09-12T22:27:20Z" level=info msg="getting lease" 2025-09-12T22:27:20.849157702Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:27:20Z" level=info msg="invalidating lease" time="2025-09-12T22:27:20Z" level=info msg="getting lease" 2025-09-12T22:27:20.858430478Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:27:31.858824204Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:27:31.894843903Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:27:31.894928533Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:27:31.903176223Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 233.096830394} 2025-09-12T22:27:31.903249366Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:27:31Z" level=info msg="invalidating lease" time="2025-09-12T22:27:31Z" level=info msg="getting lease" 2025-09-12T22:27:31.910643693Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:27:31Z" level=info msg="invalidating lease" time="2025-09-12T22:27:31Z" level=info msg="getting lease" 2025-09-12T22:27:31.919861776Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:27:42.920867356Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:27:42.961306766Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:27:42.96143158Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:27:42Z" level=info msg="invalidating lease" time="2025-09-12T22:27:42Z" level=info msg="getting lease" 2025-09-12T22:27:42.970708313Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:27:42Z" level=info msg="invalidating lease" time="2025-09-12T22:27:42Z" level=info msg="getting lease" 2025-09-12T22:27:42.981322681Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:27:42.989771615Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 222.010235789} 2025-09-12T22:27:42.990639348Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:27:53.991660759Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:27:54.02919274Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:27:54.029279927Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:27:54.037921015Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 210.962084225} 2025-09-12T22:27:54.038010465Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:27:54Z" level=info msg="invalidating lease" time="2025-09-12T22:27:54Z" level=info msg="getting lease" 2025-09-12T22:27:54.045877706Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:27:54Z" level=info msg="invalidating lease" time="2025-09-12T22:27:54Z" level=info msg="getting lease" 2025-09-12T22:27:54.055503261Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:28:05.05689282Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:28:05.089928048Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:28:05.090056455Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:28:05.098268384Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 199.901745689} 2025-09-12T22:28:05.098368799Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:28:05Z" level=info msg="invalidating lease" time="2025-09-12T22:28:05Z" level=info msg="getting lease" 2025-09-12T22:28:05.107548431Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:28:05Z" level=info msg="invalidating lease" time="2025-09-12T22:28:05Z" level=info msg="getting lease" 2025-09-12T22:28:05.11559811Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:28:16.116677784Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:28:16.156864348Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:28:16.157075209Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:28:16Z" level=info msg="invalidating lease" time="2025-09-12T22:28:16Z" level=info msg="getting lease" 2025-09-12T22:28:16.167902151Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:28:16.177646233Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 188.822361424} 2025-09-12T22:28:16.177738926Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:28:16Z" level=info msg="invalidating lease" time="2025-09-12T22:28:16Z" level=info msg="getting lease" 2025-09-12T22:28:16.187347135Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:28:27.187858681Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:28:27.220526498Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:28:27.220693284Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:28:27.228612153Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 177.771393933} 2025-09-12T22:28:27.228689621Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:28:27Z" level=info msg="invalidating lease" time="2025-09-12T22:28:27Z" level=info msg="getting lease" 2025-09-12T22:28:27.237147538Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:28:27Z" level=info msg="invalidating lease" time="2025-09-12T22:28:27Z" level=info msg="getting lease" 2025-09-12T22:28:27.244569134Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:28:38.245895087Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:28:38.287913467Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:28:38.288244779Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:28:38.387831246Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 166.612175747} 2025-09-12T22:28:38.387910103Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:28:38Z" level=info msg="invalidating lease" time="2025-09-12T22:28:38Z" level=info msg="getting lease" 2025-09-12T22:28:38.399513751Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:28:38Z" level=info msg="invalidating lease" time="2025-09-12T22:28:38Z" level=info msg="getting lease" 2025-09-12T22:28:38.411698275Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:28:49.414175219Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:28:49.45627426Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:28:49.456358283Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:28:49Z" level=info msg="invalidating lease" time="2025-09-12T22:28:49Z" level=info msg="getting lease" 2025-09-12T22:28:49.46578824Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:28:49.473581841Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 155.526426435} 2025-09-12T22:28:49.473646515Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:28:49Z" level=info msg="invalidating lease" time="2025-09-12T22:28:49Z" level=info msg="getting lease" 2025-09-12T22:28:49.481285545Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:29:00.481806722Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:29:00.519554188Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:29:00.519661591Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:29:00Z" level=info msg="invalidating lease" time="2025-09-12T22:29:00Z" level=info msg="getting lease" 2025-09-12T22:29:00.619129194Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:29:00.627153789Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 144.372852208} 2025-09-12T22:29:00.627227549Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:29:00Z" level=info msg="invalidating lease" time="2025-09-12T22:29:00Z" level=info msg="getting lease" 2025-09-12T22:29:00.635484733Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:29:11.637080764Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:29:11.705205691Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:29:11.705279224Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:29:11.713657948Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 133.286346886} 2025-09-12T22:29:11.713815385Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:29:11Z" level=info msg="invalidating lease" time="2025-09-12T22:29:11Z" level=info msg="getting lease" 2025-09-12T22:29:11.722654354Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:29:11Z" level=info msg="invalidating lease" time="2025-09-12T22:29:11Z" level=info msg="getting lease" 2025-09-12T22:29:11.734089011Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:29:22.735522576Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:29:22.77563265Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:29:22.775719618Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:29:22.785587515Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 122.214419335} 2025-09-12T22:29:22.785664671Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:29:22Z" level=info msg="invalidating lease" time="2025-09-12T22:29:22Z" level=info msg="getting lease" 2025-09-12T22:29:22.875603451Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:29:22Z" level=info msg="invalidating lease" time="2025-09-12T22:29:22Z" level=info msg="getting lease" 2025-09-12T22:29:22.884444041Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:29:33.884763043Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:29:33.921407033Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:29:33.921502131Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:29:33.933117793Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 111.066890783} 2025-09-12T22:29:33.933211473Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:29:33Z" level=info msg="invalidating lease" time="2025-09-12T22:29:33Z" level=info msg="getting lease" 2025-09-12T22:29:33.942287144Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:29:33Z" level=info msg="invalidating lease" time="2025-09-12T22:29:33Z" level=info msg="getting lease" 2025-09-12T22:29:33.95287361Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:29:44.953495741Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:29:44.992332946Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:29:44.992413162Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:29:45Z" level=info msg="invalidating lease" time="2025-09-12T22:29:45Z" level=info msg="getting lease" 2025-09-12T22:29:45.000600594Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:29:45.008278339Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 99.991730801} 2025-09-12T22:29:45.008365924Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:29:45Z" level=info msg="invalidating lease" time="2025-09-12T22:29:45Z" level=info msg="getting lease" 2025-09-12T22:29:45.026587911Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:29:56.027763153Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:29:56.063072402Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:29:56.063157266Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:29:56.16481716Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 88.835189284} 2025-09-12T22:29:56.164896421Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:29:56Z" level=info msg="invalidating lease" time="2025-09-12T22:29:56Z" level=info msg="getting lease" 2025-09-12T22:29:56.173132254Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:29:56Z" level=info msg="invalidating lease" time="2025-09-12T22:29:56Z" level=info msg="getting lease" 2025-09-12T22:29:56.18364392Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:30:07.184129674Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:30:07.216823191Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:30:07.216920867Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:30:07.22538027Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 77.774626564} 2025-09-12T22:30:07.225454256Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:30:07Z" level=info msg="invalidating lease" time="2025-09-12T22:30:07Z" level=info msg="getting lease" 2025-09-12T22:30:07.318680505Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:30:07Z" level=info msg="invalidating lease" time="2025-09-12T22:30:07Z" level=info msg="getting lease" 2025-09-12T22:30:07.327825485Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:30:18.328864824Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:30:18.364908616Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:30:18.365050304Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:30:18.374734877Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 66.62527293} 2025-09-12T22:30:18.37484103Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:30:18Z" level=info msg="invalidating lease" time="2025-09-12T22:30:18Z" level=info msg="getting lease" 2025-09-12T22:30:18.382793974Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:30:18Z" level=info msg="invalidating lease" time="2025-09-12T22:30:18Z" level=info msg="getting lease" 2025-09-12T22:30:18.394563187Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:30:29.395380521Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:30:29.436656634Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:30:29.436743165Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:30:29Z" level=info msg="invalidating lease" time="2025-09-12T22:30:29Z" level=info msg="getting lease" 2025-09-12T22:30:29.446970152Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:30:29.454750065Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 55.545255912} 2025-09-12T22:30:29.454838538Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:30:29Z" level=info msg="invalidating lease" time="2025-09-12T22:30:29Z" level=info msg="getting lease" 2025-09-12T22:30:29.465867273Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:30:40.46673364Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:30:40.502084063Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:30:40.502168324Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:30:40.513262701Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 44.486742766} 2025-09-12T22:30:40.513330304Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:30:40Z" level=info msg="invalidating lease" time="2025-09-12T22:30:40Z" level=info msg="getting lease" 2025-09-12T22:30:40.52149675Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:30:40Z" level=info msg="invalidating lease" time="2025-09-12T22:30:40Z" level=info msg="getting lease" 2025-09-12T22:30:40.531838766Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:30:51.532909899Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:30:51.569308742Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:30:51.569381483Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:30:51.576841158Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 33.423166795} 2025-09-12T22:30:51.576927972Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:30:51Z" level=info msg="invalidating lease" time="2025-09-12T22:30:51Z" level=info msg="getting lease" 2025-09-12T22:30:51.584199522Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:30:51Z" level=info msg="invalidating lease" time="2025-09-12T22:30:51Z" level=info msg="getting lease" 2025-09-12T22:30:51.594295154Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:31:02.595396827Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:31:02.636334476Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:31:02.636420606Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:31:02Z" level=info msg="invalidating lease" time="2025-09-12T22:31:02Z" level=info msg="getting lease" 2025-09-12T22:31:02.646126284Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:31:02.655871388Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 22.344136029} 2025-09-12T22:31:02.655958208Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:31:02Z" level=info msg="invalidating lease" time="2025-09-12T22:31:02Z" level=info msg="getting lease" 2025-09-12T22:31:02.666423379Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:31:13.667560792Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:31:13.711452055Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:31:13.711577251Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:31:13.72094193Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 11.279063887} 2025-09-12T22:31:13.72106349Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:31:13Z" level=info msg="invalidating lease" time="2025-09-12T22:31:13Z" level=info msg="getting lease" 2025-09-12T22:31:13.73197125Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:31:13Z" level=info msg="invalidating lease" time="2025-09-12T22:31:13Z" level=info msg="getting lease" 2025-09-12T22:31:13.741866881Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:31:24.743067104Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:31:24.774196165Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:31:24.774299462Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:31:24Z" level=info msg="invalidating lease" time="2025-09-12T22:31:24Z" level=info msg="getting lease" 2025-09-12T22:31:24.786514612Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:31:24.796466197Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-0", "remaining time in seconds": 0.203539629} 2025-09-12T22:31:24.796532567Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:31:24Z" level=info msg="invalidating lease" time="2025-09-12T22:31:24Z" level=info msg="getting lease" 2025-09-12T22:31:24.805853592Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "1.203539629s"} 2025-09-12T22:31:26.010748255Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:31:26.045702204Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:24:42Z"} 2025-09-12T22:31:26.045780779Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:31:26.054376195Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-0"} 2025-09-12T22:31:26.086192187Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:31:26.086372071Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:31:26.086418224Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:31:26.086446667Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:31:26.087227809Z INFO controllers.NodeHealthCheck.resource manager deleted remediation CR {"name": "worker-0-0-wrm5v"} 2025-09-12T22:31:26.087287375Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} 2025-09-12T22:31:26.087405095Z DEBUG events [remediation] Deleted remediation CR of kind FenceAgentsRemediation with name worker-0-0-wrm5v {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"302072"}, "reason": "RemediationRemoved"} time="2025-09-12T22:31:26Z" level=info msg="invalidating lease" time="2025-09-12T22:31:26Z" level=info msg="getting lease" 2025-09-12T22:31:26.097884324Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:31:26Z" level=info msg="invalidating lease" time="2025-09-12T22:31:26Z" level=info msg="getting lease" 2025-09-12T22:31:26.151797676Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:31:26.154148617Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:31:26.20161191Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:31:26.201735174Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-wrm5v", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:31:26.245606966Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-12T22:31:26Z" level=info msg="invalidating lease" time="2025-09-12T22:31:26Z" level=info msg="getting lease" 2025-09-12T22:31:26.398161745Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:31:26Z" level=info msg="invalidating lease" time="2025-09-12T22:31:26Z" level=info msg="getting lease" 2025-09-12T22:31:26.447935938Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:31:26Z" level=info msg="invalidating lease" time="2025-09-12T22:31:26Z" level=info msg="getting lease" 2025-09-12T22:31:26.46147493Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-far-worker", "new status": {"observedNodes":3,"healthyNodes":3,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-09-12T22:24:31Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-09-12T22:26:25Z"}, "patch": "{\"status\":{\"healthyNodes\":3,\"phase\":\"Enabled\",\"reason\":\"NHC is enabled, no ongoing remediation\",\"unhealthyNodes\":null}}"} 2025-09-12T22:31:26.684749382Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-09-12T22:31:26.68494506Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:31:26.725669428Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:31:26Z" level=info msg="invalidating lease" time="2025-09-12T22:31:26Z" level=info msg="getting lease" 2025-09-12T22:31:26.734227562Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-12T22:31:26Z" level=info msg="invalidating lease" time="2025-09-12T22:31:26Z" level=info msg="getting lease" 2025-09-12T22:31:26.741845687Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:31:26Z" level=info msg="invalidating lease" time="2025-09-12T22:31:26Z" level=info msg="getting lease" 2025-09-12T22:31:26.749938971Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-09-12T22:31:37.15245759Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:31:37.187648686Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-12T22:31:37Z" level=info msg="invalidating lease" time="2025-09-12T22:31:37Z" level=info msg="getting lease" 2025-09-12T22:31:37.193474879Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:31:37Z" level=info msg="invalidating lease" time="2025-09-12T22:31:37Z" level=info msg="getting lease" 2025-09-12T22:31:37.201880661Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:31:37Z" level=info msg="invalidating lease" time="2025-09-12T22:31:37Z" level=info msg="getting lease" 2025-09-12T22:31:37.210639016Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-09-12T22:40:00.0252766Z INFO nodehealthcheck-resource validate update {"name": "nhc-far-worker"} 2025-09-12T22:40:00.037846022Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:40:00.070131171Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} time="2025-09-12T22:40:00Z" level=info msg="invalidating lease" time="2025-09-12T22:40:00Z" level=info msg="getting lease" 2025-09-12T22:40:00.076679569Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:40:00Z" level=info msg="invalidating lease" time="2025-09-12T22:40:00Z" level=info msg="getting lease" 2025-09-12T22:40:00.083724807Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:40:00Z" level=info msg="invalidating lease" time="2025-09-12T22:40:00Z" level=info msg="getting lease" 2025-09-12T22:40:00.089926504Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-09-12T22:41:26.673394942Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-far-worker"} 2025-09-12T22:41:26.673640746Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-far-worker"} 2025-09-12T22:41:26.673783474Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:41:26.710907036Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown", "duration left": "29.289100981s"} 2025-09-12T22:41:26.767367584Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:41:26Z" level=info msg="invalidating lease" time="2025-09-12T22:41:26Z" level=info msg="getting lease" 2025-09-12T22:41:26.868413278Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:41:26Z" level=info msg="invalidating lease" time="2025-09-12T22:41:26Z" level=info msg="getting lease" 2025-09-12T22:41:26.970507853Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-far-worker", "new status": {"observedNodes":3,"healthyNodes":2,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-09-12T22:24:31Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-09-12T22:31:26Z"}, "patch": "{\"status\":{\"healthyNodes\":2}}"} 2025-09-12T22:41:27.213102001Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "30.289100981s"} 2025-09-12T22:41:57.503360236Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:41:57.522227486Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-09-12T22:41:57.522663431Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-0", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"307676"}, "reason": "DetectedUnhealthy"} 2025-09-12T22:41:57.54609175Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:41:57Z" level=info msg="invalidating lease" time="2025-09-12T22:41:57Z" level=info msg="getting lease" 2025-09-12T22:41:57.647820438Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:41:57Z" level=info msg="invalidating lease" time="2025-09-12T22:41:57Z" level=info msg="getting lease" 2025-09-12T22:41:57.655470394Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:41:57.672286291Z INFO controllers.NodeHealthCheck.resource manager Attempting to obtain Node Lease {"Node name": "worker-0-0"} time="2025-09-12T22:41:57Z" level=info msg="request lease" time="2025-09-12T22:41:57Z" level=info msg="getting lease" time="2025-09-12T22:41:57Z" level=info msg="create lease" 2025-09-12T22:41:57.684120671Z INFO controllers.NodeHealthCheck.resource manager Creating a remediation CR {"CR name": "", "CR kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} 2025-09-12T22:41:57.71565082Z DEBUG events [remediation] Created remediation object for node worker-0-0 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"307676"}, "reason": "RemediationCreated"} 2025-09-12T22:41:57.741180446Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:41:57.741413164Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-nsmhb", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:41:57.74145134Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:41:57.741474881Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-nsmhb", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:41:57.74207864Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-far-worker", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-nsmhb","uid":"627403c2-464d-49c7-ac5c-59ccfc267bf9","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1"},"started":"2025-09-12T22:41:57Z","templateName":"fenceagentsremediationtemplate-test"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-09-12T22:24:31Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-09-12T22:41:26Z"}, "patch": "{\"status\":{\"phase\":\"Remediating\",\"reason\":\"NHC is remediating 1 nodes\",\"unhealthyNodes\":[{\"name\":\"worker-0-0\",\"remediations\":[{\"resource\":{\"apiVersion\":\"fence-agents-remediation.medik8s.io/v1alpha1\",\"kind\":\"FenceAgentsRemediation\",\"name\":\"worker-0-0-nsmhb\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"627403c2-464d-49c7-ac5c-59ccfc267bf9\"},\"started\":\"2025-09-12T22:41:57Z\",\"templateName\":\"fenceagentsremediationtemplate-test\"}]}]}}"} 2025-09-12T22:41:57.762954594Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:41:57.763068167Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-nsmhb", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:41:57.76309387Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:41:57.763110644Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-nsmhb", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:41:57.962705479Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-09-12T22:41:57.962979266Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:41:57.979164844Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-09-12T22:41:58.003156145Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-09-12T22:41:57Z"} 2025-09-12T22:41:58.00328229Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:41:58Z" level=info msg="invalidating lease" time="2025-09-12T22:41:58Z" level=info msg="getting lease" 2025-09-12T22:41:58.015650515Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:41:58Z" level=info msg="invalidating lease" time="2025-09-12T22:41:58Z" level=info msg="getting lease" 2025-09-12T22:41:58.100340418Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:41:58.203656844Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-0-nsmhb", "kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} time="2025-09-12T22:41:58Z" level=info msg="getting lease" 2025-09-12T22:41:58.203937089Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-far-worker", "lease name": "node-worker-0-0", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2025-09-12T22:41:58.301134423Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-09-12T22:42:04.594341359Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:42:04.594443673Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-nsmhb", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:42:04.59450527Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:42:04.59453787Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-nsmhb", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:42:04.59466463Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:42:04.611712268Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-09-12T22:42:04.629731955Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-09-12T22:41:57Z"} 2025-09-12T22:42:04.629830915Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:42:04Z" level=info msg="invalidating lease" time="2025-09-12T22:42:04Z" level=info msg="getting lease" 2025-09-12T22:42:04.638183134Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:42:04Z" level=info msg="invalidating lease" time="2025-09-12T22:42:04Z" level=info msg="getting lease" 2025-09-12T22:42:04.647752289Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:42:04.663583399Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-0-nsmhb", "kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} time="2025-09-12T22:42:04Z" level=info msg="getting lease" 2025-09-12T22:42:04.663755438Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-far-worker", "lease name": "node-worker-0-0", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2025-09-12T22:42:04.673038145Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-09-12T22:42:07.067743197Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:42:07.067812935Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-nsmhb", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:42:07.067838952Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-far-worker","uid":"d98f1ab0-deb1-4601-9b9e-5f0e9a1c52a4","controller":false}} 2025-09-12T22:42:07.067908868Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-far-worker", "Remediation CR Name": "worker-0-0-nsmhb", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-09-12T22:42:07.06801569Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:42:07.084549768Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-0", "condition type": "Ready", "condition status": "Unknown"} 2025-09-12T22:42:07.099683737Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:42:07.099771757Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:42:07Z" level=info msg="invalidating lease" time="2025-09-12T22:42:07Z" level=info msg="getting lease" 2025-09-12T22:42:07.105342115Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:42:07Z" level=info msg="invalidating lease" time="2025-09-12T22:42:07Z" level=info msg="getting lease" 2025-09-12T22:42:07.110516224Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:42:07.120904066Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-0-nsmhb", "kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} time="2025-09-12T22:42:07Z" level=info msg="getting lease" 2025-09-12T22:42:07.121108438Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-far-worker", "lease name": "node-worker-0-0", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2025-09-12T22:42:07.126343219Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-09-12T22:43:43.988175098Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-far-worker"} 2025-09-12T22:43:43.988596322Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-0", "NHC": "nhc-far-worker"} 2025-09-12T22:43:43.988927024Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:43:44.056522939Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:43:44.056691164Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:43:44Z" level=info msg="invalidating lease" time="2025-09-12T22:43:44Z" level=info msg="getting lease" 2025-09-12T22:43:44.068327601Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:43:44Z" level=info msg="invalidating lease" time="2025-09-12T22:43:44Z" level=info msg="getting lease" 2025-09-12T22:43:44.077702992Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:43:44.08751065Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-far-worker", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-0","remediations":[{"resource":{"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-0-nsmhb","uid":"627403c2-464d-49c7-ac5c-59ccfc267bf9","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1"},"started":"2025-09-12T22:41:57Z","templateName":"fenceagentsremediationtemplate-test"}],"conditionsHealthyTimestamp":"2025-09-12T22:43:44Z"}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-09-12T22:24:31Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-09-12T22:41:57Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"conditionsHealthyTimestamp\":\"2025-09-12T22:43:44Z\",\"name\":\"worker-0-0\",\"remediations\":[{\"resource\":{\"apiVersion\":\"fence-agents-remediation.medik8s.io/v1alpha1\",\"kind\":\"FenceAgentsRemediation\",\"name\":\"worker-0-0-nsmhb\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"627403c2-464d-49c7-ac5c-59ccfc267bf9\"},\"started\":\"2025-09-12T22:41:57Z\",\"templateName\":\"fenceagentsremediationtemplate-test\"}]}]}}"} 2025-09-12T22:43:44.314644511Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:43:55.315374006Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:43:55.349617306Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:43:55.349702559Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:43:55.357694608Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:43:55Z" level=info msg="invalidating lease" time="2025-09-12T22:43:55Z" level=info msg="getting lease" 2025-09-12T22:43:55.365947754Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:43:55Z" level=info msg="invalidating lease" time="2025-09-12T22:43:55Z" level=info msg="getting lease" 2025-09-12T22:43:55.376528568Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:44:06.377793834Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:44:06.419597198Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:44:06.419686298Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:44:06Z" level=info msg="invalidating lease" time="2025-09-12T22:44:06Z" level=info msg="getting lease" 2025-09-12T22:44:06.428786247Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:44:06.441009698Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:44:06Z" level=info msg="invalidating lease" time="2025-09-12T22:44:06Z" level=info msg="getting lease" 2025-09-12T22:44:06.521376155Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:44:17.522119378Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:44:17.554906585Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:44:17.555052393Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:44:17.563285145Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:44:17Z" level=info msg="invalidating lease" time="2025-09-12T22:44:17Z" level=info msg="getting lease" 2025-09-12T22:44:17.570613305Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:44:17Z" level=info msg="invalidating lease" time="2025-09-12T22:44:17Z" level=info msg="getting lease" 2025-09-12T22:44:17.579326387Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:44:28.579662679Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:44:28.616972763Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:44:28.617152137Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:44:28.625827512Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:44:28Z" level=info msg="invalidating lease" time="2025-09-12T22:44:28Z" level=info msg="getting lease" 2025-09-12T22:44:28.634049978Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:44:28Z" level=info msg="invalidating lease" time="2025-09-12T22:44:28Z" level=info msg="getting lease" 2025-09-12T22:44:28.643245471Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:44:39.644475562Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:44:39.680764311Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:44:39.680867756Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:44:39.68871483Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:44:39Z" level=info msg="invalidating lease" time="2025-09-12T22:44:39Z" level=info msg="getting lease" 2025-09-12T22:44:39.697044346Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:44:39Z" level=info msg="invalidating lease" time="2025-09-12T22:44:39Z" level=info msg="getting lease" 2025-09-12T22:44:39.704937975Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:44:50.705959886Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:44:50.738865384Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:44:50.739146265Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:44:50.747423552Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:44:50Z" level=info msg="invalidating lease" time="2025-09-12T22:44:50Z" level=info msg="getting lease" 2025-09-12T22:44:50.755611184Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:44:50Z" level=info msg="invalidating lease" time="2025-09-12T22:44:50Z" level=info msg="getting lease" 2025-09-12T22:44:50.764871664Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:45:01.765475776Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:45:01.816096678Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:45:01.816174544Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:45:01.823822028Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:45:01Z" level=info msg="invalidating lease" time="2025-09-12T22:45:01Z" level=info msg="getting lease" 2025-09-12T22:45:01.835096696Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:45:01Z" level=info msg="invalidating lease" time="2025-09-12T22:45:01Z" level=info msg="getting lease" 2025-09-12T22:45:01.845151094Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:45:12.845955474Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:45:12.888721731Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:45:12.888799874Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:45:12Z" level=info msg="invalidating lease" time="2025-09-12T22:45:12Z" level=info msg="getting lease" 2025-09-12T22:45:12.897717473Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:45:12.907417278Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:45:12Z" level=info msg="invalidating lease" time="2025-09-12T22:45:12Z" level=info msg="getting lease" 2025-09-12T22:45:12.91636957Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:45:23.917392895Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:45:23.952785168Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:45:23.952886346Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:45:23.960104588Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:45:23Z" level=info msg="invalidating lease" time="2025-09-12T22:45:23Z" level=info msg="getting lease" 2025-09-12T22:45:23.968623132Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:45:23Z" level=info msg="invalidating lease" time="2025-09-12T22:45:23Z" level=info msg="getting lease" 2025-09-12T22:45:23.978582288Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:45:34.979455266Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:45:35.014073132Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:45:35.014161777Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:45:35.021185638Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:45:35Z" level=info msg="invalidating lease" time="2025-09-12T22:45:35Z" level=info msg="getting lease" 2025-09-12T22:45:35.028638822Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:45:35Z" level=info msg="invalidating lease" time="2025-09-12T22:45:35Z" level=info msg="getting lease" 2025-09-12T22:45:35.036535483Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:45:46.037931068Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:45:46.076705856Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:45:46.076828543Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:45:46.178057395Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:45:46Z" level=info msg="invalidating lease" time="2025-09-12T22:45:46Z" level=info msg="getting lease" 2025-09-12T22:45:46.185377222Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:45:46Z" level=info msg="invalidating lease" time="2025-09-12T22:45:46Z" level=info msg="getting lease" 2025-09-12T22:45:46.194562972Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:45:57.195505111Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:45:57.232734117Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:45:57.232800731Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:45:57.242733239Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:45:57Z" level=info msg="invalidating lease" time="2025-09-12T22:45:57Z" level=info msg="getting lease" 2025-09-12T22:45:57.344701863Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:45:57Z" level=info msg="invalidating lease" time="2025-09-12T22:45:57Z" level=info msg="getting lease" 2025-09-12T22:45:57.35616055Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:46:08.356926641Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:46:08.391447714Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:46:08.391544705Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:46:08.492707791Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:46:08Z" level=info msg="invalidating lease" time="2025-09-12T22:46:08Z" level=info msg="getting lease" 2025-09-12T22:46:08.50056993Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:46:08Z" level=info msg="invalidating lease" time="2025-09-12T22:46:08Z" level=info msg="getting lease" 2025-09-12T22:46:08.507287764Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:46:19.508581165Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:46:19.541405633Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:46:19.541499836Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:46:19.641693582Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:46:19Z" level=info msg="invalidating lease" time="2025-09-12T22:46:19Z" level=info msg="getting lease" 2025-09-12T22:46:19.650050895Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:46:19Z" level=info msg="invalidating lease" time="2025-09-12T22:46:19Z" level=info msg="getting lease" 2025-09-12T22:46:19.658644273Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:46:30.659800537Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:46:30.7020925Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:46:30.702194433Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:46:30.71124855Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:46:30Z" level=info msg="invalidating lease" time="2025-09-12T22:46:30Z" level=info msg="getting lease" 2025-09-12T22:46:30.802975476Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:46:30Z" level=info msg="invalidating lease" time="2025-09-12T22:46:30Z" level=info msg="getting lease" 2025-09-12T22:46:30.812391108Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:46:41.813369055Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:46:41.85435053Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:46:41.854454873Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:46:41.863927985Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:46:41Z" level=info msg="invalidating lease" time="2025-09-12T22:46:41Z" level=info msg="getting lease" 2025-09-12T22:46:41.873546054Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:46:41Z" level=info msg="invalidating lease" time="2025-09-12T22:46:41Z" level=info msg="getting lease" 2025-09-12T22:46:41.888785944Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:46:52.890420206Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:46:52.933731706Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:46:52.933831936Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:46:52.941808422Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:46:52Z" level=info msg="invalidating lease" time="2025-09-12T22:46:52Z" level=info msg="getting lease" 2025-09-12T22:46:52.950497852Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:46:52Z" level=info msg="invalidating lease" time="2025-09-12T22:46:52Z" level=info msg="getting lease" 2025-09-12T22:46:52.9591978Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:47:03.96030035Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:47:03.999142598Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:47:03.999247458Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:47:04.009089007Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:47:04Z" level=info msg="invalidating lease" time="2025-09-12T22:47:04Z" level=info msg="getting lease" 2025-09-12T22:47:04.018672096Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:47:04Z" level=info msg="invalidating lease" time="2025-09-12T22:47:04Z" level=info msg="getting lease" 2025-09-12T22:47:04.026954204Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:47:15.028386001Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:47:15.064942452Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:47:15.065080254Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:47:15Z" level=info msg="invalidating lease" time="2025-09-12T22:47:15Z" level=info msg="getting lease" 2025-09-12T22:47:15.073305844Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:47:15Z" level=info msg="invalidating lease" time="2025-09-12T22:47:15Z" level=info msg="getting lease" 2025-09-12T22:47:15.081201096Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:47:15.08965996Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:47:26.090468016Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:47:26.131172289Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:47:26.131260249Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:47:26.139093679Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:47:26Z" level=info msg="invalidating lease" time="2025-09-12T22:47:26Z" level=info msg="getting lease" 2025-09-12T22:47:26.14838174Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:47:26Z" level=info msg="invalidating lease" time="2025-09-12T22:47:26Z" level=info msg="getting lease" 2025-09-12T22:47:26.155737988Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:47:37.157479396Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:47:37.190058163Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:47:37.190138991Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:47:37.198601248Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:47:37Z" level=info msg="invalidating lease" time="2025-09-12T22:47:37Z" level=info msg="getting lease" 2025-09-12T22:47:37.206794558Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:47:37Z" level=info msg="invalidating lease" time="2025-09-12T22:47:37Z" level=info msg="getting lease" 2025-09-12T22:47:37.216775396Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:47:48.217289293Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:47:48.260940124Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:47:48.261043851Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:47:48.26814767Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:47:48Z" level=info msg="invalidating lease" time="2025-09-12T22:47:48Z" level=info msg="getting lease" 2025-09-12T22:47:48.275836848Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:47:48Z" level=info msg="invalidating lease" time="2025-09-12T22:47:48Z" level=info msg="getting lease" 2025-09-12T22:47:48.28409706Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:47:59.284585653Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:47:59.321551224Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:47:59.32164652Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:47:59.328635223Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:47:59Z" level=info msg="invalidating lease" time="2025-09-12T22:47:59Z" level=info msg="getting lease" 2025-09-12T22:47:59.336223908Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:47:59Z" level=info msg="invalidating lease" time="2025-09-12T22:47:59Z" level=info msg="getting lease" 2025-09-12T22:47:59.345704023Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:48:10.346969234Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:48:10.378491404Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:48:10.378596727Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:48:10.385609423Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:48:10Z" level=info msg="invalidating lease" time="2025-09-12T22:48:10Z" level=info msg="getting lease" 2025-09-12T22:48:10.393664046Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:48:10Z" level=info msg="invalidating lease" time="2025-09-12T22:48:10Z" level=info msg="getting lease" 2025-09-12T22:48:10.411390425Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:48:21.412161107Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:48:21.450402818Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:48:21.450471006Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:48:21.458343808Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:48:21Z" level=info msg="invalidating lease" time="2025-09-12T22:48:21Z" level=info msg="getting lease" 2025-09-12T22:48:21.466415587Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:48:21Z" level=info msg="invalidating lease" time="2025-09-12T22:48:21Z" level=info msg="getting lease" 2025-09-12T22:48:21.474275984Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:48:32.474658779Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:48:32.519684866Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:48:32.519762776Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:48:32.528432884Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:48:32Z" level=info msg="invalidating lease" time="2025-09-12T22:48:32Z" level=info msg="getting lease" 2025-09-12T22:48:32.538143144Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:48:32Z" level=info msg="invalidating lease" time="2025-09-12T22:48:32Z" level=info msg="getting lease" 2025-09-12T22:48:32.548080507Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:48:43.548491022Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:48:43.584081321Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:48:43.584167757Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:48:43.595567336Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:48:43Z" level=info msg="invalidating lease" time="2025-09-12T22:48:43Z" level=info msg="getting lease" 2025-09-12T22:48:43.605274422Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:48:43Z" level=info msg="invalidating lease" time="2025-09-12T22:48:43Z" level=info msg="getting lease" 2025-09-12T22:48:43.614053434Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:48:54.615103874Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:48:54.648190646Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:48:54.648291086Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:48:54Z" level=info msg="invalidating lease" time="2025-09-12T22:48:54Z" level=info msg="getting lease" 2025-09-12T22:48:54.656445474Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:48:54Z" level=info msg="invalidating lease" time="2025-09-12T22:48:54Z" level=info msg="getting lease" 2025-09-12T22:48:54.664446122Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:48:54.676030619Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:49:05.676810359Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:49:05.711190172Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:49:05.711277252Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:49:05.718972715Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:49:05Z" level=info msg="invalidating lease" time="2025-09-12T22:49:05Z" level=info msg="getting lease" 2025-09-12T22:49:05.729693681Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:49:05Z" level=info msg="invalidating lease" time="2025-09-12T22:49:05Z" level=info msg="getting lease" 2025-09-12T22:49:05.73844531Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:49:16.738826836Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:49:16.778068131Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:49:16.778169764Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:49:16.786576889Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:49:16Z" level=info msg="invalidating lease" time="2025-09-12T22:49:16Z" level=info msg="getting lease" 2025-09-12T22:49:16.795303408Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:49:16Z" level=info msg="invalidating lease" time="2025-09-12T22:49:16Z" level=info msg="getting lease" 2025-09-12T22:49:16.804215808Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:49:27.804888467Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:49:27.846078056Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:49:27.846382021Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:49:27.853811224Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:49:27Z" level=info msg="invalidating lease" time="2025-09-12T22:49:27Z" level=info msg="getting lease" 2025-09-12T22:49:27.862025201Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:49:27Z" level=info msg="invalidating lease" time="2025-09-12T22:49:27Z" level=info msg="getting lease" 2025-09-12T22:49:27.87514899Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:49:38.876022804Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:49:38.913752137Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:49:38.913837214Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:49:38.923793914Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:49:38Z" level=info msg="invalidating lease" time="2025-09-12T22:49:38Z" level=info msg="getting lease" 2025-09-12T22:49:38.933076074Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:49:38Z" level=info msg="invalidating lease" time="2025-09-12T22:49:38Z" level=info msg="getting lease" 2025-09-12T22:49:38.941187188Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:49:49.941972215Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:49:49.977203849Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:49:49.977343002Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:49:49Z" level=info msg="invalidating lease" time="2025-09-12T22:49:49Z" level=info msg="getting lease" 2025-09-12T22:49:49.986843075Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:49:49Z" level=info msg="invalidating lease" time="2025-09-12T22:49:49Z" level=info msg="getting lease" 2025-09-12T22:49:49.995174002Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:49:50.004233139Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:50:01.005469137Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:50:01.044785791Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:50:01.044891309Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:50:01.053931862Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:50:01Z" level=info msg="invalidating lease" time="2025-09-12T22:50:01Z" level=info msg="getting lease" 2025-09-12T22:50:01.143831924Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:50:01Z" level=info msg="invalidating lease" time="2025-09-12T22:50:01Z" level=info msg="getting lease" 2025-09-12T22:50:01.155838335Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:50:12.1570456Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:50:12.190097622Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:50:12.190208179Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:50:12.199531787Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:50:12Z" level=info msg="invalidating lease" time="2025-09-12T22:50:12Z" level=info msg="getting lease" 2025-09-12T22:50:12.20847812Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:50:12Z" level=info msg="invalidating lease" time="2025-09-12T22:50:12Z" level=info msg="getting lease" 2025-09-12T22:50:12.21749431Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:50:23.218563534Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:50:23.259381788Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:50:23.259470975Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:50:23.268172616Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:50:23Z" level=info msg="invalidating lease" time="2025-09-12T22:50:23Z" level=info msg="getting lease" 2025-09-12T22:50:23.276463045Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:50:23Z" level=info msg="invalidating lease" time="2025-09-12T22:50:23Z" level=info msg="getting lease" 2025-09-12T22:50:23.284764578Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:50:34.286238719Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:50:34.322737231Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:50:34.322829664Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:50:34.33169996Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:50:34Z" level=info msg="invalidating lease" time="2025-09-12T22:50:34Z" level=info msg="getting lease" 2025-09-12T22:50:34.340367026Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:50:34Z" level=info msg="invalidating lease" time="2025-09-12T22:50:34Z" level=info msg="getting lease" 2025-09-12T22:50:34.350624944Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:50:45.351042936Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:50:45.38665105Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:50:45.386731534Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:50:45.395826309Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:50:45Z" level=info msg="invalidating lease" time="2025-09-12T22:50:45Z" level=info msg="getting lease" 2025-09-12T22:50:45.404545537Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:50:45Z" level=info msg="invalidating lease" time="2025-09-12T22:50:45Z" level=info msg="getting lease" 2025-09-12T22:50:45.416250887Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:50:56.418257814Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:50:56.45420664Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:50:56.45428155Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:50:56.464089683Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:50:56Z" level=info msg="invalidating lease" time="2025-09-12T22:50:56Z" level=info msg="getting lease" 2025-09-12T22:50:56.473724861Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:50:56Z" level=info msg="invalidating lease" time="2025-09-12T22:50:56Z" level=info msg="getting lease" 2025-09-12T22:50:56.482506162Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:51:07.483834289Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:51:07.51925565Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:51:07.51936618Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:51:07.529399948Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:51:07Z" level=info msg="invalidating lease" time="2025-09-12T22:51:07Z" level=info msg="getting lease" 2025-09-12T22:51:07.541962448Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:51:07Z" level=info msg="invalidating lease" time="2025-09-12T22:51:07Z" level=info msg="getting lease" 2025-09-12T22:51:07.554170243Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-09-12T22:51:18.555379244Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-far-worker"} 2025-09-12T22:51:18.588340403Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-far-worker", "type": "Succeeded", "status": "True", "reason": "", "message": "", "lastTransition": "2025-09-12T22:42:07Z"} 2025-09-12T22:51:18.588425799Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-2"} time="2025-09-12T22:51:18Z" level=info msg="invalidating lease" time="2025-09-12T22:51:18Z" level=info msg="getting lease" 2025-09-12T22:51:18.597077171Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-0"} 2025-09-12T22:51:18.605783296Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-far-worker", "node": "worker-0-1"} time="2025-09-12T22:51:18Z" level=info msg="invalidating lease" time="2025-09-12T22:51:18Z" level=info msg="getting lease" 2025-09-12T22:51:18.616034394Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-far-worker", "error": null, "requeue": false, "requeuAfter": "11s"}