[root@cert-rhosp-02 ~]# oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.18.0-0.nightly-2026-02-11-045751 True False 119m Cluster version is 4.18.0-0.nightly-2026-02-11-045751 [root@cert-rhosp-02 ~]# oc project openshift-workload-availability Already on project "openshift-workload-availability" on server "https://api.ocp-edge-cluster-0.qe.lab.redhat.com:6443". [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 165m v1.31.14 master-0-1 Ready control-plane,master 164m v1.31.14 master-0-2 Ready control-plane,master 165m v1.31.14 worker-0-0 Ready worker 132m v1.31.14 worker-0-1 Ready worker 132m v1.31.14 worker-0-2 Ready worker 133m v1.31.14 [root@cert-rhosp-02 ~]# oc get csv NAME DISPLAY VERSION REPLACES PHASE node-healthcheck-operator.v0.10.2 Node Health Check Operator 0.10.2 node-healthcheck-operator.v0.10.1 Succeeded self-node-remediation.v0.11.0 Self Node Remediation Operator 0.11.0 self-node-remediation.v0.10.2 Succeeded [root@cert-rhosp-02 ~]# oc get pods NAME READY STATUS RESTARTS AGE node-healthcheck-controller-manager-7955d56b69-gkp5r 2/2 Running 0 66m node-healthcheck-controller-manager-7955d56b69-v64wh 2/2 Running 0 66m node-healthcheck-node-remediation-console-plugin-857f5d58dslhsg 1/1 Running 0 66m self-node-remediation-controller-manager-748bdbc654-jcttx 2/2 Running 0 62m self-node-remediation-controller-manager-748bdbc654-rc25k 2/2 Running 1 (62m ago) 62m self-node-remediation-ds-8h4pc 1/1 Running 0 61m self-node-remediation-ds-hgrmc 1/1 Running 0 61m self-node-remediation-ds-k8b45 1/1 Running 0 61m self-node-remediation-ds-php5q 1/1 Running 0 61m self-node-remediation-ds-r98ck 1/1 Running 0 61m self-node-remediation-ds-wrzmr 1/1 Running 0 61m [root@cert-rhosp-02 ~]# cat nhc_snr.ymal apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: name: nhc-worker-test spec: minHealthy: 2 healthyDelay: 5s stormCooldownDuration: 60s remediationTemplate: apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate name: selfnoderemediationtemplate-sample namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/worker operator: Exists unhealthyConditions: - duration: 30s status: 'False' type: Ready - duration: 30s status: Unknown type: Ready --- apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate metadata: namespace: openshift-workload-availability name: selfnoderemediationtemplate-sample spec: template: spec: remediationStrategy: Automatic [root@cert-rhosp-02 ~]# oc apply -f nhc_snr.ymal nodehealthcheck.remediation.medik8s.io/nhc-worker-test created selfnoderemediationtemplate.self-node-remediation.medik8s.io/selfnoderemediationtemplate-sample created [root@cert-rhosp-02 ~]# oc apply -f nhc_snr.ymal nodehealthcheck.remediation.medik8s.io/nhc-worker-test created selfnoderemediationtemplate.self-node-remediation.medik8s.io/selfnoderemediationtemplate-sample created [root@cert-rhosp-02 ~]# oc get nhc NAME AGE nhc-worker-test 19s [root@cert-rhosp-02 ~]# oc get snr No resources found in openshift-workload-availability namespace. [root@cert-rhosp-02 ~]# oc get snrt NAME AGE self-node-remediation-automatic-strategy-template 74m selfnoderemediationtemplate-sample 26s [root@cert-rhosp-02 ~]# oc get nhc -oyaml | yq .items[0].status conditions: - lastTransitionTime: "2026-02-11T12:57:28Z" message: No issues found, NodeHealthCheck is enabled. reason: NodeHealthCheckEnabled status: "False" type: Disabled healthyNodes: 3 lastUpdateTime: "2026-02-11T12:57:28Z" observedNodes: 3 phase: Enabled reason: NHC is enabled, no ongoing remediation [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 3h7m v1.31.14 master-0-1 Ready control-plane,master 3h7m v1.31.14 master-0-2 Ready control-plane,master 3h7m v1.31.14 worker-0-0 Ready worker 155m v1.31.14 worker-0-1 Ready worker 155m v1.31.14 worker-0-2 Ready worker 155m v1.31.14 [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 3h13m v1.31.14 master-0-1 Ready control-plane,master 3h12m v1.31.14 master-0-2 Ready control-plane,master 3h13m v1.31.14 worker-0-0 Ready worker 160m v1.31.14 worker-0-1 Ready worker 160m v1.31.14 worker-0-2 NotReady,SchedulingDisabled worker 160m v1.31.14 [root@cert-rhosp-02 ~]# oc get snr -o yaml apiVersion: v1 items: - apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediation metadata: annotations: remediation.medik8s.io/node-name: worker-0-2 remediation.medik8s.io/template-name: selfnoderemediationtemplate-sample creationTimestamp: "2026-02-11T13:07:37Z" finalizers: - self-node-remediation.medik8s.io/snr-finalizer generateName: worker-0-2- generation: 1 labels: app.kubernetes.io/part-of: node-healthcheck-controller name: worker-0-2-44nt7 namespace: openshift-workload-availability ownerReferences: - apiVersion: remediation.medik8s.io/v1alpha1 controller: false kind: NodeHealthCheck name: nhc-worker-test uid: 687640f7-dc4a-467d-a7ee-cf2304729c15 resourceVersion: "83637" uid: 1593c6e8-4226-4d16-b629-0ddb780f46e6 spec: remediationStrategy: Automatic status: conditions: - lastTransitionTime: "2026-02-11T13:07:37Z" message: "" reason: RemediationStarted status: "True" type: Processing - lastTransitionTime: "2026-02-11T13:07:37Z" message: "" reason: RemediationStarted status: Unknown type: Succeeded phase: Pre-Reboot-Completed timeAssumedRebooted: "2026-02-11T13:09:38Z" kind: List metadata: resourceVersion: "" selfLink: "" [root@cert-rhosp-02 ~]# oc get nhc -oyaml | yq .items[0].status conditions: - lastTransitionTime: "2026-02-11T12:57:28Z" message: No issues found, NodeHealthCheck is enabled. reason: NodeHealthCheckEnabled status: "False" type: Disabled healthyNodes: 2 lastUpdateTime: "2026-02-11T13:07:37Z" observedNodes: 3 phase: Remediating reason: NHC is remediating 1 nodes unhealthyNodes: - name: worker-0-2 remediations: - resource: apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediation name: worker-0-2-44nt7 namespace: openshift-workload-availability uid: 1593c6e8-4226-4d16-b629-0ddb780f46e6 started: "2026-02-11T13:07:37Z" templateName: selfnoderemediationtemplate-sample [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 3h15m v1.31.14 master-0-1 Ready control-plane,master 3h15m v1.31.14 master-0-2 Ready control-plane,master 3h15m v1.31.14 worker-0-0 Ready worker 162m v1.31.14 worker-0-1 Ready worker 162m v1.31.14 worker-0-2 Ready worker 163m v1.31.14 Case 2: [root@cert-rhosp-02 ~]# oc debug node/worker-0-2 -- chroot /host bash -c "date" Starting pod/worker-0-2-debug ... To use host binaries, run `chroot /host` Wed Feb 11 13:29:34 UTC 2026 Removing debug pod ... [root@cert-rhosp-02 ~]# oc debug node/worker-0-1 -- chroot /host bash -c "date" Starting pod/worker-0-1-debug ... To use host binaries, run `chroot /host` Wed Feb 11 13:29:46 UTC 2026 Removing debug pod ... [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 3h35m v1.31.14 master-0-1 Ready control-plane,master 3h35m v1.31.14 master-0-2 Ready control-plane,master 3h35m v1.31.14 worker-0-0 Ready worker 3h3m v1.31.14 worker-0-1 NotReady worker 3h3m v1.31.14 worker-0-2 NotReady worker 3h3m v1.31.14 [root@cert-rhosp-02 ~]# oc get nhc -oyaml | yq .items[0].status conditions: - lastTransitionTime: "2026-02-11T12:57:28Z" message: No issues found, NodeHealthCheck is enabled. reason: NodeHealthCheckEnabled status: "False" type: Disabled healthyNodes: 1 lastUpdateTime: "2026-02-11T13:31:32Z" observedNodes: 3 phase: Enabled reason: NHC is enabled, no ongoing remediation [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 4h6m v1.31.14 master-0-1 Ready control-plane,master 4h5m v1.31.14 master-0-2 Ready control-plane,master 4h6m v1.31.14 worker-0-0 Ready worker 3h33m v1.31.14 worker-0-1 Ready worker 3h33m v1.31.14 worker-0-2 NotReady worker 3h33m v1.31.14 [root@cert-rhosp-02 ~]# oc get nhc NAME AGE nhc-worker-test 34m [root@cert-rhosp-02 ~]# oc get snr No resources found in openshift-workload-availability namespace. [root@cert-rhosp-02 ~]# oc get snr NAME AGE worker-0-2-qwqfd 52s [root@cert-rhosp-02 ~]# oc get snr -oyaml apiVersion: v1 items: - apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediation metadata: annotations: remediation.medik8s.io/node-name: worker-0-2 remediation.medik8s.io/template-name: selfnoderemediationtemplate-sample creationTimestamp: "2026-02-11T14:02:24Z" finalizers: - self-node-remediation.medik8s.io/snr-finalizer generateName: worker-0-2- generation: 1 labels: app.kubernetes.io/part-of: node-healthcheck-controller name: worker-0-2-qwqfd namespace: openshift-workload-availability ownerReferences: - apiVersion: remediation.medik8s.io/v1alpha1 controller: false kind: NodeHealthCheck name: nhc-worker-test uid: 687640f7-dc4a-467d-a7ee-cf2304729c15 resourceVersion: "103799" uid: ce801684-ad69-497f-b070-9706ab85a6f5 spec: remediationStrategy: Automatic status: conditions: - lastTransitionTime: "2026-02-11T14:02:24Z" message: "" reason: RemediationStarted status: "True" type: Processing - lastTransitionTime: "2026-02-11T14:02:24Z" message: "" reason: RemediationStarted status: Unknown type: Succeeded phase: Pre-Reboot-Completed timeAssumedRebooted: "2026-02-11T14:04:25Z" kind: List metadata: resourceVersion: "" selfLink: "" [root@cert-rhosp-02 ~]# oc get nhc -oyaml | yq .items[0].status conditions: - lastTransitionTime: "2026-02-11T12:57:28Z" message: No issues found, NodeHealthCheck is enabled. reason: NodeHealthCheckEnabled status: "False" type: Disabled - lastTransitionTime: "2026-02-11T14:02:24Z" message: Storm cooldown completed reason: HealthyNodeThresholdChange status: "False" type: StormCooldownActive - lastTransitionTime: "2026-02-11T14:02:24Z" message: Storm mode is deactivated, remediation can occur normally reason: HealthyNodeThresholdChange status: "False" type: StormActive healthyNodes: 2 lastUpdateTime: "2026-02-11T14:02:24Z" observedNodes: 3 phase: Remediating reason: NHC is remediating 1 nodes unhealthyNodes: - name: worker-0-2 remediations: - resource: apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediation name: worker-0-2-qwqfd namespace: openshift-workload-availability uid: ce801684-ad69-497f-b070-9706ab85a6f5 started: "2026-02-11T14:02:24Z" templateName: selfnoderemediationtemplate-sample [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 4h19m v1.31.14 master-0-1 Ready control-plane,master 4h19m v1.31.14 master-0-2 Ready control-plane,master 4h19m v1.31.14 worker-0-0 Ready worker 3h47m v1.31.14 worker-0-1 Ready worker 3h47m v1.31.14 worker-0-2 Ready worker 3h47m v1.31.14 Key Logs: 2026-02-11T13:31:57.329042303Z INFO controllers.NodeHealthCheck Skipped remediation because the number of healthy nodes selected by the selector is 1 and should equal or exceed 2 {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:31:57.329060882Z INFO controllers.NodeHealthCheck Storm recovery mode activated {"nhc": "nhc-worker-test"} 2026-02-11T13:31:57.329143837Z DEBUG events [remediation] Skipped remediation because the number of healthy nodes selected by the selector is 1 and should equal or exceed 2 {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92198"}, "reason": "RemediationSkipped"} 2026-02-11T13:31:57.329163592Z DEBUG events [remediation] Storm recovery mode activated - delaying remediation until threshold is reached {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92198"}, "reason": "StormRecoveryStarted"} 2026-02-11T14:02:24.223902954Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-2","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-qwqfd","uid":"ce801684-ad69-497f-b070-9706ab85a6f5","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2026-02-11T14:02:24Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."},{"type":"StormCooldownActive","status":"False","lastTransitionTime":"2026-02-11T14:02:24Z","reason":"HealthyNodeThresholdChange","message":"Storm cooldown completed"},{"type":"StormActive","status":"False","lastTransitionTime":"2026-02-11T14:02:24Z","reason":"HealthyNodeThresholdChange","message":"Storm mode is deactivated, remediation can occur normally"}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2026-02-11T14:01:23Z"}, "patch": "{\"status\":{\"conditions\":[{\"lastTransitionTime\":\"2026-02-11T12:57:28Z\",\"message\":\"No issues found, NodeHealthCheck is enabled.\",\"reason\":\"NodeHealthCheckEnabled\",\"status\":\"False\",\"type\":\"Disabled\"},{\"lastTransitionTime\":\"2026-02-11T14:02:24Z\",\"message\":\"Storm cooldown completed\",\"reason\":\"HealthyNodeThresholdChange\",\"status\":\"False\",\"type\":\"StormCooldownActive\"},{\"lastTransitionTime\":\"2026-02-11T14:02:24Z\",\"message\":\"Storm mode is deactivated, remediation can occur normally\",\"reason\":\"HealthyNodeThresholdChange\",\"status\":\"False\",\"type\":\"StormActive\"}],\"phase\":\"Remediating\",\"reason\":\"NHC is remediating 1 nodes\",\"unhealthyNodes\":[{\"name\":\"worker-0-2\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-2-qwqfd\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"ce801684-ad69-497f-b070-9706ab85a6f5\"},\"started\":\"2026-02-11T14:02:24Z\",\"templateName\":\"selfnoderemediationtemplate-sample\"}]}]}}"} NHC Logs when healthy Nodes >= minHealthy without storm: [root@cert-rhosp-02 ~]# oc logs pod/node-healthcheck-controller-manager-7955d56b69-gkp5r -c manager 2026-02-11T11:39:02.279578389Z INFO setup Go Version: go1.25.3 (Red Hat 1.25.3-1.el9_7) X:strictfipsruntime 2026-02-11T11:39:02.279711002Z INFO setup Go OS/Arch: linux/amd64 2026-02-11T11:39:02.279714464Z INFO setup Operator Version: 333f49c 2026-02-11T11:39:02.279716388Z INFO setup Git Commit: 333f49ce59a566b59f89df5d44789e6d55871a25 2026-02-11T11:39:02.27971804Z INFO setup Build Date: 2026-02-02T11:55:43+00:00 2026-02-11T11:39:02.279719735Z INFO setup HTTP/2 for metrics and webhook server disabled 2026-02-11T11:39:02.301256227Z INFO setup supported control plane topology {"topology": "HighlyAvailable"} 2026-02-11T11:39:02.301295716Z INFO setup Cluster capabilities {"IsOnOpenshift": true, "HasMachineAPI": true} 2026-02-11T11:39:02.301523622Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "remediation.medik8s.io/v1alpha1, Kind=NodeHealthCheck", "path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2026-02-11T11:39:02.301609569Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2026-02-11T11:39:02.301648212Z INFO setup starting manager 2026-02-11T11:39:02.301694647Z INFO controller-runtime.metrics Starting metrics server 2026-02-11T11:39:02.30174187Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-02-11T11:39:02.30178058Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": "127.0.0.1:8080", "secure": false} 2026-02-11T11:39:02.301782376Z INFO controller-runtime.webhook Starting webhook server 2026-02-11T11:39:02.30199262Z INFO controller-runtime.certwatcher Updated current TLS certificate 2026-02-11T11:39:02.302061632Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-02-11T11:39:02.302113166Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} I0211 11:39:02.401989 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/e1f13584.medik8s.io... I0211 11:39:02.407478 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/e1f13584.medik8s.io 2026-02-11T11:39:02.407545067Z DEBUG events node-healthcheck-controller-manager-7955d56b69-gkp5r_effaefc9-48a1-4ee4-ab67-28d21eadc619 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"e1f13584.medik8s.io","uid":"778af5f5-a240-429b-b2b5-037e919991bc","apiVersion":"coordination.k8s.io/v1","resourceVersion":"56231"}, "reason": "LeaderElection"} 2026-02-11T11:39:02.40777295Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "channel source: 0xc00079c000"} 2026-02-11T11:39:02.407792305Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "channel source: 0xc00079c230"} 2026-02-11T11:39:02.407803514Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1alpha1.NodeHealthCheck"} 2026-02-11T11:39:02.407817651Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.MachineHealthCheck"} 2026-02-11T11:39:02.407826609Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1.Node"} 2026-02-11T11:39:02.40783675Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1.Node"} 2026-02-11T11:39:02.407844268Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.Machine"} I0211 11:39:02.408669 1 shared_informer.go:350] "Waiting for caches to sync" controller="feature gate accessor" 2026-02-11T11:39:02.507898214Z INFO MHCChecker found termination handler MHC, will ignore Nodes with Terminating condition 2026-02-11T11:39:02.507925106Z INFO MHCChecker MHC Checker status changed, notifying NHC controller 2026-02-11T11:39:02.507965371Z INFO adding all NHCs to reconcile queue for handling MHC event I0211 11:39:02.509767 1 shared_informer.go:357] "Caches are synced" controller="feature gate accessor" I0211 11:39:02.509796 1 simple_featuregate_reader.go:171] Starting feature-gate-detector 2026-02-11T11:39:02.509888363Z INFO FeatureGateAccessor FeatureGates initialized I0211 11:39:02.509957 1 recorder_logging.go:49] &Event{ObjectMeta:{dummy.18932e42b408103e.692b2fa5 dummy 0 0001-01-01 00:00:00 +0000 UTC map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Pod,Namespace:dummy,Name:dummy,UID:,APIVersion:v1,ResourceVersion:,FieldPath:,},Reason:FeatureGatesInitialized,Message:FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AWSEFSDriverVolumeMetrics", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BareMetalLoadBalancer", "BuildCSIVolumes", "ChunkSizeMiB", "CloudDualStackNodeIPs", "DisableKubeletCloudCredentialProviders", "GCPLabelsTags", "HardwareSpeed", "IngressControllerLBSubnetsAWS", "KMSv1", "ManagedBootImages", "ManagedBootImagesAWS", "MultiArchInstallAWS", "MultiArchInstallGCP", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "NewOLM", "NodeDisruptionPolicy", "OnClusterBuild", "PersistentIPsForVirtualization", "PrivateHostedZoneAWS", "SetEIPForNLBIngressController", "StreamingCollectionEncodingToJSON", "StreamingCollectionEncodingToProtobuf", "VSphereControlPlaneMachineSet", "VSphereDriverConfiguration", "VSphereMultiVCenters", "VSphereStaticIPs", "ValidatingAdmissionPolicy"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AdditionalRoutingCapabilities", "AutomatedEtcdBackup", "BootcNodeManagement", "CSIDriverSharedResource", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ConsolePluginContentSecurityPolicy", "DNSNameResolver", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNS", "GatewayAPI", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "InsightsRuntimeExtractor", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "MachineAPIProviderOpenStack", "MachineConfigNodes", "MaxUnavailableStatefulSet", "MetricsCollectionProfiles", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PinnedImages", "PlatformOperators", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SignatureStores", "SigstoreImageVerification", "TranslateStreamCloseWebsocketRequests", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiNetworks", "VolumeAttributesClass", "VolumeGroupSnapshot"}},Source:EventSource{Component:,Host:,},FirstTimestamp:2026-02-11 11:39:02.509883454 +0000 UTC m=+0.257959569,LastTimestamp:2026-02-11 11:39:02.509883454 +0000 UTC m=+0.257959569,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:,ReportingInstance:,} 2026-02-11T11:39:02.60866798Z INFO Starting Controller {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck"} 2026-02-11T11:39:02.608676133Z INFO Starting Controller {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck"} 2026-02-11T11:39:02.608708325Z INFO Starting workers {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "worker count": 1} 2026-02-11T11:39:02.608693876Z INFO Starting workers {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "worker count": 1} 2026-02-11T11:39:02.821396028Z INFO unknown field "spec.contentSecurityPolicy" 2026-02-11T11:39:02.82147563Z INFO console-plugin successfully created / updated console plugin resources 2026-02-11T12:57:28.372504255Z INFO nodehealthcheck-resource validate create {"name": "nhc-worker-test"} 2026-02-11T12:57:28.375201147Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T12:57:28.395525982Z INFO controllers.NodeHealthCheck enabling NHC, valid config, no conflicting MHC configured in the cluster {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T12:57:28.395629383Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2026-02-11T12:57:28.395648487Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation template CRs {"kind": "SelfNodeRemediationTemplate"} 2026-02-11T12:57:28.395656388Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2026-02-11T12:57:28.395659937Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation CRs {"kind": "SelfNodeRemediation"} 2026-02-11T12:57:28.395951018Z DEBUG events [remediation] No issues found, NodeHealthCheck is enabled. {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"80276"}, "reason": "Enabled"} 2026-02-11T12:57:28.403142031Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T12:57:28Z" level=info msg="invalidating lease" time="2026-02-11T12:57:28Z" level=info msg="getting lease" 2026-02-11T12:57:28.604475063Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T12:57:28Z" level=info msg="invalidating lease" time="2026-02-11T12:57:28Z" level=info msg="getting lease" 2026-02-11T12:57:28.60694865Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} time="2026-02-11T12:57:28Z" level=info msg="invalidating lease" time="2026-02-11T12:57:28Z" level=info msg="getting lease" 2026-02-11T12:57:28.609280316Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":3,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation"}, "patch": "{\"status\":{\"conditions\":[{\"lastTransitionTime\":\"2026-02-11T12:57:28Z\",\"message\":\"No issues found, NodeHealthCheck is enabled.\",\"reason\":\"NodeHealthCheckEnabled\",\"status\":\"False\",\"type\":\"Disabled\"}],\"healthyNodes\":3,\"observedNodes\":3,\"phase\":\"Enabled\",\"reason\":\"NHC is enabled, no ongoing remediation\"}}"} 2026-02-11T12:57:28.813914571Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "0s"} 2026-02-11T13:07:06.284175847Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-worker-test"} 2026-02-11T13:07:06.28425011Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-worker-test"} 2026-02-11T13:07:06.284307425Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:07:06.293438799Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown", "duration left": "29.706564505s"} 2026-02-11T13:07:06.306879806Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:07:06Z" level=info msg="invalidating lease" time="2026-02-11T13:07:06Z" level=info msg="getting lease" 2026-02-11T13:07:06.404870754Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:07:06Z" level=info msg="invalidating lease" time="2026-02-11T13:07:06Z" level=info msg="getting lease" 2026-02-11T13:07:06.505367544Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":2,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2026-02-11T12:57:28Z"}, "patch": "{\"status\":{\"healthyNodes\":2}}"} 2026-02-11T13:07:06.717280299Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "30.706564505s"} 2026-02-11T13:07:37.424973536Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:07:37.429112173Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T13:07:37.429253192Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-2", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"83178"}, "reason": "DetectedUnhealthy"} 2026-02-11T13:07:37.43270169Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:07:37Z" level=info msg="invalidating lease" time="2026-02-11T13:07:37Z" level=info msg="getting lease" 2026-02-11T13:07:37.434057689Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:07:37Z" level=info msg="invalidating lease" time="2026-02-11T13:07:37Z" level=info msg="getting lease" 2026-02-11T13:07:37.435304604Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T13:07:37.438182148Z INFO controllers.NodeHealthCheck.resource manager Attempting to obtain Node Lease {"Node name": "worker-0-2"} time="2026-02-11T13:07:37Z" level=info msg="request lease" time="2026-02-11T13:07:37Z" level=info msg="getting lease" time="2026-02-11T13:07:37Z" level=info msg="create lease" 2026-02-11T13:07:37.440450252Z INFO controllers.NodeHealthCheck.resource manager Creating a remediation CR {"CR name": "", "CR kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} 2026-02-11T13:07:37.448100892Z DEBUG events [remediation] Created remediation object for node worker-0-2 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"83178"}, "reason": "RemediationCreated"} 2026-02-11T13:07:37.450154518Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-2","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-44nt7","uid":"1593c6e8-4226-4d16-b629-0ddb780f46e6","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2026-02-11T13:07:37Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2026-02-11T13:07:06Z"}, "patch": "{\"status\":{\"phase\":\"Remediating\",\"reason\":\"NHC is remediating 1 nodes\",\"unhealthyNodes\":[{\"name\":\"worker-0-2\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-2-44nt7\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"1593c6e8-4226-4d16-b629-0ddb780f46e6\"},\"started\":\"2026-02-11T13:07:37Z\",\"templateName\":\"selfnoderemediationtemplate-sample\"}]}]}}"} 2026-02-11T13:07:37.655444126Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2026-02-11T13:07:37.6586838Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:07:37.658732542Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:07:37.658746467Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:07:37.658750824Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:07:37.658781774Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:07:37.662952575Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:07:37.663005409Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:07:37.663018549Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:07:37.663024865Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:07:37.663082224Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T13:07:37.667152498Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:07:37Z" level=info msg="invalidating lease" time="2026-02-11T13:07:37Z" level=info msg="getting lease" 2026-02-11T13:07:37.668962869Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:07:37Z" level=info msg="invalidating lease" time="2026-02-11T13:07:37Z" level=info msg="getting lease" 2026-02-11T13:07:37.675369862Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T13:07:37.688133133Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:07:37.688180432Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:07:37.688193093Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:07:37.688200103Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:07:37.774197465Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists{"CR name": "worker-0-2-44nt7", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2026-02-11T13:07:37Z" level=info msg="getting lease" 2026-02-11T13:07:37.774276881Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-worker-test", "lease name": "node-worker-0-2", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2026-02-11T13:07:37.873740995Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2026-02-11T13:07:37.873836663Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:07:37.878170694Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T13:07:37.881925934Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-11T13:07:37Z"} 2026-02-11T13:07:37.881955644Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:07:37Z" level=info msg="invalidating lease" time="2026-02-11T13:07:37Z" level=info msg="getting lease" 2026-02-11T13:07:37.883626668Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:07:37Z" level=info msg="invalidating lease" time="2026-02-11T13:07:37Z" level=info msg="getting lease" 2026-02-11T13:07:37.885224053Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T13:07:37.888087637Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists{"CR name": "worker-0-2-44nt7", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2026-02-11T13:07:37Z" level=info msg="getting lease" 2026-02-11T13:07:37.888131023Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-worker-test", "lease name": "node-worker-0-2", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2026-02-11T13:07:37.889767971Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2026-02-11T13:07:38.695731633Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:07:38.695774067Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:07:38.695793091Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:07:38.695799584Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:07:38.695837387Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:07:38.700179738Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T13:07:38.704440804Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-11T13:07:37Z"} 2026-02-11T13:07:38.704467442Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:07:38Z" level=info msg="invalidating lease" time="2026-02-11T13:07:38Z" level=info msg="getting lease" 2026-02-11T13:07:38.705938795Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:07:38Z" level=info msg="invalidating lease" time="2026-02-11T13:07:38Z" level=info msg="getting lease" 2026-02-11T13:07:38.707722286Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T13:07:38.710910017Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists{"CR name": "worker-0-2-44nt7", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2026-02-11T13:07:38Z" level=info msg="getting lease" 2026-02-11T13:07:38.710955412Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-worker-test", "lease name": "node-worker-0-2", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2026-02-11T13:07:38.712784025Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2026-02-11T13:09:28.782499458Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-worker-test"} 2026-02-11T13:09:28.782570388Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-worker-test"} 2026-02-11T13:09:28.782637837Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:09:28.789899461Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "False", "duration left": "29.210104082s"} 2026-02-11T13:09:28.797825708Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-11T13:07:37Z"} 2026-02-11T13:09:28.797859085Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:09:28Z" level=info msg="invalidating lease" time="2026-02-11T13:09:28Z" level=info msg="getting lease" 2026-02-11T13:09:28.897471329Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:09:28Z" level=info msg="invalidating lease" time="2026-02-11T13:09:28Z" level=info msg="getting lease" 2026-02-11T13:09:28.899519425Z INFO controllers.NodeHealthCheck Ignoring node, because it was unhealthy, and is likely to be unhealthy again. {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T13:09:28.89972298Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "30.210104082s"} 2026-02-11T13:09:30.605173662Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-worker-test"} 2026-02-11T13:09:30.605244744Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-worker-test"} 2026-02-11T13:09:30.605293298Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:09:30.621198462Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-11T13:07:37Z"} 2026-02-11T13:09:30.621254005Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:09:30Z" level=info msg="invalidating lease" time="2026-02-11T13:09:30Z" level=info msg="getting lease" 2026-02-11T13:09:30.720762573Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:09:30Z" level=info msg="invalidating lease" time="2026-02-11T13:09:30Z" level=info msg="getting lease" 2026-02-11T13:09:30.725152275Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T13:09:30.726808728Z INFO controllers.NodeHealthCheck.resource manager setting a delay for node getting healthy {"node name": "worker-0-2", "delay in seconds": 5} 2026-02-11T13:09:30.736240261Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:09:30.736281177Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:09:30.736290862Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:09:30.736294728Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:09:30.736365058Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-2","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-44nt7","uid":"1593c6e8-4226-4d16-b629-0ddb780f46e6","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2026-02-11T13:07:37Z","templateName":"selfnoderemediationtemplate-sample"}],"conditionsHealthyTimestamp":"2026-02-11T13:09:30Z","healthyDelayed":true}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2026-02-11T13:07:37Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"conditionsHealthyTimestamp\":\"2026-02-11T13:09:30Z\",\"healthyDelayed\":true,\"name\":\"worker-0-2\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-2-44nt7\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"1593c6e8-4226-4d16-b629-0ddb780f46e6\"},\"started\":\"2026-02-11T13:07:37Z\",\"templateName\":\"selfnoderemediationtemplate-sample\"}]}]}}"} 2026-02-11T13:09:30.942551215Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "6s"} 2026-02-11T13:09:30.94264079Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:09:30.950657921Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-11T13:07:37Z"} 2026-02-11T13:09:30.950679918Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:09:30Z" level=info msg="invalidating lease" time="2026-02-11T13:09:30Z" level=info msg="getting lease" 2026-02-11T13:09:30.952374013Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T13:09:30.954030413Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 4.045971844} 2026-02-11T13:09:30.95405999Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:09:30Z" level=info msg="invalidating lease" time="2026-02-11T13:09:30Z" level=info msg="getting lease" 2026-02-11T13:09:30.955940351Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "5.045971844s"} 2026-02-11T13:09:36.002459263Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:09:36.009572494Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-11T13:07:37Z"} 2026-02-11T13:09:36.00964344Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:09:36Z" level=info msg="invalidating lease" time="2026-02-11T13:09:36Z" level=info msg="getting lease" 2026-02-11T13:09:36.011282639Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:09:36Z" level=info msg="invalidating lease" time="2026-02-11T13:09:36Z" level=info msg="getting lease" 2026-02-11T13:09:36.012779759Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T13:09:36.014376902Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-2"} 2026-02-11T13:09:36.019688191Z INFO controllers.NodeHealthCheck.resource manager deleted remediation CR {"name": "worker-0-2-44nt7"} 2026-02-11T13:09:36.019844327Z DEBUG events [remediation] Deleted remediation CR of kind SelfNodeRemediation with name worker-0-2-44nt7 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"84353"}, "reason": "RemediationRemoved"} 2026-02-11T13:09:36.019915001Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "11s"} 2026-02-11T13:09:36.020740684Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:09:36.020762396Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:09:36.02077264Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:09:36.020781606Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:09:36.020815179Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:09:36.110450407Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:09:36Z" level=info msg="invalidating lease" time="2026-02-11T13:09:36Z" level=info msg="getting lease" 2026-02-11T13:09:36.112216504Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T13:09:36.113964969Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-2"} 2026-02-11T13:09:36.115847583Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:09:36Z" level=info msg="invalidating lease" time="2026-02-11T13:09:36Z" level=info msg="getting lease" 2026-02-11T13:09:36.117954052Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "17s"} 2026-02-11T13:09:39.011615643Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:09:39.01170088Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:09:39.011733116Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:09:39.011756698Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:09:39.011797328Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:09:39.019176408Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:09:39Z" level=info msg="invalidating lease" time="2026-02-11T13:09:39Z" level=info msg="getting lease" 2026-02-11T13:09:39.02523782Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:09:39Z" level=info msg="invalidating lease" time="2026-02-11T13:09:39Z" level=info msg="getting lease" 2026-02-11T13:09:39.124585136Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T13:09:39.126839601Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-2"} 2026-02-11T13:09:39.128837018Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "17s"} 2026-02-11T13:09:47.020205216Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:09:47.031116793Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:09:47Z" level=info msg="invalidating lease" time="2026-02-11T13:09:47Z" level=info msg="getting lease" 2026-02-11T13:09:47.03264353Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T13:09:47.034611344Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-2"} 2026-02-11T13:09:47.036475941Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:09:47Z" level=info msg="invalidating lease" time="2026-02-11T13:09:47Z" level=info msg="getting lease" 2026-02-11T13:09:47.038257813Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "17s"} 2026-02-11T13:09:49.166563664Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:09:49.166616375Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:09:49.166635733Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:09:49.166642883Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:09:49.166685891Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:09:49.178755047Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:09:49Z" level=info msg="invalidating lease" time="2026-02-11T13:09:49Z" level=info msg="getting lease" 2026-02-11T13:09:49.278378833Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:09:49Z" level=info msg="invalidating lease" time="2026-02-11T13:09:49Z" level=info msg="getting lease" 2026-02-11T13:09:49.379253635Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T13:09:49.381236363Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-2"} 2026-02-11T13:09:49.383299921Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "17s"} 2026-02-11T13:09:50.201430769Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T13:09:50.201470617Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-44nt7", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T13:09:50.201529923Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:09:50.211215702Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:09:50Z" level=info msg="invalidating lease" time="2026-02-11T13:09:50Z" level=info msg="getting lease" 2026-02-11T13:09:50.213136453Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:09:50Z" level=info msg="invalidating lease" time="2026-02-11T13:09:50Z" level=info msg="getting lease" 2026-02-11T13:09:50.312242101Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} time="2026-02-11T13:09:50Z" level=info msg="invalidating lease" time="2026-02-11T13:09:50Z" level=info msg="getting lease" 2026-02-11T13:09:50.415904755Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":3,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2026-02-11T13:09:30Z"}, "patch": "{\"status\":{\"healthyNodes\":3,\"phase\":\"Enabled\",\"reason\":\"NHC is enabled, no ongoing remediation\",\"unhealthyNodes\":null}}"} 2026-02-11T13:09:50.621067099Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "0s"} 2026-02-11T13:10:04.0387538Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:10:04.047164421Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:10:04Z" level=info msg="invalidating lease" time="2026-02-11T13:10:04Z" level=info msg="getting lease" 2026-02-11T13:10:04.048776096Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} time="2026-02-11T13:10:04Z" level=info msg="invalidating lease" time="2026-02-11T13:10:04Z" level=info msg="getting lease" 2026-02-11T13:10:04.050234044Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:10:04Z" level=info msg="invalidating lease" time="2026-02-11T13:10:04Z" level=info msg="getting lease" 2026-02-11T13:10:04.051739012Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "0s"} [root@cert-rhosp-02 ~]# NHC Logs when healthy Nodes >= minHealthy with storm in starting healthy Nodes <= minHealthy so storm activated and when nodes started getting in Ready State stormCoolDown activated: 2026-02-11T11:39:03.185796187Z INFO setup Go Version: go1.25.3 (Red Hat 1.25.3-1.el9_7) X:strictfipsruntime 2026-02-11T11:39:03.185900692Z INFO setup Go OS/Arch: linux/amd64 2026-02-11T11:39:03.185903484Z INFO setup Operator Version: 333f49c 2026-02-11T11:39:03.185905296Z INFO setup Git Commit: 333f49ce59a566b59f89df5d44789e6d55871a25 2026-02-11T11:39:03.185913808Z INFO setup Build Date: 2026-02-02T11:55:43+00:00 2026-02-11T11:39:03.185915653Z INFO setup HTTP/2 for metrics and webhook server disabled 2026-02-11T11:39:03.205080933Z INFO setup supported control plane topology {"topology": "HighlyAvailable"} 2026-02-11T11:39:03.205164379Z INFO setup Cluster capabilities {"IsOnOpenshift": true, "HasMachineAPI": true} 2026-02-11T11:39:03.205383117Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "remediation.medik8s.io/v1alpha1, Kind=NodeHealthCheck", "path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2026-02-11T11:39:03.205487139Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2026-02-11T11:39:03.205536484Z INFO setup starting manager 2026-02-11T11:39:03.205588109Z INFO controller-runtime.metrics Starting metrics server 2026-02-11T11:39:03.205685876Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": "127.0.0.1:8080", "secure": false} 2026-02-11T11:39:03.205728169Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-02-11T11:39:03.205752427Z INFO controller-runtime.webhook Starting webhook server 2026-02-11T11:39:03.206083973Z INFO controller-runtime.certwatcher Updated current TLS certificate 2026-02-11T11:39:03.206190617Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-02-11T11:39:03.20630256Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} I0211 11:39:03.306331 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/e1f13584.medik8s.io... I0211 13:28:02.388595 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/e1f13584.medik8s.io 2026-02-11T13:28:02.388640241Z DEBUG events node-healthcheck-controller-manager-7955d56b69-v64wh_927fcfe8-7744-44fa-a71c-2dd143a6a207 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"e1f13584.medik8s.io","uid":"778af5f5-a240-429b-b2b5-037e919991bc","apiVersion":"coordination.k8s.io/v1","resourceVersion":"90915"}, "reason": "LeaderElection"} 2026-02-11T13:28:02.388840406Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "channel source: 0xc000784540"} 2026-02-11T13:28:02.388859672Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "channel source: 0xc0007847e0"} 2026-02-11T13:28:02.388875118Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1.Node"} 2026-02-11T13:28:02.388880903Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.Machine"} 2026-02-11T13:28:02.388882361Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.MachineHealthCheck"} 2026-02-11T13:28:02.388885561Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1alpha1.NodeHealthCheck"} 2026-02-11T13:28:02.388870393Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1.Node"} I0211 13:28:02.390706 1 shared_informer.go:350] "Waiting for caches to sync" controller="feature gate accessor" 2026-02-11T13:28:02.489957729Z INFO MHCChecker found termination handler MHC, will ignore Nodes with Terminating condition 2026-02-11T13:28:02.48997442Z INFO Starting Controller {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck"} 2026-02-11T13:28:02.48999107Z INFO Starting workers {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "worker count": 1} 2026-02-11T13:28:02.489986706Z INFO MHCChecker MHC Checker status changed, notifying NHC controller 2026-02-11T13:28:02.490068994Z INFO Starting Controller {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck"} 2026-02-11T13:28:02.490088032Z INFO Starting workers {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "worker count": 1} 2026-02-11T13:28:02.490131712Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:28:02.490193756Z INFO adding all NHCs to reconcile queue for handling MHC event I0211 13:28:02.491195 1 shared_informer.go:357] "Caches are synced" controller="feature gate accessor" I0211 13:28:02.491233 1 simple_featuregate_reader.go:171] Starting feature-gate-detector 2026-02-11T13:28:02.491357735Z INFO FeatureGateAccessor FeatureGates initialized I0211 13:28:02.491430 1 recorder_logging.go:49] &Event{ObjectMeta:{dummy.18933435694d6400.692b2fa5 dummy 0 0001-01-01 00:00:00 +0000 UTC map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Pod,Namespace:dummy,Name:dummy,UID:,APIVersion:v1,ResourceVersion:,FieldPath:,},Reason:FeatureGatesInitialized,Message:FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AWSEFSDriverVolumeMetrics", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BareMetalLoadBalancer", "BuildCSIVolumes", "ChunkSizeMiB", "CloudDualStackNodeIPs", "DisableKubeletCloudCredentialProviders", "GCPLabelsTags", "HardwareSpeed", "IngressControllerLBSubnetsAWS", "KMSv1", "ManagedBootImages", "ManagedBootImagesAWS", "MultiArchInstallAWS", "MultiArchInstallGCP", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "NewOLM", "NodeDisruptionPolicy", "OnClusterBuild", "PersistentIPsForVirtualization", "PrivateHostedZoneAWS", "SetEIPForNLBIngressController", "StreamingCollectionEncodingToJSON", "StreamingCollectionEncodingToProtobuf", "VSphereControlPlaneMachineSet", "VSphereDriverConfiguration", "VSphereMultiVCenters", "VSphereStaticIPs", "ValidatingAdmissionPolicy"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AdditionalRoutingCapabilities", "AutomatedEtcdBackup", "BootcNodeManagement", "CSIDriverSharedResource", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ConsolePluginContentSecurityPolicy", "DNSNameResolver", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNS", "GatewayAPI", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "InsightsRuntimeExtractor", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "MachineAPIProviderOpenStack", "MachineConfigNodes", "MaxUnavailableStatefulSet", "MetricsCollectionProfiles", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PinnedImages", "PlatformOperators", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SignatureStores", "SigstoreImageVerification", "TranslateStreamCloseWebsocketRequests", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiNetworks", "VolumeAttributesClass", "VolumeGroupSnapshot"}},Source:EventSource{Component:,Host:,},FirstTimestamp:2026-02-11 13:28:02.491327488 +0000 UTC m=+6539.326633544,LastTimestamp:2026-02-11 13:28:02.491327488 +0000 UTC m=+6539.326633544,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:,ReportingInstance:,} 2026-02-11T13:28:02.500088282Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2026-02-11T13:28:02.500125976Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation template CRs {"kind": "SelfNodeRemediationTemplate"} 2026-02-11T13:28:02.500133869Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2026-02-11T13:28:02.500140086Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation CRs {"kind": "SelfNodeRemediation"} 2026-02-11T13:28:02.504778702Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:28:02Z" level=info msg="invalidating lease" time="2026-02-11T13:28:02Z" level=info msg="getting lease" 2026-02-11T13:28:02.606713609Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} 2026-02-11T13:28:02.607468712Z ERROR controllers.NodeHealthCheck.resource manager failed to get remediation CRs for healthy node {"node": "worker-0-1", "error": "failed to get all remediation objects with kind self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation and apiVersion self-node-remediation.medik8s.io/v1alpha1: selfnoderemediations.self-node-remediation.medik8s.io is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot list resource \"selfnoderemediations\" in API group \"self-node-remediation.medik8s.io\" at the cluster scope", "errorVerbose": "selfnoderemediations.self-node-remediation.medik8s.io is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot list resource \"selfnoderemediations\" in API group \"self-node-remediation.medik8s.io\" at the cluster scope\nfailed to get all remediation objects with kind self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation and apiVersion self-node-remediation.medik8s.io/v1alpha1\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ListRemediationCRs\n\t/app/node-healthcheck-operator/controllers/resources/manager.go:277\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).HandleHealthyNode\n\t/app/node-healthcheck-operator/controllers/resources/manager.go:322\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:279\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} github.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).HandleHealthyNode /app/node-healthcheck-operator/controllers/resources/manager.go:326 github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:279 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-11T13:28:02.607563149Z ERROR controllers.NodeHealthCheck failed to handle healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1", "error": "failed to get all remediation objects with kind self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation and apiVersion self-node-remediation.medik8s.io/v1alpha1: selfnoderemediations.self-node-remediation.medik8s.io is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot list resource \"selfnoderemediations\" in API group \"self-node-remediation.medik8s.io\" at the cluster scope", "errorVerbose": "selfnoderemediations.self-node-remediation.medik8s.io is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot list resource \"selfnoderemediations\" in API group \"self-node-remediation.medik8s.io\" at the cluster scope\nfailed to get all remediation objects with kind self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation and apiVersion self-node-remediation.medik8s.io/v1alpha1\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ListRemediationCRs\n\t/app/node-healthcheck-operator/controllers/resources/manager.go:277\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).HandleHealthyNode\n\t/app/node-healthcheck-operator/controllers/resources/manager.go:322\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:279\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:281 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-11T13:28:02.608038747Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":0,"healthyNodes":0,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2026-02-11T13:09:50Z"}, "patch": "{\"status\":{\"healthyNodes\":0,\"observedNodes\":0}}"} 2026-02-11T13:28:02.804413688Z INFO unknown field "spec.contentSecurityPolicy" 2026-02-11T13:28:02.80456184Z INFO console-plugin successfully created / updated console plugin resources 2026-02-11T13:28:02.813592748Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": "failed to get all remediation objects with kind self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation and apiVersion self-node-remediation.medik8s.io/v1alpha1: selfnoderemediations.self-node-remediation.medik8s.io is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot list resource \"selfnoderemediations\" in API group \"self-node-remediation.medik8s.io\" at the cluster scope", "errorCauses": [{"error": "failed to get all remediation objects with kind self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation and apiVersion self-node-remediation.medik8s.io/v1alpha1: selfnoderemediations.self-node-remediation.medik8s.io is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot list resource \"selfnoderemediations\" in API group \"self-node-remediation.medik8s.io\" at the cluster scope", "errorVerbose": "selfnoderemediations.self-node-remediation.medik8s.io is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot list resource \"selfnoderemediations\" in API group \"self-node-remediation.medik8s.io\" at the cluster scope\nfailed to get all remediation objects with kind self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation and apiVersion self-node-remediation.medik8s.io/v1alpha1\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ListRemediationCRs\n\t/app/node-healthcheck-operator/controllers/resources/manager.go:277\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).HandleHealthyNode\n\t/app/node-healthcheck-operator/controllers/resources/manager.go:322\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:279\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"}], "requeue": false, "requeuAfter": "0s"} 2026-02-11T13:28:02.813706806Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-worker-test"}, "namespace": "", "name": "nhc-worker-test", "reconcileID": "dcc79342-07e6-4565-b283-6fa20fb6b4e6", "error": "failed to get all remediation objects with kind self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation and apiVersion self-node-remediation.medik8s.io/v1alpha1: selfnoderemediations.self-node-remediation.medik8s.io is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot list resource \"selfnoderemediations\" in API group \"self-node-remediation.medik8s.io\" at the cluster scope", "errorCauses": [{"error": "failed to get all remediation objects with kind self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation and apiVersion self-node-remediation.medik8s.io/v1alpha1: selfnoderemediations.self-node-remediation.medik8s.io is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot list resource \"selfnoderemediations\" in API group \"self-node-remediation.medik8s.io\" at the cluster scope", "errorVerbose": "selfnoderemediations.self-node-remediation.medik8s.io is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot list resource \"selfnoderemediations\" in API group \"self-node-remediation.medik8s.io\" at the cluster scope\nfailed to get all remediation objects with kind self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation and apiVersion self-node-remediation.medik8s.io/v1alpha1\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ListRemediationCRs\n\t/app/node-healthcheck-operator/controllers/resources/manager.go:277\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).HandleHealthyNode\n\t/app/node-healthcheck-operator/controllers/resources/manager.go:322\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:279\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:340\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:353 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:300 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.1 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:202 2026-02-11T13:28:02.813815107Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:28:02.821343854Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:28:02Z" level=info msg="invalidating lease" time="2026-02-11T13:28:02Z" level=info msg="getting lease" 2026-02-11T13:28:02.822706925Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:28:02Z" level=info msg="invalidating lease" time="2026-02-11T13:28:02Z" level=info msg="getting lease" 2026-02-11T13:28:02.824295351Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} time="2026-02-11T13:28:02Z" level=info msg="invalidating lease" time="2026-02-11T13:28:02Z" level=info msg="getting lease" 2026-02-11T13:28:02.825713657Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":3,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2026-02-11T13:28:02Z"}, "patch": "{\"status\":{\"healthyNodes\":3,\"observedNodes\":3}}"} 2026-02-11T13:28:03.030493686Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "0s"} 2026-02-11T13:28:03.030638993Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:28:03.03893748Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} time="2026-02-11T13:28:03Z" level=info msg="invalidating lease" time="2026-02-11T13:28:03Z" level=info msg="getting lease" 2026-02-11T13:28:03.040397308Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:28:03Z" level=info msg="invalidating lease" time="2026-02-11T13:28:03Z" level=info msg="getting lease" 2026-02-11T13:28:03.042080798Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:28:03Z" level=info msg="invalidating lease" time="2026-02-11T13:28:03Z" level=info msg="getting lease" 2026-02-11T13:28:03.043617828Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "0s"} 2026-02-11T13:31:26.88389975Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-worker-test"} 2026-02-11T13:31:26.883975398Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-worker-test"} 2026-02-11T13:31:26.884038265Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:31:26.891232619Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown", "duration left": "29.108769819s"} 2026-02-11T13:31:26.909775009Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:31:27Z" level=info msg="invalidating lease" time="2026-02-11T13:31:27Z" level=info msg="getting lease" 2026-02-11T13:31:27.010178598Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T13:31:27Z" level=info msg="invalidating lease" time="2026-02-11T13:31:27Z" level=info msg="getting lease" 2026-02-11T13:31:27.111233528Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":2,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2026-02-11T13:28:02Z"}, "patch": "{\"status\":{\"healthyNodes\":2}}"} 2026-02-11T13:31:27.316729531Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "30.108769819s"} 2026-02-11T13:31:32.099729828Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-1", "NHC": "nhc-worker-test"} 2026-02-11T13:31:32.099796914Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-1", "NHC": "nhc-worker-test"} 2026-02-11T13:31:32.099849092Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:31:32.1089824Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown", "duration left": "29.89102047s"} 2026-02-11T13:31:32.109033392Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown", "duration left": "23.89096733s"} 2026-02-11T13:31:32.119723004Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:31:32Z" level=info msg="invalidating lease" time="2026-02-11T13:31:32Z" level=info msg="getting lease" 2026-02-11T13:31:32.220841067Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":1,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2026-02-11T13:31:27Z"}, "patch": "{\"status\":{\"healthyNodes\":1}}"} 2026-02-11T13:31:32.428093408Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "24.89096733s"} 2026-02-11T13:31:57.320103937Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:31:57.323866893Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown", "duration left": "4.676134383s"} 2026-02-11T13:31:57.323896981Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T13:31:57.324035432Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-2", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92198"}, "reason": "DetectedUnhealthy"} 2026-02-11T13:31:57.327582374Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:31:57Z" level=info msg="invalidating lease" time="2026-02-11T13:31:57Z" level=info msg="getting lease" 2026-02-11T13:31:57.329042303Z INFO controllers.NodeHealthCheck Skipped remediation because the number of healthy nodes selected by the selector is 1 and should equal or exceed 2 {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:31:57.329060882Z INFO controllers.NodeHealthCheck Storm recovery mode activated {"nhc": "nhc-worker-test"} 2026-02-11T13:31:57.329143837Z DEBUG events [remediation] Skipped remediation because the number of healthy nodes selected by the selector is 1 and should equal or exceed 2 {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92198"}, "reason": "RemediationSkipped"} 2026-02-11T13:31:57.329163592Z DEBUG events [remediation] Storm recovery mode activated - delaying remediation until threshold is reached {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92198"}, "reason": "StormRecoveryStarted"} 2026-02-11T13:31:57.329304639Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":1,"unhealthyNodes":[{"name":"worker-0-2"}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."},{"type":"StormCooldownActive","status":"False","lastTransitionTime":"2026-02-11T13:31:57Z","reason":"HealthyNodeThresholdChange","message":"Cooldown cleared - storm threshold triggered"},{"type":"StormActive","status":"True","lastTransitionTime":"2026-02-11T13:31:57Z","reason":"HealthyNodeThresholdChange","message":"Storm mode is activated - preventing any new remediation until the storm is over and cooldown duration expired"}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2026-02-11T13:31:32Z"}, "patch": "{\"status\":{\"conditions\":[{\"lastTransitionTime\":\"2026-02-11T12:57:28Z\",\"message\":\"No issues found, NodeHealthCheck is enabled.\",\"reason\":\"NodeHealthCheckEnabled\",\"status\":\"False\",\"type\":\"Disabled\"},{\"lastTransitionTime\":\"2026-02-11T13:31:57Z\",\"message\":\"Cooldown cleared - storm threshold triggered\",\"reason\":\"HealthyNodeThresholdChange\",\"status\":\"False\",\"type\":\"StormCooldownActive\"},{\"lastTransitionTime\":\"2026-02-11T13:31:57Z\",\"message\":\"Storm mode is activated - preventing any new remediation until the storm is over and cooldown duration expired\",\"reason\":\"HealthyNodeThresholdChange\",\"status\":\"True\",\"type\":\"StormActive\"}],\"unhealthyNodes\":[{\"name\":\"worker-0-2\"}]}}"} 2026-02-11T13:31:57.534824369Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "5.676134383s"} 2026-02-11T13:32:03.211303424Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:32:03.214987144Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T13:32:03.215039068Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T13:32:03.215115659Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-1", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92630"}, "reason": "DetectedUnhealthy"} 2026-02-11T13:32:03.215164434Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-2", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92630"}, "reason": "DetectedUnhealthy"} 2026-02-11T13:32:03.218939544Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T13:32:03Z" level=info msg="invalidating lease" time="2026-02-11T13:32:03Z" level=info msg="getting lease" 2026-02-11T13:32:03.220661992Z INFO controllers.NodeHealthCheck Skipped remediation because the number of healthy nodes selected by the selector is 1 and should equal or exceed 2 {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T13:32:03.22078334Z DEBUG events [remediation] Skipped remediation because the number of healthy nodes selected by the selector is 1 and should equal or exceed 2 {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92630"}, "reason": "RemediationSkipped"} 2026-02-11T13:32:03.220895971Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":1,"unhealthyNodes":[{"name":"worker-0-2"},{"name":"worker-0-1"}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."},{"type":"StormCooldownActive","status":"False","lastTransitionTime":"2026-02-11T13:31:57Z","reason":"HealthyNodeThresholdChange","message":"Cooldown cleared - storm threshold triggered"},{"type":"StormActive","status":"True","lastTransitionTime":"2026-02-11T13:31:57Z","reason":"HealthyNodeThresholdChange","message":"Storm mode is activated - preventing any new remediation until the storm is over and cooldown duration expired"}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2026-02-11T13:31:57Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"name\":\"worker-0-2\"},{\"name\":\"worker-0-1\"}]}}"} 2026-02-11T13:32:03.428686434Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "0s"} 2026-02-11T14:01:13.204612068Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-1", "NHC": "nhc-worker-test"} 2026-02-11T14:01:13.204722488Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-1", "NHC": "nhc-worker-test"} 2026-02-11T14:01:13.204800796Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:01:13.210817815Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "False", "duration left": "29.789185468s"} 2026-02-11T14:01:13.210871839Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T14:01:13.210999158Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-2", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92666"}, "reason": "DetectedUnhealthy"} 2026-02-11T14:01:13.216519042Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T14:01:13Z" level=info msg="invalidating lease" time="2026-02-11T14:01:13Z" level=info msg="getting lease" 2026-02-11T14:01:13.317175023Z INFO controllers.NodeHealthCheck Ignoring node, because it was unhealthy, and is likely to be unhealthy again. {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} 2026-02-11T14:01:13.317192423Z INFO controllers.NodeHealthCheck Skipped remediation because the number of healthy nodes selected by the selector is 1 and should equal or exceed 2 {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:01:13.317321056Z DEBUG events [remediation] Skipped remediation because the number of healthy nodes selected by the selector is 1 and should equal or exceed 2 {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92666"}, "reason": "RemediationSkipped"} 2026-02-11T14:01:13.317460846Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "30.789185468s"} 2026-02-11T14:01:23.729085089Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-1", "NHC": "nhc-worker-test"} 2026-02-11T14:01:23.72914524Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-1", "NHC": "nhc-worker-test"} 2026-02-11T14:01:23.72919807Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:01:23.734270805Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T14:01:23.734415543Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-2", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92666"}, "reason": "DetectedUnhealthy"} 2026-02-11T14:01:23.744103321Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T14:01:23Z" level=info msg="invalidating lease" time="2026-02-11T14:01:23Z" level=info msg="getting lease" 2026-02-11T14:01:23.845113176Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:01:23Z" level=info msg="invalidating lease" time="2026-02-11T14:01:23Z" level=info msg="getting lease" 2026-02-11T14:01:23.84689909Z INFO controllers.NodeHealthCheck The cluster regained health after the storm, a cooldown period now begins as a safety measure before normal operations resume. {"nhc": "nhc-worker-test"} 2026-02-11T14:01:23.846930392Z INFO controllers.NodeHealthCheck Storm recovery active: skipping creation of new remediations 2026-02-11T14:01:23.84703302Z DEBUG events [remediation] Storm recovery mode will exit after cooldown delay {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92666"}, "reason": "StormRecoveryCooldownStarted"} 2026-02-11T14:01:23.847069631Z DEBUG events [remediation] Storm recovery active: skipping creation of new remediations {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"92666"}, "reason": "RemediationSkipped"} 2026-02-11T14:01:23.847172658Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-2"}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."},{"type":"StormCooldownActive","status":"True","lastTransitionTime":"2026-02-11T14:01:23Z","reason":"HealthyNodeThresholdChange","message":"Storm cooldown delay started - waiting before resuming normal remediation"},{"type":"StormActive","status":"True","lastTransitionTime":"2026-02-11T13:31:57Z","reason":"HealthyNodeThresholdChange","message":"Storm mode is activated - preventing any new remediation until the storm is over and cooldown duration expired"}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2026-02-11T13:32:03Z"}, "patch": "{\"status\":{\"conditions\":[{\"lastTransitionTime\":\"2026-02-11T12:57:28Z\",\"message\":\"No issues found, NodeHealthCheck is enabled.\",\"reason\":\"NodeHealthCheckEnabled\",\"status\":\"False\",\"type\":\"Disabled\"},{\"lastTransitionTime\":\"2026-02-11T14:01:23Z\",\"message\":\"Storm cooldown delay started - waiting before resuming normal remediation\",\"reason\":\"HealthyNodeThresholdChange\",\"status\":\"True\",\"type\":\"StormCooldownActive\"},{\"lastTransitionTime\":\"2026-02-11T13:31:57Z\",\"message\":\"Storm mode is activated - preventing any new remediation until the storm is over and cooldown duration expired\",\"reason\":\"HealthyNodeThresholdChange\",\"status\":\"True\",\"type\":\"StormActive\"}],\"healthyNodes\":2,\"unhealthyNodes\":[{\"name\":\"worker-0-2\"}]}}"} 2026-02-11T14:01:24.052944482Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "1m1s"} 2026-02-11T14:01:44.107122368Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:01:44.119142213Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T14:01:44.119481791Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-2", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"102822"}, "reason": "DetectedUnhealthy"} 2026-02-11T14:01:44.128633996Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T14:01:44Z" level=info msg="invalidating lease" time="2026-02-11T14:01:44Z" level=info msg="getting lease" 2026-02-11T14:01:44.130747122Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:01:44Z" level=info msg="invalidating lease" time="2026-02-11T14:01:44Z" level=info msg="getting lease" 2026-02-11T14:01:44.132486213Z INFO controllers.NodeHealthCheck Storm recovery active: skipping creation of new remediations 2026-02-11T14:01:44.132729425Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "39.867515631s"} 2026-02-11T14:01:44.132790683Z DEBUG events [remediation] Storm recovery active: skipping creation of new remediations {"type": "Warning", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"102822"}, "reason": "RemediationSkipped"} 2026-02-11T14:02:24.001243277Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:02:24.005643726Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T14:02:24.005767031Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-2", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"102822"}, "reason": "DetectedUnhealthy"} 2026-02-11T14:02:24.010461031Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T14:02:24Z" level=info msg="invalidating lease" time="2026-02-11T14:02:24Z" level=info msg="getting lease" 2026-02-11T14:02:24.110971988Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:02:24Z" level=info msg="invalidating lease" time="2026-02-11T14:02:24Z" level=info msg="getting lease" 2026-02-11T14:02:24.112898544Z INFO controllers.NodeHealthCheck Storm recovery mode deactivated {"nhc": "nhc-worker-test"} 2026-02-11T14:02:24.112947409Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T14:02:24.113040654Z DEBUG events [remediation] Storm recovery mode deactivated - normal remediation resumed {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"102822"}, "reason": "StormRecoveryEnded"} 2026-02-11T14:02:24.211117505Z INFO controllers.NodeHealthCheck.resource manager Attempting to obtain Node Lease {"Node name": "worker-0-2"} time="2026-02-11T14:02:24Z" level=info msg="request lease" time="2026-02-11T14:02:24Z" level=info msg="getting lease" time="2026-02-11T14:02:24Z" level=info msg="create lease" 2026-02-11T14:02:24.214556635Z INFO controllers.NodeHealthCheck.resource manager Creating a remediation CR {"CR name": "", "CR kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} 2026-02-11T14:02:24.221278624Z DEBUG events [remediation] Created remediation object for node worker-0-2 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"102822"}, "reason": "RemediationCreated"} 2026-02-11T14:02:24.223902954Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-2","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-qwqfd","uid":"ce801684-ad69-497f-b070-9706ab85a6f5","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2026-02-11T14:02:24Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."},{"type":"StormCooldownActive","status":"False","lastTransitionTime":"2026-02-11T14:02:24Z","reason":"HealthyNodeThresholdChange","message":"Storm cooldown completed"},{"type":"StormActive","status":"False","lastTransitionTime":"2026-02-11T14:02:24Z","reason":"HealthyNodeThresholdChange","message":"Storm mode is deactivated, remediation can occur normally"}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2026-02-11T14:01:23Z"}, "patch": "{\"status\":{\"conditions\":[{\"lastTransitionTime\":\"2026-02-11T12:57:28Z\",\"message\":\"No issues found, NodeHealthCheck is enabled.\",\"reason\":\"NodeHealthCheckEnabled\",\"status\":\"False\",\"type\":\"Disabled\"},{\"lastTransitionTime\":\"2026-02-11T14:02:24Z\",\"message\":\"Storm cooldown completed\",\"reason\":\"HealthyNodeThresholdChange\",\"status\":\"False\",\"type\":\"StormCooldownActive\"},{\"lastTransitionTime\":\"2026-02-11T14:02:24Z\",\"message\":\"Storm mode is deactivated, remediation can occur normally\",\"reason\":\"HealthyNodeThresholdChange\",\"status\":\"False\",\"type\":\"StormActive\"}],\"phase\":\"Remediating\",\"reason\":\"NHC is remediating 1 nodes\",\"unhealthyNodes\":[{\"name\":\"worker-0-2\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-2-qwqfd\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"ce801684-ad69-497f-b070-9706ab85a6f5\"},\"started\":\"2026-02-11T14:02:24Z\",\"templateName\":\"selfnoderemediationtemplate-sample\"}]}]}}"} 2026-02-11T14:02:24.229363477Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:02:24.22942866Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:02:24.229442093Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:02:24.229446405Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:02:24.23366328Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:02:24.233692372Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:02:24.23370152Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:02:24.233705477Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:02:24.25680638Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:02:24.256846526Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:02:24.25686012Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:02:24.256864105Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:02:24.431298354Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2026-02-11T14:02:24.431391198Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:02:24.435448738Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T14:02:24.439171791Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-11T14:02:24Z"} 2026-02-11T14:02:24.439192254Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T14:02:24Z" level=info msg="invalidating lease" time="2026-02-11T14:02:24Z" level=info msg="getting lease" 2026-02-11T14:02:24.440564962Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:02:24Z" level=info msg="invalidating lease" time="2026-02-11T14:02:24Z" level=info msg="getting lease" 2026-02-11T14:02:24.442262362Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T14:02:24.445575372Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-2-qwqfd", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2026-02-11T14:02:24Z" level=info msg="getting lease" 2026-02-11T14:02:24.445620896Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-worker-test", "lease name": "node-worker-0-2", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2026-02-11T14:02:24.447366004Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2026-02-11T14:02:25.259878939Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:02:25.25991354Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:02:25.259928773Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:02:25.259933025Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:02:25.259999653Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:02:25.264262218Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "Unknown"} 2026-02-11T14:02:25.267968026Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-11T14:02:24Z"} 2026-02-11T14:02:25.267990275Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T14:02:25Z" level=info msg="invalidating lease" time="2026-02-11T14:02:25Z" level=info msg="getting lease" 2026-02-11T14:02:25.269629435Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:02:25Z" level=info msg="invalidating lease" time="2026-02-11T14:02:25Z" level=info msg="getting lease" 2026-02-11T14:02:25.271071359Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T14:02:25.274721516Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-2-qwqfd", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2026-02-11T14:02:25Z" level=info msg="getting lease" 2026-02-11T14:02:25.274764692Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-worker-test", "lease name": "node-worker-0-2", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2026-02-11T14:02:25.276463328Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2026-02-11T14:04:13.366628087Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-worker-test"} 2026-02-11T14:04:13.366683657Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-worker-test"} 2026-02-11T14:04:13.366721298Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:04:13.371268862Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-2", "condition type": "Ready", "condition status": "False", "duration left": "29.628733469s"} 2026-02-11T14:04:13.375885797Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-11T14:02:24Z"} 2026-02-11T14:04:13.375922335Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:04:13Z" level=info msg="invalidating lease" time="2026-02-11T14:04:13Z" level=info msg="getting lease" 2026-02-11T14:04:13.476743422Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T14:04:13Z" level=info msg="invalidating lease" time="2026-02-11T14:04:13Z" level=info msg="getting lease" 2026-02-11T14:04:13.478772101Z INFO controllers.NodeHealthCheck Ignoring node, because it was unhealthy, and is likely to be unhealthy again. {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T14:04:13.478993796Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "30.628733469s"} 2026-02-11T14:04:16.296406526Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-worker-test"} 2026-02-11T14:04:16.296480349Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-2", "NHC": "nhc-worker-test"} 2026-02-11T14:04:16.296529607Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:04:16.309238782Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-11T14:02:24Z"} 2026-02-11T14:04:16.309358018Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:04:16Z" level=info msg="invalidating lease" time="2026-02-11T14:04:16Z" level=info msg="getting lease" 2026-02-11T14:04:16.409867984Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T14:04:16.412047914Z INFO controllers.NodeHealthCheck.resource manager setting a delay for node getting healthy {"node name": "worker-0-2", "delay in seconds": 5} 2026-02-11T14:04:16.421929612Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} 2026-02-11T14:04:16.421927103Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:04:16.422074267Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:04:16.422088275Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:04:16.422093115Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} time="2026-02-11T14:04:16Z" level=info msg="invalidating lease" time="2026-02-11T14:04:16Z" level=info msg="getting lease" 2026-02-11T14:04:16.424431889Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-2","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-2-qwqfd","uid":"ce801684-ad69-497f-b070-9706ab85a6f5","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2026-02-11T14:02:24Z","templateName":"selfnoderemediationtemplate-sample"}],"conditionsHealthyTimestamp":"2026-02-11T14:04:16Z","healthyDelayed":true}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."},{"type":"StormCooldownActive","status":"False","lastTransitionTime":"2026-02-11T14:02:24Z","reason":"HealthyNodeThresholdChange","message":"Storm cooldown completed"},{"type":"StormActive","status":"False","lastTransitionTime":"2026-02-11T14:02:24Z","reason":"HealthyNodeThresholdChange","message":"Storm mode is deactivated, remediation can occur normally"}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2026-02-11T14:02:24Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"conditionsHealthyTimestamp\":\"2026-02-11T14:04:16Z\",\"healthyDelayed\":true,\"name\":\"worker-0-2\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-2-qwqfd\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"ce801684-ad69-497f-b070-9706ab85a6f5\"},\"started\":\"2026-02-11T14:02:24Z\",\"templateName\":\"selfnoderemediationtemplate-sample\"}]}]}}"} 2026-02-11T14:04:16.630731182Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "6s"} 2026-02-11T14:04:16.630825439Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:04:16.639807287Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-11T14:02:24Z"} 2026-02-11T14:04:16.639833953Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:04:16Z" level=info msg="invalidating lease" time="2026-02-11T14:04:16Z" level=info msg="getting lease" 2026-02-11T14:04:16.641574003Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T14:04:16.643254241Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "worker-0-2", "remaining time in seconds": 4.3567473549999995} 2026-02-11T14:04:16.643271824Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T14:04:16Z" level=info msg="invalidating lease" time="2026-02-11T14:04:16Z" level=info msg="getting lease" 2026-02-11T14:04:16.645114108Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "5.356747355s"} 2026-02-11T14:04:22.002103879Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:04:22.009593505Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-worker-test", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2026-02-11T14:02:24Z"} 2026-02-11T14:04:22.009618487Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T14:04:22.011229052Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-2"} 2026-02-11T14:04:22.017901091Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:04:22.017931913Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:04:22.017946639Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:04:22.017952689Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:04:22.018070233Z INFO controllers.NodeHealthCheck.resource manager deleted remediation CR {"name": "worker-0-2-qwqfd"} 2026-02-11T14:04:22.018097578Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} 2026-02-11T14:04:22.018161403Z DEBUG events [remediation] Deleted remediation CR of kind SelfNodeRemediation with name worker-0-2-qwqfd {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"104322"}, "reason": "RemediationRemoved"} time="2026-02-11T14:04:22Z" level=info msg="invalidating lease" time="2026-02-11T14:04:22Z" level=info msg="getting lease" 2026-02-11T14:04:22.019926042Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:04:22Z" level=info msg="invalidating lease" time="2026-02-11T14:04:22Z" level=info msg="getting lease" 2026-02-11T14:04:22.110534815Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "11s"} 2026-02-11T14:04:22.11060136Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:04:22.117749535Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T14:04:22Z" level=info msg="invalidating lease" time="2026-02-11T14:04:22Z" level=info msg="getting lease" 2026-02-11T14:04:22.11939286Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:04:22Z" level=info msg="invalidating lease" time="2026-02-11T14:04:22Z" level=info msg="getting lease" 2026-02-11T14:04:22.121035095Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T14:04:22.122577199Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-2"} 2026-02-11T14:04:22.124518085Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "17s"} 2026-02-11T14:04:26.010650372Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:04:26.010685442Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:04:26.010699832Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:04:26.010704312Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:04:26.010737057Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:04:26.019665926Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:04:26Z" level=info msg="invalidating lease" time="2026-02-11T14:04:26Z" level=info msg="getting lease" 2026-02-11T14:04:26.022462964Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T14:04:26.024438424Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-2"} 2026-02-11T14:04:26.027810451Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} 2026-02-11T14:04:26.042824788Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:04:26.042856307Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:04:26.042867195Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:04:26.04287097Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} time="2026-02-11T14:04:26Z" level=info msg="invalidating lease" time="2026-02-11T14:04:26Z" level=info msg="getting lease" 2026-02-11T14:04:26.043196959Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "17s"} 2026-02-11T14:04:26.043278116Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:04:26.119997518Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T14:04:26Z" level=info msg="invalidating lease" time="2026-02-11T14:04:26Z" level=info msg="getting lease" 2026-02-11T14:04:26.121743876Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:04:26Z" level=info msg="invalidating lease" time="2026-02-11T14:04:26Z" level=info msg="getting lease" 2026-02-11T14:04:26.123638942Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} 2026-02-11T14:04:26.125449477Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "worker-0-2"} 2026-02-11T14:04:26.127427198Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "17s"} 2026-02-11T14:04:27.0771126Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-worker-test","uid":"687640f7-dc4a-467d-a7ee-cf2304729c15","controller":false}} 2026-02-11T14:04:27.077163261Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-worker-test", "Remediation CR Name": "worker-0-2-qwqfd", "Remediation CR Kind": "SelfNodeRemediation"} 2026-02-11T14:04:27.077241424Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:04:27.094756415Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T14:04:27Z" level=info msg="invalidating lease" time="2026-02-11T14:04:27Z" level=info msg="getting lease" 2026-02-11T14:04:27.096927427Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:04:27Z" level=info msg="invalidating lease" time="2026-02-11T14:04:27Z" level=info msg="getting lease" 2026-02-11T14:04:27.101061723Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} time="2026-02-11T14:04:27Z" level=info msg="invalidating lease" time="2026-02-11T14:04:27Z" level=info msg="getting lease" 2026-02-11T14:04:27.106526989Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-worker-test", "new status": {"observedNodes":3,"healthyNodes":3,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2026-02-11T12:57:28Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."},{"type":"StormCooldownActive","status":"False","lastTransitionTime":"2026-02-11T14:02:24Z","reason":"HealthyNodeThresholdChange","message":"Storm cooldown completed"},{"type":"StormActive","status":"False","lastTransitionTime":"2026-02-11T14:02:24Z","reason":"HealthyNodeThresholdChange","message":"Storm mode is deactivated, remediation can occur normally"}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2026-02-11T14:04:16Z"}, "patch": "{\"status\":{\"healthyNodes\":3,\"phase\":\"Enabled\",\"reason\":\"NHC is enabled, no ongoing remediation\",\"unhealthyNodes\":null}}"} 2026-02-11T14:04:27.313710722Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "0s"} 2026-02-11T14:04:33.111555367Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-worker-test"} 2026-02-11T14:04:33.118784788Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-0"} time="2026-02-11T14:04:33Z" level=info msg="invalidating lease" time="2026-02-11T14:04:33Z" level=info msg="getting lease" 2026-02-11T14:04:33.120143616Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-1"} time="2026-02-11T14:04:33Z" level=info msg="invalidating lease" time="2026-02-11T14:04:33Z" level=info msg="getting lease" 2026-02-11T14:04:33.121644836Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-worker-test", "node": "worker-0-2"} time="2026-02-11T14:04:33Z" level=info msg="invalidating lease" time="2026-02-11T14:04:33Z" level=info msg="getting lease" 2026-02-11T14:04:33.123157369Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-worker-test", "error": null, "requeue": false, "requeuAfter": "0s"}