========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.21.0-0.nightly-2025-12-22-170804 True False 6d12h Cluster version is 4.21.0-0.nightly-2025-12-22-170804 ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get csv NAME DISPLAY VERSION REPLACES PHASE fence-agents-remediation.v0.6.0 Fence Agents Remediation Operator 0.6.0 fence-agents-remediation.v0.5.1 Succeeded machine-deletion-remediation.v0.5.0 Machine Deletion Remediation operator 0.5.0 machine-deletion-remediation.v0.4.1 Succeeded node-healthcheck-operator.v0.10.1 Node Health Check Operator 0.10.1 node-healthcheck-operator.v0.10.0 Succeeded node-maintenance-operator.v5.5.0 Node Maintenance Operator 5.5.0 node-maintenance-operator.v5.4.1 Succeeded self-node-remediation.v0.11.0 Self Node Remediation Operator 0.11.0 self-node-remediation.v0.10.2 Succeeded ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get pods NAME READY STATUS RESTARTS AGE fence-agents-remediation-controller-manager-74bfb5d6bd-b4fkz 2/2 Running 0 4h40m fence-agents-remediation-controller-manager-74bfb5d6bd-zqj79 2/2 Running 0 4h40m machine-deletion-remediation-controller-manager-649b8f7d4d6c2h4 2/2 Running 0 4h44m node-healthcheck-controller-manager-6d9f47cbbb-7z8lv 2/2 Running 0 4h57m node-healthcheck-controller-manager-6d9f47cbbb-bs4pk 2/2 Running 0 4h57m node-healthcheck-node-remediation-console-plugin-7c85894848pbwz 1/1 Running 0 4h57m node-maintenance-operator-controller-manager-855c67587f-5kmrh 1/1 Running 0 4h42m self-node-remediation-controller-manager-57c976f744-lxjrv 2/2 Running 0 4h46m self-node-remediation-controller-manager-57c976f744-s7wsl 2/2 Running 0 4h46m self-node-remediation-ds-5hp9f 1/1 Running 0 4h46m self-node-remediation-ds-9pcg9 1/1 Running 0 4h46m self-node-remediation-ds-g87hc 1/1 Running 0 4h46m self-node-remediation-ds-grtnv 1/1 Running 0 4h46m self-node-remediation-ds-nh5jb 1/1 Running 0 4h46m self-node-remediation-ds-t4bpw 1/1 Running 0 4h46m ========================================================================================================================================================== [root@cert-rhosp-02 ~]# cat test.yaml --- kind: MachineDeletionRemediationTemplate apiVersion: machine-deletion-remediation.medik8s.io/v1alpha1 metadata: name: mdr-template namespace: openshift-workload-availability spec: template: spec: {} --- apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate metadata: namespace: openshift-workload-availability name: selfnoderemediationtemplate-sample spec: template: spec: remediationStrategy: Automatic --- apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate metadata: namespace: openshift-workload-availability name: selfnoderemediationtemplate-sample-2 spec: template: spec: remediationStrategy: Automatic --- apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate metadata: name: far-template namespace: openshift-workload-availability spec: template: spec: agent: fence_ipmilan retrycount: 1 retryinterval: 1s timeout: 1s nodeparameters: '--ipport': master-0-0: '6230' master-0-1: '6231' master-0-2: '6232' worker-0-0: '6233' worker-0-1: '6234' worker-0-2: '6235' sharedparameters: '--action': reboot '--lanplus': '' '--ip': 192.168.123.1 '--password': password '--username': admin --- apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: name: nhc-mdr-snr spec: minHealthy: 30% healthyDelay: 5s escalatingRemediations: - remediationTemplate: apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate name: selfnoderemediationtemplate-sample namespace: openshift-workload-availability order: 0 timeout: 60s - remediationTemplate: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate name: far-template namespace: openshift-workload-availability order: 1 timeout: 60s - remediationTemplate: apiVersion: machine-deletion-remediation.medik8s.io/v1alpha1 kind: MachineDeletionRemediationTemplate name: mdr-template namespace: openshift-workload-availability order: 2 timeout: 600s selector: matchExpressions: - key: node-role.kubernetes.io/worker operator: Exists unhealthyConditions: - type: Ready status: "False" duration: 30s - type: Ready status: Unknown duration: 30s ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc apply -f test.yaml machinedeletionremediationtemplate.machine-deletion-remediation.medik8s.io/mdr-template created selfnoderemediationtemplate.self-node-remediation.medik8s.io/selfnoderemediationtemplate-sample created selfnoderemediationtemplate.self-node-remediation.medik8s.io/selfnoderemediationtemplate-sample-2 created fenceagentsremediationtemplate.fence-agents-remediation.medik8s.io/far-template created nodehealthcheck.remediation.medik8s.io/nhc-mdr-snr created ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get nhc ; oc get snr; oc get far NAME AGE nhc-mdr-snr 16s No resources found in openshift-workload-availability namespace. No resources found in openshift-workload-availability namespace. ========================================================================================================================================================== [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep fence-agents-remediation-controller-manager) ========================================================================================================================================================== [root@cert-rhosp-02 ~]# echo $PODS pod/fence-agents-remediation-controller-manager-74bfb5d6bd-b4fkz pod/fence-agents-remediation-controller-manager-74bfb5d6bd-zqj79 ========================================================================================================================================================== [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/fence-agents-remediation-controller-manager-74bfb5d6bd-b4fkz "worker-0-2" == pod/fence-agents-remediation-controller-manager-74bfb5d6bd-zqj79 "worker-0-0" ========================================================================================================================================================== [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep node-healthcheck-controller-manager) ========================================================================================================================================================== [root@cert-rhosp-02 ~]# echo $PODS pod/node-healthcheck-controller-manager-6d9f47cbbb-7z8lv pod/node-healthcheck-controller-manager-6d9f47cbbb-bs4pk ========================================================================================================================================================== [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/node-healthcheck-controller-manager-6d9f47cbbb-7z8lv "master-0-2" == pod/node-healthcheck-controller-manager-6d9f47cbbb-bs4pk "master-0-1" ========================================================================================================================================================== [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep self-node-remediation-controller-manager) ========================================================================================================================================================== [root@cert-rhosp-02 ~]# echo $PODS pod/self-node-remediation-controller-manager-57c976f744-lxjrv pod/self-node-remediation-controller-manager-57c976f744-s7wsl ========================================================================================================================================================== [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/self-node-remediation-controller-manager-57c976f744-lxjrv "worker-0-2" == pod/self-node-remediation-controller-manager-57c976f744-s7wsl "worker-0-1" ========================================================================================================================================================== [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep machine-deletion-remediation-controller) ========================================================================================================================================================== [root@cert-rhosp-02 ~]# echo $PODS pod/machine-deletion-remediation-controller-manager-649b8f7d4d6c2h4 ========================================================================================================================================================== [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/machine-deletion-remediation-controller-manager-649b8f7d4d6c2h4 "worker-0-2" ========================================================================================================================================================== [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep node-maintenance-operator-controller-manager) ========================================================================================================================================================== [root@cert-rhosp-02 ~]# echo $PODS pod/node-maintenance-operator-controller-manager-855c67587f-5kmrh ========================================================================================================================================================== [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/node-maintenance-operator-controller-manager-855c67587f-5kmrh "master-0-2" ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 6d22h v1.34.2 master-0-1 Ready control-plane,master 6d22h v1.34.2 master-0-2 Ready control-plane,master 6d22h v1.34.2 worker-0-0 Ready worker 6d21h v1.34.2 worker-0-1 Ready worker 6d21h v1.34.2 worker-0-2 Ready worker 6d21h v1.34.2 ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get fartemplate; oc get snrt; oc get MachineDeletionRemediationTemplate NAME AGE far-template 9m11s NAME AGE self-node-remediation-automatic-strategy-template 4h58m selfnoderemediationtemplate-sample 9m11s selfnoderemediationtemplate-sample-2 9m11s NAME AGE mdr-template 9m11s ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 6d22h v1.34.2 master-0-1 Ready control-plane,master 6d22h v1.34.2 master-0-2 Ready control-plane,master 6d22h v1.34.2 worker-0-0 Ready worker 6d21h v1.34.2 worker-0-1 NotReady worker 6d21h v1.34.2 worker-0-2 Ready worker 6d21h v1.34.2 ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get nhc ; oc get snr; oc get far; oc get mdr NAME AGE nhc-mdr-snr 20m NAME AGE worker-0-1-nb6bc 3m41s NAME AGE worker-0-1-jqcbg 2m41s NAME AGE worker-0-1 2m39s ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 6d22h v1.34.2 master-0-1 Ready control-plane,master 6d22h v1.34.2 master-0-2 Ready control-plane,master 6d22h v1.34.2 worker-0-0 Ready worker 6d21h v1.34.2 worker-0-1 NotReady,SchedulingDisabled worker 6d21h v1.34.2 worker-0-2 Ready worker 6d21h v1.34.2 ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get nhc ; oc get snr; oc get far; oc get mdr NAME AGE nhc-mdr-snr 24m No resources found in openshift-workload-availability namespace. NAME AGE worker-0-1-jqcbg 6m33s No resources found in openshift-workload-availability namespace. ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 6d22h v1.34.2 master-0-1 Ready control-plane,master 6d22h v1.34.2 master-0-2 Ready control-plane,master 6d22h v1.34.2 worker-0-0 Ready worker 6d21h v1.34.2 worker-0-2 Ready worker 6d21h v1.34.2 ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get nhc ; oc get snr; oc get far; oc get mdr NAME AGE nhc-mdr-snr 61m No resources found in openshift-workload-availability namespace. No resources found in openshift-workload-availability namespace. No resources found in openshift-workload-availability namespace. ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc get nhc -o yaml apiVersion: v1 items: - apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","metadata":{"annotations":{},"name":"nhc-mdr-snr"},"spec":{"escalatingRemediations":[{"order":0,"remediationTemplate":{"apiVersion":"self-node-remediation.medik8s.io/v1alpha1","kind":"SelfNodeRemediationTemplate","name":"selfnoderemediationtemplate-sample","namespace":"openshift-workload-availability"},"timeout":"60s"},{"order":1,"remediationTemplate":{"apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","kind":"FenceAgentsRemediationTemplate","name":"far-template","namespace":"openshift-workload-availability"},"timeout":"60s"},{"order":2,"remediationTemplate":{"apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1","kind":"MachineDeletionRemediationTemplate","name":"mdr-template","namespace":"openshift-workload-availability"},"timeout":"600s"}],"healthyDelay":"5s","minHealthy":"30%","selector":{"matchExpressions":[{"key":"node-role.kubernetes.io/worker","operator":"Exists"}]},"unhealthyConditions":[{"duration":"30s","status":"False","type":"Ready"},{"duration":"30s","status":"Unknown","type":"Ready"}]}} creationTimestamp: "2025-12-30T18:45:45Z" generation: 1 name: nhc-mdr-snr resourceVersion: "3631657" uid: 054d5047-6762-4082-b08d-a52405f32d1e spec: escalatingRemediations: - order: 0 remediationTemplate: apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate name: selfnoderemediationtemplate-sample namespace: openshift-workload-availability timeout: 60s - order: 1 remediationTemplate: apiVersion: fence-agents-remediation.medik8s.io/v1alpha1 kind: FenceAgentsRemediationTemplate name: far-template namespace: openshift-workload-availability timeout: 60s - order: 2 remediationTemplate: apiVersion: machine-deletion-remediation.medik8s.io/v1alpha1 kind: MachineDeletionRemediationTemplate name: mdr-template namespace: openshift-workload-availability timeout: 600s healthyDelay: 5s minHealthy: 30% selector: matchExpressions: - key: node-role.kubernetes.io/worker operator: Exists unhealthyConditions: - duration: 30s status: "False" type: Ready - duration: 30s status: Unknown type: Ready status: conditions: - lastTransitionTime: "2025-12-30T18:45:45Z" message: No issues found, NodeHealthCheck is enabled. reason: NodeHealthCheckEnabled status: "False" type: Disabled healthyNodes: 3 lastUpdateTime: "2025-12-30T19:23:32Z" observedNodes: 3 phase: Enabled reason: NHC is enabled, no ongoing remediation kind: List metadata: resourceVersion: "" NHC Logs: ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc logs pod/node-healthcheck-controller-manager-6d9f47cbbb-7z8lv 2025-12-30T13:45:11.746608981Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-12-30T13:45:11.74671625Z INFO setup Go OS/Arch: linux/amd64 2025-12-30T13:45:11.746719645Z INFO setup Operator Version: 2ef588d 2025-12-30T13:45:11.74672156Z INFO setup Git Commit: 2ef588d65de4087c46447401cd6e757459d79210 2025-12-30T13:45:11.746723333Z INFO setup Build Date: 2025-11-11T13:40:45+00:00 2025-12-30T13:45:11.746725163Z INFO setup HTTP/2 for metrics and webhook server disabled 2025-12-30T13:45:11.771548663Z INFO setup supported control plane topology {"topology": "HighlyAvailable"} 2025-12-30T13:45:11.771581462Z INFO setup Cluster capabilities {"IsOnOpenshift": true, "HasMachineAPI": true} 2025-12-30T13:45:11.771830433Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "remediation.medik8s.io/v1alpha1, Kind=NodeHealthCheck", "path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2025-12-30T13:45:11.771901307Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2025-12-30T13:45:11.771944289Z INFO setup starting manager 2025-12-30T13:45:11.771988607Z INFO controller-runtime.metrics Starting metrics server 2025-12-30T13:45:11.772032527Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-12-30T13:45:11.772054733Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": "127.0.0.1:8080", "secure": false} 2025-12-30T13:45:11.77208786Z INFO controller-runtime.webhook Starting webhook server 2025-12-30T13:45:11.772304099Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-12-30T13:45:11.772364686Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-12-30T13:45:11.772446602Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} I1230 13:45:11.872859 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/e1f13584.medik8s.io... 2025-12-30T18:45:45.83648136Z INFO nodehealthcheck-resource validate create {"name": "nhc-mdr-snr"} I1230 18:57:49.264470 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/e1f13584.medik8s.io 2025-12-30T18:57:49.264534051Z DEBUG events node-healthcheck-controller-manager-6d9f47cbbb-7z8lv_b5ea5a1a-5904-41d8-b398-312bafc0c956 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"e1f13584.medik8s.io","uid":"ba5e9ac2-85d9-4886-81f8-d83c5442ac35","apiVersion":"coordination.k8s.io/v1","resourceVersion":"3618700"}, "reason": "LeaderElection"} 2025-12-30T18:57:49.264782538Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "channel source: 0xc0006d6700"} 2025-12-30T18:57:49.26481281Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1.Node"} 2025-12-30T18:57:49.264810615Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1alpha1.NodeHealthCheck"} 2025-12-30T18:57:49.264824583Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "channel source: 0xc0006d6930"} 2025-12-30T18:57:49.264832577Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.MachineHealthCheck"} 2025-12-30T18:57:49.264838493Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1.Node"} 2025-12-30T18:57:49.26484102Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.Machine"} I1230 18:57:49.266072 1 shared_informer.go:313] Waiting for caches to sync for feature gate accessor 2025-12-30T18:57:49.365639367Z INFO MHCChecker found termination handler MHC, will ignore Nodes with Terminating condition 2025-12-30T18:57:49.365684352Z INFO MHCChecker MHC Checker status changed, notifying NHC controller 2025-12-30T18:57:49.365720324Z INFO adding all NHCs to reconcile queue for handling MHC event I1230 18:57:49.366820 1 shared_informer.go:320] Caches are synced for feature gate accessor I1230 18:57:49.366846 1 simple_featuregate_reader.go:171] Starting feature-gate-detector 2025-12-30T18:57:49.366955503Z INFO FeatureGateAccessor FeatureGates initialized I1230 18:57:49.366994 1 recorder_logging.go:49] &Event{ObjectMeta:{dummy.1886133ffeb95ab4.4b2a179c dummy 0 0001-01-01 00:00:00 +0000 UTC map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Pod,Namespace:dummy,Name:dummy,UID:,APIVersion:v1,ResourceVersion:,FieldPath:,},Reason:FeatureGatesInitialized,Message:FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "NewOLM", "NewOLMWebhookProviderOpenshiftServiceCA", "OpenShiftPodSecurityAdmission", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "HyperShiftOnlyDynamicResourceAllocation", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}},Source:EventSource{Component:,Host:,},FirstTimestamp:2025-12-30 18:57:49.366926004 +0000 UTC m=+18757.640302464,LastTimestamp:2025-12-30 18:57:49.366926004 +0000 UTC m=+18757.640302464,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:,ReportingInstance:,} 2025-12-30T18:57:49.465825518Z INFO Starting Controller {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck"} 2025-12-30T18:57:49.465860601Z INFO Starting workers {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "worker count": 1} 2025-12-30T18:57:49.46582575Z INFO Starting Controller {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck"} 2025-12-30T18:57:49.465938222Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:57:49.465942673Z INFO Starting workers {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "worker count": 1} 2025-12-30T18:57:59.942321153Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:57:59.943220376Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":0,"healthyNodes":0,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-12-30T18:45:45Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-12-30T18:45:45Z"}, "patch": "{\"status\":{\"healthyNodes\":0,\"observedNodes\":0}}"} 2025-12-30T18:58:00.10579544Z INFO console-plugin successfully created / updated console plugin resources 2025-12-30T18:58:00.152939023Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T18:58:00.153193214Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "a3f0ddfa-b8c8-4ac4-adb0-76bed754b0e6", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.158524938Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:58:00.161873996Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.162153437Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T18:58:00.162207681Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "7d3587e0-2625-4fed-b465-7ab57d1e7e58", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.172494713Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:58:00.177131508Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.177465665Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T18:58:00.177537396Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "d9012f4d-4cc5-4701-862b-d95928f8817a", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.197941547Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:58:00.201976642Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.202217883Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T18:58:00.20227053Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "12a8c02e-b78b-4bc0-8dd7-ab1e20896aa8", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.242674913Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:58:00.24666184Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.246974933Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T18:58:00.247056801Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "70caf7e4-4831-462e-a726-c99f5fb9505f", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.328205743Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:58:00.333078076Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.333406163Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T18:58:00.333484658Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "2e9118d5-6a25-421b-a37a-753b4ecbba0b", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.495891831Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:58:00.500060051Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.500429803Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T18:58:00.500517012Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "f9d5e116-544b-41f3-b1e2-c2b613975d78", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.820640679Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:58:00.824859417Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:00.825203477Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T18:58:00.825279755Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "6f5f36f3-b521-46f7-b171-dad953656b37", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:01.465530737Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:58:01.470364944Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:01.470632065Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T18:58:01.470690549Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "5f7c6f1e-624f-4aeb-83d2-421b22165f91", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:02.751436587Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:58:02.755491525Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:02.755735619Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T18:58:02.755798355Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "11d98cc9-91f4-479f-8724-c7f3717a84de", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:05.316905329Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:58:05.320072166Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:05.320391834Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T18:58:05.320493724Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "9c910278-ce2d-41e4-a372-e57e8f21e907", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:10.441262596Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:58:10.445622357Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:10.445912814Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T18:58:10.445967489Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "1d054381-4997-41cb-b1b9-4193316f01f4", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:128\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T18:58:20.686446939Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T18:58:20.703663236Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-12-30T18:58:20.70378354Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation template CRs {"kind": "SelfNodeRemediationTemplate"} 2025-12-30T18:58:20.703813251Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-12-30T18:58:20.703827812Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation CRs {"kind": "SelfNodeRemediation"} 2025-12-30T18:58:20.703842842Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-12-30T18:58:20.703855006Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation template CRs {"kind": "FenceAgentsRemediationTemplate"} 2025-12-30T18:58:20.703867931Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-12-30T18:58:20.703879222Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation CRs {"kind": "FenceAgentsRemediation"} 2025-12-30T18:58:20.703892936Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-12-30T18:58:20.703907243Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation template CRs {"kind": "MachineDeletionRemediationTemplate"} 2025-12-30T18:58:20.703919619Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-12-30T18:58:20.703930463Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation CRs {"kind": "MachineDeletionRemediation"} 2025-12-30T18:58:20.717299678Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-12-30T18:58:30Z" level=info msg="invalidating lease" time="2025-12-30T18:58:30Z" level=info msg="getting lease" 2025-12-30T18:58:30.828060613Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T18:58:30Z" level=info msg="invalidating lease" time="2025-12-30T18:58:30Z" level=info msg="getting lease" 2025-12-30T18:58:30.837810415Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T18:58:30Z" level=info msg="invalidating lease" time="2025-12-30T18:58:30Z" level=info msg="getting lease" 2025-12-30T18:58:30.845900453Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":3,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-12-30T18:45:45Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-12-30T18:57:59Z"}, "patch": "{\"status\":{\"healthyNodes\":3,\"observedNodes\":3}}"} 2025-12-30T18:58:31.052305062Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "0s"} E1230 18:59:48.541686 1 leaderelection.go:429] Failed to update lock optimistically: Put "https://172.30.0.1:443/apis/coordination.k8s.io/v1/namespaces/openshift-workload-availability/leases/e1f13584.medik8s.io?timeout=5s": context deadline exceeded, falling back to slow path 2025-12-30T19:02:06.902037865Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-1", "NHC": "nhc-mdr-snr"} 2025-12-30T19:02:06.902135399Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-1", "NHC": "nhc-mdr-snr"} 2025-12-30T19:02:06.902196725Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:02:06.914803327Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown", "duration left": "29.085199193s"} 2025-12-30T19:02:06.926276803Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:02:07Z" level=info msg="invalidating lease" time="2025-12-30T19:02:07Z" level=info msg="getting lease" 2025-12-30T19:02:07.027238634Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:02:07Z" level=info msg="invalidating lease" time="2025-12-30T19:02:07Z" level=info msg="getting lease" 2025-12-30T19:02:07.130678522Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-12-30T18:45:45Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-12-30T18:58:30Z"}, "patch": "{\"status\":{\"healthyNodes\":2}}"} 2025-12-30T19:02:07.338053932Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "30.085199193s"} 2025-12-30T19:02:37.423849151Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:02:37.431591731Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown"} 2025-12-30T19:02:37.431771565Z DEBUG events [remediation] Node matches unhealthy condition. Node "worker-0-1", condition type "Ready", condition status "Unknown" {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"3621737"}, "reason": "DetectedUnhealthy"} 2025-12-30T19:02:37.439144958Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:02:37Z" level=info msg="invalidating lease" time="2025-12-30T19:02:37Z" level=info msg="getting lease" 2025-12-30T19:02:37.443646147Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:02:37Z" level=info msg="invalidating lease" time="2025-12-30T19:02:37Z" level=info msg="getting lease" 2025-12-30T19:02:37.447677579Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-12-30T19:02:37.451204753Z INFO controllers.NodeHealthCheck.resource manager Attempting to obtain Node Lease {"Node name": "worker-0-1"} time="2025-12-30T19:02:37Z" level=info msg="request lease" time="2025-12-30T19:02:37Z" level=info msg="getting lease" time="2025-12-30T19:02:37Z" level=info msg="create lease" 2025-12-30T19:02:37.453885072Z INFO controllers.NodeHealthCheck.resource manager Creating a remediation CR {"CR name": "", "CR kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} 2025-12-30T19:02:37.462042333Z DEBUG events [remediation] Created remediation object for node worker-0-1 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"3621737"}, "reason": "RemediationCreated"} 2025-12-30T19:02:37.540648997Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-1","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-nb6bc","uid":"d8994bd3-82f8-41e5-b57e-02476ef456ce","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-12-30T19:02:37Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-12-30T18:45:45Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-12-30T19:02:07Z"}, "patch": "{\"status\":{\"phase\":\"Remediating\",\"reason\":\"NHC is remediating 1 nodes\",\"unhealthyNodes\":[{\"name\":\"worker-0-1\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-1-nb6bc\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"d8994bd3-82f8-41e5-b57e-02476ef456ce\"},\"started\":\"2025-12-30T19:02:37Z\",\"templateName\":\"selfnoderemediationtemplate-sample\"}]}]}}"} 2025-12-30T19:02:37.668419198Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:02:37.668475555Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:02:37.668487757Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:02:37.668493491Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:02:37.672144935Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:02:37.672169549Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:02:37.672177876Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:02:37.672181858Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:02:37.709810013Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:02:37.709845296Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:02:37.70985545Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:02:37.709859418Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:02:37.74781869Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "1m0s"} 2025-12-30T19:02:37.747917785Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:02:37.784055334Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown"} 2025-12-30T19:02:37.808936146Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:02:37Z"} 2025-12-30T19:02:37.808973647Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:02:37Z" level=info msg="invalidating lease" time="2025-12-30T19:02:37Z" level=info msg="getting lease" 2025-12-30T19:02:37.925573102Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:02:38Z" level=info msg="invalidating lease" time="2025-12-30T19:02:38Z" level=info msg="getting lease" 2025-12-30T19:02:38.007183054Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-12-30T19:02:38.011358773Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-1-nb6bc", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2025-12-30T19:02:38Z" level=info msg="getting lease" 2025-12-30T19:02:38.011443396Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-1", "NHC is lease owner": true, "lease expiration time": "1m0s"} 2025-12-30T19:02:38.011467891Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:02:37Z"} 2025-12-30T19:02:38.017032979Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "58.988535568s"} 2025-12-30T19:02:38.709350447Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:02:38.709402134Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:02:38.709420517Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:02:38.709429259Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:02:38.709470806Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:02:38.717262697Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown"} 2025-12-30T19:02:38.724724403Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:02:37Z"} 2025-12-30T19:02:38.724754221Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:02:38Z" level=info msg="invalidating lease" time="2025-12-30T19:02:38Z" level=info msg="getting lease" 2025-12-30T19:02:38.732837991Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:02:38Z" level=info msg="invalidating lease" time="2025-12-30T19:02:38Z" level=info msg="getting lease" 2025-12-30T19:02:38.738128466Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-12-30T19:02:38.741783311Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-1-nb6bc", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2025-12-30T19:02:38Z" level=info msg="getting lease" 2025-12-30T19:02:38.741857973Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-1", "NHC is lease owner": true, "lease expiration time": "1m0s"} 2025-12-30T19:02:38.741883695Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:02:37Z"} 2025-12-30T19:02:38.746286885Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "58.258120617s"} 2025-12-30T19:03:37.004603065Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:03:37.012532948Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown"} 2025-12-30T19:03:37.020449948Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:02:37Z"} 2025-12-30T19:03:37.020472173Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:03:37Z" level=info msg="invalidating lease" time="2025-12-30T19:03:37Z" level=info msg="getting lease" 2025-12-30T19:03:37.02499812Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:03:37Z" level=info msg="invalidating lease" time="2025-12-30T19:03:37Z" level=info msg="getting lease" 2025-12-30T19:03:37.029531562Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-12-30T19:03:37.03285651Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-1-nb6bc", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2025-12-30T19:03:37Z" level=info msg="getting lease" 2025-12-30T19:03:37.032906963Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-1", "NHC is lease owner": true, "lease expiration time": "1m0s"} 2025-12-30T19:03:37.032927574Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:02:37Z"} 2025-12-30T19:03:37.032935405Z INFO controllers.NodeHealthCheck remediation timed out {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:03:37.032938728Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:02:37Z"} 2025-12-30T19:03:37.041144169Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:37.041189961Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:03:37.04119986Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:37.041204231Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:03:37.045780673Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:37.045880841Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:03:37.045937846Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:37.045959843Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:03:37.121701201Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-1","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-nb6bc","uid":"d8994bd3-82f8-41e5-b57e-02476ef456ce","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-12-30T19:02:37Z","timedOut":"2025-12-30T19:03:37Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-12-30T18:45:45Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-12-30T19:02:37Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"name\":\"worker-0-1\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-1-nb6bc\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"d8994bd3-82f8-41e5-b57e-02476ef456ce\"},\"started\":\"2025-12-30T19:02:37Z\",\"templateName\":\"selfnoderemediationtemplate-sample\",\"timedOut\":\"2025-12-30T19:03:37Z\"}]}]}}"} 2025-12-30T19:03:37.327822496Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "1s"} 2025-12-30T19:03:37.327907735Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:03:37.335445944Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown"} 2025-12-30T19:03:37.342503811Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:03:37.342523441Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:03:37Z" level=info msg="invalidating lease" time="2025-12-30T19:03:37Z" level=info msg="getting lease" 2025-12-30T19:03:37.346929295Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:03:37Z" level=info msg="invalidating lease" time="2025-12-30T19:03:37Z" level=info msg="getting lease" 2025-12-30T19:03:37.351153479Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-12-30T19:03:37.354353957Z INFO controllers.NodeHealthCheck.resource manager Attempting to obtain Node Lease {"Node name": "worker-0-1"} time="2025-12-30T19:03:37Z" level=info msg="request lease" time="2025-12-30T19:03:37Z" level=info msg="getting lease" time="2025-12-30T19:03:37Z" level=info msg="renew lease owned by NodeHealthCheck-nhc-mdr-snr setAcquireTime=false" 2025-12-30T19:03:37.357094288Z INFO controllers.NodeHealthCheck.resource manager Creating a remediation CR {"CR name": "", "CR kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} 2025-12-30T19:03:37.364839862Z DEBUG events [remediation] Created remediation object for node worker-0-1 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"3622988"}, "reason": "RemediationCreated"} 2025-12-30T19:03:37.444963885Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-1","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-nb6bc","uid":"d8994bd3-82f8-41e5-b57e-02476ef456ce","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-12-30T19:02:37Z","timedOut":"2025-12-30T19:03:37Z","templateName":"selfnoderemediationtemplate-sample"},{"resource":{"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-jqcbg","uid":"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1"},"started":"2025-12-30T19:03:37Z","templateName":"far-template"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-12-30T18:45:45Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-12-30T19:03:37Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"name\":\"worker-0-1\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-1-nb6bc\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"d8994bd3-82f8-41e5-b57e-02476ef456ce\"},\"started\":\"2025-12-30T19:02:37Z\",\"templateName\":\"selfnoderemediationtemplate-sample\",\"timedOut\":\"2025-12-30T19:03:37Z\"},{\"resource\":{\"apiVersion\":\"fence-agents-remediation.medik8s.io/v1alpha1\",\"kind\":\"FenceAgentsRemediation\",\"name\":\"worker-0-1-jqcbg\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1\"},\"started\":\"2025-12-30T19:03:37Z\",\"templateName\":\"far-template\"}]}]}}"} 2025-12-30T19:03:37.474852767Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:37.474901729Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-12-30T19:03:37.474912008Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:37.474916064Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-12-30T19:03:37.480276268Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:37.480307406Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-12-30T19:03:37.480315741Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:37.480320123Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-12-30T19:03:37.651287034Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "1m0s"} 2025-12-30T19:03:37.651367672Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:03:37.659828503Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown"} 2025-12-30T19:03:37.668471874Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:03:37.668501969Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:03:37.66850751Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:03:37Z" level=info msg="invalidating lease" time="2025-12-30T19:03:37Z" level=info msg="getting lease" 2025-12-30T19:03:37.673500386Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:03:37Z" level=info msg="invalidating lease" time="2025-12-30T19:03:37Z" level=info msg="getting lease" 2025-12-30T19:03:37.680971822Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-12-30T19:03:37.684930191Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-1-jqcbg", "kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} time="2025-12-30T19:03:37Z" level=info msg="getting lease" 2025-12-30T19:03:37.685021264Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-1", "NHC is lease owner": true, "lease expiration time": "1m0s"} 2025-12-30T19:03:37.685050567Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:03:37.769944058Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "59.314953802s"} 2025-12-30T19:03:38.328237792Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:03:38.336790603Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown"} 2025-12-30T19:03:38.344483088Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:03:38.344542057Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:03:38.34455898Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:03:38Z" level=info msg="invalidating lease" time="2025-12-30T19:03:38Z" level=info msg="getting lease" 2025-12-30T19:03:38.349291678Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:03:38Z" level=info msg="invalidating lease" time="2025-12-30T19:03:38Z" level=info msg="getting lease" 2025-12-30T19:03:38.35381999Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-12-30T19:03:38.35728656Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-1-jqcbg", "kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} time="2025-12-30T19:03:38Z" level=info msg="getting lease" 2025-12-30T19:03:38.357356503Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-1", "NHC is lease owner": true, "lease expiration time": "1m0s"} 2025-12-30T19:03:38.357385761Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:03:38.362440061Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "58.642619134s"} 2025-12-30T19:03:38.89791941Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:38.898032419Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-12-30T19:03:38.898063446Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:38.898078356Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-12-30T19:03:38.898145485Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:03:38.906313358Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown"} 2025-12-30T19:03:38.915175269Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:03:38.915255978Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:38Z"} 2025-12-30T19:03:38.915272507Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:03:38Z" level=info msg="invalidating lease" time="2025-12-30T19:03:38Z" level=info msg="getting lease" 2025-12-30T19:03:38.920124284Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:03:38Z" level=info msg="invalidating lease" time="2025-12-30T19:03:38Z" level=info msg="getting lease" 2025-12-30T19:03:38.925130863Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-12-30T19:03:38.929014089Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-1-jqcbg", "kind": "FenceAgentsRemediation", "namespace": "openshift-workload-availability"} time="2025-12-30T19:03:38Z" level=info msg="getting lease" 2025-12-30T19:03:38.929066154Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-1", "NHC is lease owner": true, "lease expiration time": "1m0s"} 2025-12-30T19:03:38.929092567Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:38Z"} 2025-12-30T19:03:38.929103355Z INFO controllers.NodeHealthCheck remediation failed {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:03:38.92910819Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:38Z"} 2025-12-30T19:03:38.936016869Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:38.936076724Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-12-30T19:03:38.936097816Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:38.936111262Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-12-30T19:03:39.017474332Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-1","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-nb6bc","uid":"d8994bd3-82f8-41e5-b57e-02476ef456ce","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-12-30T19:02:37Z","timedOut":"2025-12-30T19:03:37Z","templateName":"selfnoderemediationtemplate-sample"},{"resource":{"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-jqcbg","uid":"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1"},"started":"2025-12-30T19:03:37Z","timedOut":"2025-12-30T19:03:38Z","templateName":"far-template"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-12-30T18:45:45Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-12-30T19:03:37Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"name\":\"worker-0-1\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-1-nb6bc\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"d8994bd3-82f8-41e5-b57e-02476ef456ce\"},\"started\":\"2025-12-30T19:02:37Z\",\"templateName\":\"selfnoderemediationtemplate-sample\",\"timedOut\":\"2025-12-30T19:03:37Z\"},{\"resource\":{\"apiVersion\":\"fence-agents-remediation.medik8s.io/v1alpha1\",\"kind\":\"FenceAgentsRemediation\",\"name\":\"worker-0-1-jqcbg\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1\"},\"started\":\"2025-12-30T19:03:37Z\",\"templateName\":\"far-template\",\"timedOut\":\"2025-12-30T19:03:38Z\"}]}]}}"} 2025-12-30T19:03:39.224108191Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "1s"} 2025-12-30T19:03:39.224215747Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:03:39.233074405Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown"} 2025-12-30T19:03:39.242579318Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:03:39.242614802Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:38Z"} 2025-12-30T19:03:39.242620209Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:03:39Z" level=info msg="invalidating lease" time="2025-12-30T19:03:39Z" level=info msg="getting lease" 2025-12-30T19:03:39.342970304Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:03:39Z" level=info msg="invalidating lease" time="2025-12-30T19:03:39Z" level=info msg="getting lease" 2025-12-30T19:03:39.348007376Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-12-30T19:03:39.351612776Z INFO controllers.NodeHealthCheck.resource manager Attempting to obtain Node Lease {"Node name": "worker-0-1"} time="2025-12-30T19:03:39Z" level=info msg="request lease" time="2025-12-30T19:03:39Z" level=info msg="getting lease" time="2025-12-30T19:03:39Z" level=info msg="renew lease owned by NodeHealthCheck-nhc-mdr-snr setAcquireTime=false" 2025-12-30T19:03:39.354523344Z INFO controllers.NodeHealthCheck.resource manager Creating a remediation CR {"CR name": "worker-0-1", "CR kind": "MachineDeletionRemediation", "namespace": "openshift-workload-availability"} 2025-12-30T19:03:39.357621351Z DEBUG events [remediation] Created remediation object for node worker-0-1 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"3623030"}, "reason": "RemediationCreated"} 2025-12-30T19:03:39.362728752Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:39.362779967Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-12-30T19:03:39.362790048Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:39.362794297Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-12-30T19:03:39.443810311Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"worker-0-1","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-nb6bc","uid":"d8994bd3-82f8-41e5-b57e-02476ef456ce","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-12-30T19:02:37Z","timedOut":"2025-12-30T19:03:37Z","templateName":"selfnoderemediationtemplate-sample"},{"resource":{"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-jqcbg","uid":"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1"},"started":"2025-12-30T19:03:37Z","timedOut":"2025-12-30T19:03:38Z","templateName":"far-template"},{"resource":{"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-1","uid":"98c1158b-3290-42f3-98e5-5b13a91da5a1","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1"},"started":"2025-12-30T19:03:39Z","templateName":"mdr-template"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-12-30T18:45:45Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-12-30T19:03:39Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"name\":\"worker-0-1\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"worker-0-1-nb6bc\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"d8994bd3-82f8-41e5-b57e-02476ef456ce\"},\"started\":\"2025-12-30T19:02:37Z\",\"templateName\":\"selfnoderemediationtemplate-sample\",\"timedOut\":\"2025-12-30T19:03:37Z\"},{\"resource\":{\"apiVersion\":\"fence-agents-remediation.medik8s.io/v1alpha1\",\"kind\":\"FenceAgentsRemediation\",\"name\":\"worker-0-1-jqcbg\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1\"},\"started\":\"2025-12-30T19:03:37Z\",\"templateName\":\"far-template\",\"timedOut\":\"2025-12-30T19:03:38Z\"},{\"resource\":{\"apiVersion\":\"machine-deletion-remediation.medik8s.io/v1alpha1\",\"kind\":\"MachineDeletionRemediation\",\"name\":\"worker-0-1\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"98c1158b-3290-42f3-98e5-5b13a91da5a1\"},\"started\":\"2025-12-30T19:03:39Z\",\"templateName\":\"mdr-template\"}]}]}}"} 2025-12-30T19:03:39.574245935Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:39.574279918Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-12-30T19:03:39.574289105Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:39.574293131Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-12-30T19:03:39.578830787Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:39.578860529Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-12-30T19:03:39.578869575Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:03:39.578874121Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1", "Remediation CR Kind": "MachineDeletionRemediation"} 2025-12-30T19:03:39.650890953Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-12-30T19:03:39.65101248Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:03:39.66779795Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown"} 2025-12-30T19:03:39.676927813Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:03:39.676960149Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:38Z"} 2025-12-30T19:03:39.676969286Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:39Z"} 2025-12-30T19:03:39.676974869Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:39Z"} 2025-12-30T19:03:39.676979306Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:03:39Z" level=info msg="invalidating lease" time="2025-12-30T19:03:39Z" level=info msg="getting lease" 2025-12-30T19:03:39.682614162Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:03:39Z" level=info msg="invalidating lease" time="2025-12-30T19:03:39Z" level=info msg="getting lease" 2025-12-30T19:03:39.688012935Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-12-30T19:03:39.691567113Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-1", "kind": "MachineDeletionRemediation", "namespace": "openshift-workload-availability"} time="2025-12-30T19:03:39Z" level=info msg="getting lease" 2025-12-30T19:03:39.691634819Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-1", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2025-12-30T19:03:39.691664291Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:39Z"} 2025-12-30T19:03:39.698146761Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "9m59.308341331s"} 2025-12-30T19:03:40.224591624Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:03:40.232339961Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "worker-0-1", "condition type": "Ready", "condition status": "Unknown"} 2025-12-30T19:03:40.240315031Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:03:40.240336621Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:38Z"} 2025-12-30T19:03:40.240344703Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:39Z"} 2025-12-30T19:03:40.240349389Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:39Z"} 2025-12-30T19:03:40.24035408Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:03:40Z" level=info msg="invalidating lease" time="2025-12-30T19:03:40Z" level=info msg="getting lease" 2025-12-30T19:03:40.245347476Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:03:40Z" level=info msg="invalidating lease" time="2025-12-30T19:03:40Z" level=info msg="getting lease" 2025-12-30T19:03:40.250658495Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-12-30T19:03:40.2540089Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "worker-0-1", "kind": "MachineDeletionRemediation", "namespace": "openshift-workload-availability"} time="2025-12-30T19:03:40Z" level=info msg="getting lease" 2025-12-30T19:03:40.254062206Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-mdr-snr", "lease name": "node-worker-0-1", "NHC is lease owner": true, "lease expiration time": "10m0s"} 2025-12-30T19:03:40.25408252Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:39Z"} 2025-12-30T19:03:40.259439116Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "9m58.745922171s"} 2025-12-30T19:09:35.057284665Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-1", "NHC": "nhc-mdr-snr"} 2025-12-30T19:09:35.057393107Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:09:35.086003025Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:09:35.086045008Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:38Z"} 2025-12-30T19:09:35.086053056Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:39Z"} 2025-12-30T19:09:35.08605779Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:39Z"} 2025-12-30T19:09:35.08606179Z INFO controllers.NodeHealthCheck Going to delete orphaned remediation CRs {"NodeHealthCheck name": "nhc-mdr-snr", "count": 3} time="2025-12-30T19:09:35Z" level=info msg="invalidating lease" time="2025-12-30T19:09:35Z" level=info msg="getting lease" time="2025-12-30T19:09:35Z" level=error msg="leases.coordination.k8s.io \"node-worker-0-1\" not foundfailed to delete lease to be invalidated" 2025-12-30T19:09:35.09752309Z ERROR controllers.NodeHealthCheck.nhc lease manager failed to invalidate lease {"NodeHealthCheck name": "nhc-mdr-snr", "node name": "worker-0-1", "error": "leases.coordination.k8s.io \"node-worker-0-1\" not found"} github.com/medik8s/node-healthcheck-operator/controllers/resources.(*nhcLeaseManager).InvalidateLease /app/node-healthcheck-operator/controllers/resources/lease.go:152 github.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).CleanUp /app/node-healthcheck-operator/controllers/resources/manager.go:464 github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).deleteOrphanedRemediationCRs /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:500 github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:268 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T19:09:35.097589554Z ERROR controllers.NodeHealthCheck failed to clean up orphaned node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1", "error": "leases.coordination.k8s.io \"node-worker-0-1\" not found"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).deleteOrphanedRemediationCRs /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:501 github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:268 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T19:09:35.098050375Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":0,"healthyNodes":0,"unhealthyNodes":[{"name":"worker-0-1","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-nb6bc","uid":"d8994bd3-82f8-41e5-b57e-02476ef456ce","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-12-30T19:02:37Z","timedOut":"2025-12-30T19:03:37Z","templateName":"selfnoderemediationtemplate-sample"},{"resource":{"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-jqcbg","uid":"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1"},"started":"2025-12-30T19:03:37Z","timedOut":"2025-12-30T19:03:38Z","templateName":"far-template"},{"resource":{"kind":"MachineDeletionRemediation","namespace":"openshift-workload-availability","name":"worker-0-1","uid":"98c1158b-3290-42f3-98e5-5b13a91da5a1","apiVersion":"machine-deletion-remediation.medik8s.io/v1alpha1"},"started":"2025-12-30T19:03:39Z","templateName":"mdr-template"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-12-30T18:45:45Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-12-30T19:03:39Z"}, "patch": "{\"status\":{\"healthyNodes\":0,\"observedNodes\":0}}"} 2025-12-30T19:09:35.306618513Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": "leases.coordination.k8s.io \"node-worker-0-1\" not found", "errorCauses": [{"error": "leases.coordination.k8s.io \"node-worker-0-1\" not found"}], "requeue": false, "requeuAfter": "0s"} 2025-12-30T19:09:35.306675186Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-mdr-snr"}, "namespace": "", "name": "nhc-mdr-snr", "reconcileID": "773b8e8d-1c9e-4d0f-82ca-166ffe8c355d", "error": "leases.coordination.k8s.io \"node-worker-0-1\" not found", "errorCauses": [{"error": "leases.coordination.k8s.io \"node-worker-0-1\" not found"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-12-30T19:09:35.311870429Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:09:35.328959274Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:37Z"} 2025-12-30T19:09:35.329038932Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:38Z"} 2025-12-30T19:09:35.329067051Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "PermanentNodeDeletionExpected", "status": "False", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:39Z"} 2025-12-30T19:09:35.329082848Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-mdr-snr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-12-30T19:03:39Z"} 2025-12-30T19:09:35.329095654Z INFO controllers.NodeHealthCheck Going to delete orphaned remediation CRs {"NodeHealthCheck name": "nhc-mdr-snr", "count": 3} time="2025-12-30T19:09:35Z" level=info msg="invalidating lease" time="2025-12-30T19:09:35Z" level=info msg="getting lease" 2025-12-30T19:09:35.335672736Z INFO controllers.NodeHealthCheck deleted orphaned remediation CR {"NodeHealthCheck name": "nhc-mdr-snr", "name": "worker-0-1-nb6bc", "for deleted node": "worker-0-1"} time="2025-12-30T19:09:35Z" level=info msg="invalidating lease" time="2025-12-30T19:09:35Z" level=info msg="getting lease" 2025-12-30T19:09:35.335728427Z DEBUG events [remediation] Deleted remediation CR of kind SelfNodeRemediation with name worker-0-1-nb6bc {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"3625295"}, "reason": "RemediationRemoved"} 2025-12-30T19:09:35.336011192Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:09:35.336055713Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:09:35.336067101Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:09:35.336071907Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:09:35.345033201Z INFO controllers.NodeHealthCheck deleted orphaned remediation CR {"NodeHealthCheck name": "nhc-mdr-snr", "name": "worker-0-1-jqcbg", "for deleted node": "worker-0-1"} time="2025-12-30T19:09:35Z" level=info msg="invalidating lease" time="2025-12-30T19:09:35Z" level=info msg="getting lease" 2025-12-30T19:09:35.345120404Z DEBUG events [remediation] Deleted remediation CR of kind FenceAgentsRemediation with name worker-0-1-jqcbg {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"3625295"}, "reason": "RemediationRemoved"} 2025-12-30T19:09:35.345378425Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:09:35.345428266Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-12-30T19:09:35.345438997Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:09:35.345443217Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-12-30T19:09:35.347977406Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:09:35.348005865Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-nb6bc", "Remediation CR Kind": "SelfNodeRemediation"} 2025-12-30T19:09:35.35222374Z INFO controllers.NodeHealthCheck deleted orphaned remediation CR {"NodeHealthCheck name": "nhc-mdr-snr", "name": "worker-0-1", "for deleted node": "worker-0-1"} 2025-12-30T19:09:35.352241293Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} 2025-12-30T19:09:35.352300038Z DEBUG events [remediation] Deleted remediation CR of kind MachineDeletionRemediation with name worker-0-1 {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"3625295"}, "reason": "RemediationRemoved"} 2025-12-30T19:09:35.352492871Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:09:35.352541235Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1", "Remediation CR Kind": "MachineDeletionRemediation"} time="2025-12-30T19:09:35Z" level=info msg="invalidating lease" time="2025-12-30T19:09:35Z" level=info msg="getting lease" 2025-12-30T19:09:35.430666365Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:09:35Z" level=info msg="invalidating lease" time="2025-12-30T19:09:35Z" level=info msg="getting lease" 2025-12-30T19:09:35.437400158Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":2,"healthyNodes":2,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-12-30T18:45:45Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-12-30T19:09:35Z"}, "patch": "{\"status\":{\"healthyNodes\":2,\"observedNodes\":2,\"phase\":\"Enabled\",\"reason\":\"NHC is enabled, no ongoing remediation\",\"unhealthyNodes\":null}}"} 2025-12-30T19:09:35.643278582Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-12-30T19:09:35.643383939Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:09:35.663529688Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:09:35Z" level=info msg="invalidating lease" time="2025-12-30T19:09:35Z" level=info msg="getting lease" 2025-12-30T19:09:35.669800145Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:09:35Z" level=info msg="invalidating lease" time="2025-12-30T19:09:35Z" level=info msg="getting lease" 2025-12-30T19:09:35.674740121Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-12-30T19:23:31.891532846Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-1", "NHC": "nhc-mdr-snr"} 2025-12-30T19:23:31.891625091Z INFO adding NHC to reconcile queue for handling node {"node": "worker-0-1", "NHC": "nhc-mdr-snr"} 2025-12-30T19:23:31.891696984Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:23:31.920441637Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} 2025-12-30T19:23:32.02092193Z INFO controllers.NodeHealthCheck.resource manager setting a delay for node getting healthy {"node name": "worker-0-1", "delay in seconds": 5} 2025-12-30T19:23:32.032555688Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} 2025-12-30T19:23:32.03285345Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:23:32.032896407Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-12-30T19:23:32.032905722Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:23:32.032910355Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} 2025-12-30T19:23:32.045032588Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-mdr-snr","uid":"054d5047-6762-4082-b08d-a52405f32d1e","controller":false}} 2025-12-30T19:23:32.045065643Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-mdr-snr", "Remediation CR Name": "worker-0-1-jqcbg", "Remediation CR Kind": "FenceAgentsRemediation"} time="2025-12-30T19:23:32Z" level=info msg="invalidating lease" time="2025-12-30T19:23:32Z" level=info msg="getting lease" 2025-12-30T19:23:32.121865837Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:23:32Z" level=info msg="invalidating lease" time="2025-12-30T19:23:32Z" level=info msg="getting lease" 2025-12-30T19:23:32.127869244Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":2,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-12-30T18:45:45Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-12-30T19:09:35Z"}, "patch": "{\"status\":{\"observedNodes\":3}}"} 2025-12-30T19:23:32.333698317Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "6s"} 2025-12-30T19:23:32.333790546Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:23:32.349171379Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:23:32Z" level=info msg="invalidating lease" time="2025-12-30T19:23:32Z" level=info msg="getting lease" 2025-12-30T19:23:32.352919721Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-12-30T19:23:32Z" level=info msg="invalidating lease" time="2025-12-30T19:23:32Z" level=info msg="getting lease" 2025-12-30T19:23:32.356957354Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:23:32Z" level=info msg="invalidating lease" time="2025-12-30T19:23:32Z" level=info msg="getting lease" 2025-12-30T19:23:32.361702718Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-mdr-snr", "new status": {"observedNodes":3,"healthyNodes":3,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-12-30T18:45:45Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-12-30T19:23:32Z"}, "patch": "{\"status\":{\"healthyNodes\":3}}"} 2025-12-30T19:23:32.56741178Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-12-30T19:23:38.334200596Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-mdr-snr"} 2025-12-30T19:23:38.353585485Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-0"} time="2025-12-30T19:23:38Z" level=info msg="invalidating lease" time="2025-12-30T19:23:38Z" level=info msg="getting lease" 2025-12-30T19:23:38.358259968Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-1"} time="2025-12-30T19:23:38Z" level=info msg="invalidating lease" time="2025-12-30T19:23:38Z" level=info msg="getting lease" 2025-12-30T19:23:38.362732107Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-mdr-snr", "node": "worker-0-2"} time="2025-12-30T19:23:38Z" level=info msg="invalidating lease" time="2025-12-30T19:23:38Z" level=info msg="getting lease" 2025-12-30T19:23:38.367931368Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-mdr-snr", "error": null, "requeue": false, "requeuAfter": "0s"} SNR Logs: ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc logs pod/self-node-remediation-controller-manager-57c976f744-lxjrv Defaulted container "manager" out of: manager, kube-rbac-proxy 2025-12-30T13:55:39.863229211Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-12-30T13:55:39.863326951Z INFO setup Go OS/Arch: linux/amd64 2025-12-30T13:55:39.863329833Z INFO setup Operator Version: d80706e 2025-12-30T13:55:39.863331697Z INFO setup Git Commit: d80706e1c305b9b65e00dee1b7cb653dcb1705e7 2025-12-30T13:55:39.863333779Z INFO setup Build Date: 2025-11-03T07:49:14+00:00 2025-12-30T13:55:39.86333559Z INFO setup HTTP/2 for metrics and webhook server disabled 2025-12-30T13:55:39.870113204Z INFO utils-taints out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 34} 2025-12-30T13:55:39.870135592Z INFO utils-taints out of service taint strategy {"isGA": true, "k8sMajorVersion": 1, "k8sMinorVersion": 34} 2025-12-30T13:55:39.870142031Z INFO setup Starting as a manager that installs the daemonset 2025-12-30T13:55:39.870149521Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig"} 2025-12-30T13:55:39.870176012Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2025-12-30T13:55:39.870234168Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2025-12-30T13:55:39.870284605Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-12-30T13:55:39.870328035Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-12-30T13:55:39.870349858Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-12-30T13:55:39.870381135Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-12-30T13:55:39.870400493Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation"} 2025-12-30T13:55:39.870416311Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2025-12-30T13:55:39.870445695Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2025-12-30T13:55:39.870505825Z INFO setup starting manager 2025-12-30T13:55:39.870620931Z INFO controller-runtime.metrics Starting metrics server 2025-12-30T13:55:39.870707647Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": "127.0.0.1:8080", "secure": false} 2025-12-30T13:55:39.870744872Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-12-30T13:55:39.870763344Z INFO controller-runtime.webhook Starting webhook server 2025-12-30T13:55:39.870926075Z INFO controller-runtime.certwatcher Updated current TLS certificate I1230 13:55:39.870933 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/547f6cb6.medik8s.io... 2025-12-30T13:55:39.871205165Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} 2025-12-30T13:55:39.871136483Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-12-30T13:55:43.651030111Z DEBUG controller-runtime.certwatcher certificate event {"event": "CHMOD \"/apiserver.local.config/certificates/apiserver.crt\""} 2025-12-30T13:55:43.651283132Z DEBUG controller-runtime.certwatcher certificate event {"event": "CHMOD \"/apiserver.local.config/certificates/apiserver.key\""} 2025-12-30T13:55:56.847148758Z INFO selfnoderemediationconfig-resource validate create {"name": "self-node-remediation-config"} 2025-12-30T13:56:06.844139621Z INFO selfnoderemediationtemplate-resource default {"name": "self-node-remediation-automatic-strategy-template"} E1230 18:57:35.112458 1 leaderelection.go:448] error retrieving resource lock openshift-workload-availability/547f6cb6.medik8s.io: Get "https://172.30.0.1:443/apis/coordination.k8s.io/v1/namespaces/openshift-workload-availability/leases/547f6cb6.medik8s.io?timeout=5s": context deadline exceeded I1230 18:57:55.121912 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/547f6cb6.medik8s.io 2025-12-30T18:57:55.121955892Z DEBUG events self-node-remediation-controller-manager-57c976f744-lxjrv_4ba89b6a-4694-408a-b2a7-93c38a9231ae became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"547f6cb6.medik8s.io","uid":"f1b17f5d-a2c6-4566-b046-79074e3c6bc5","apiVersion":"coordination.k8s.io/v1","resourceVersion":"3618726"}, "reason": "LeaderElection"} 2025-12-30T18:57:55.122140557Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1alpha1.SelfNodeRemediationConfig"} 2025-12-30T18:57:55.122188956Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1.DaemonSet"} 2025-12-30T18:57:55.122193294Z INFO Starting Controller {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig"} 2025-12-30T18:57:55.122196028Z INFO Starting EventSource {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "source": "kind source: *v1alpha1.SelfNodeRemediation"} 2025-12-30T18:57:55.122206835Z INFO Starting Controller {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation"} 2025-12-30T18:57:55.130655735Z INFO selfnoderemediationconfig-resource validate create {"name": "self-node-remediation-config"} 2025-12-30T18:57:55.225624894Z INFO Starting workers {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "worker count": 1} 2025-12-30T18:57:55.225710426Z INFO Starting workers {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "worker count": 1} 2025-12-30T18:57:55.225855262Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-12-30T18:57:55.328759722Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-12-30T18:57:55.328811148Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-12-30T18:57:55.329457543Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/12/30 18:57:55 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025-12-30T18:58:05.126257468Z INFO selfnoderemediationtemplate-resource default {"name": "self-node-remediation-automatic-strategy-template"} 2025-12-30T18:58:05.129495614Z INFO selfnoderemediationtemplate-resource validate create {"name": "self-node-remediation-automatic-strategy-template"} 2025/12/30 18:58:05 update was successful E1230 18:59:50.413822 1 leaderelection.go:441] Failed to update lock optimistically: Put "https://172.30.0.1:443/apis/coordination.k8s.io/v1/namespaces/openshift-workload-availability/leases/547f6cb6.medik8s.io?timeout=5s": net/http: request canceled (Client.Timeout exceeded while awaiting headers), falling back to slow path 2025-12-30T19:02:37.459341227Z INFO selfnoderemediation-resource validate create {"name": "worker-0-1-nb6bc"} 2025-12-30T19:02:37.562922872Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:02:37.562947203Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:02:37.563039637Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-nb6bc","uid":"d8994bd3-82f8-41e5-b57e-02476ef456ce","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3622221"}, "reason": "RemediationStarted"} 2025-12-30T19:02:37.665848866Z INFO selfnoderemediation-resource validate update {"name": "worker-0-1-nb6bc"} 2025-12-30T19:02:37.668101079Z INFO controllers.SelfNodeRemediation finalizer added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:02:37.668240371Z DEBUG events [remediation] Remediation process - successful adding finalizer {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-nb6bc","uid":"d8994bd3-82f8-41e5-b57e-02476ef456ce","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3622225"}, "reason": "AddFinalizer"} 2025-12-30T19:02:37.671826747Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:02:37.671837228Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:02:37.680537191Z INFO controllers.SelfNodeRemediation NoExecute taint added {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-12-30T19:02:06Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-12-30T19:02:06Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-12-30T19:02:37Z"}]} 2025-12-30T19:02:37.680589631Z INFO controllers.SelfNodeRemediation Marking node as unschedulable {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}, "node name": "worker-0-1"} 2025-12-30T19:02:37.680673643Z DEBUG events [remediation] Remediation process - NoExecute taint added to the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"worker-0-1","uid":"02b444ed-b331-485c-b376-7b1f58aa17dc","apiVersion":"v1","resourceVersion":"3622228"}, "reason": "AddNoExecute"} 2025-12-30T19:02:37.688528688Z DEBUG events [remediation] Remediation process - unhealthy node marked as unschedulable {"type": "Normal", "object": {"kind":"Node","name":"worker-0-1","uid":"02b444ed-b331-485c-b376-7b1f58aa17dc","apiVersion":"v1","resourceVersion":"3622230"}, "reason": "MarkUnschedulable"} 2025-12-30T19:02:37.703226068Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:02:37.703305437Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:02:37.703598405Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}, "node name": "worker-0-1"} 2025-12-30T19:02:37.710065469Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:02:37.710159854Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:02:37.710562077Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}, "node name": "worker-0-1"} 2025-12-30T19:02:38.703707512Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:02:38.70372606Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:02:38.703967016Z INFO rebootDurationCalculator No SafeTimeToAssumeNodeRebootedSeconds specified, using calculated minimum safe reboot time {"calculated minimum time in seconds": 120} 2025-12-30T19:02:38.703983603Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}, "node name": "worker-0-1", "time": "2025-12-30 19:04:38.703983056 +0000 UTC m=+18538.853171347"} 2025-12-30T19:02:38.704101488Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-nb6bc","uid":"d8994bd3-82f8-41e5-b57e-02476ef456ce","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3622245"}, "reason": "UpdateTimeAssumedRebooted"} 2025-12-30T19:02:38.709736147Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:02:38.709751264Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}, "node name": "worker-0-1", "time left": "2m0.290249655s"} 2025-12-30T19:03:37.038358517Z INFO selfnoderemediation-resource validate update {"name": "worker-0-1-nb6bc"} 2025-12-30T19:03:37.041358611Z INFO controllers.SelfNodeRemediation NHC added the timed-out annotation, remediation will be stopped {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:03:37.04150217Z DEBUG events [remediation] NHC added the timed-out annotation, remediation will be stopped {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-nb6bc","uid":"d8994bd3-82f8-41e5-b57e-02476ef456ce","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3622984"}, "reason": "RemediationStopped"} 2025-12-30T19:03:37.045918756Z INFO controllers.SelfNodeRemediation NHC added the timed-out annotation, remediation will be stopped {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:03:37.046043549Z DEBUG events [remediation] NHC added the timed-out annotation, remediation will be stopped {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-nb6bc","uid":"d8994bd3-82f8-41e5-b57e-02476ef456ce","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3622986"}, "reason": "RemediationStopped"} 2025-12-30T19:04:39.004735705Z INFO controllers.SelfNodeRemediation NHC added the timed-out annotation, remediation will be stopped {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:04:39.004966202Z DEBUG events [remediation] NHC added the timed-out annotation, remediation will be stopped {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-nb6bc","uid":"d8994bd3-82f8-41e5-b57e-02476ef456ce","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"3622986"}, "reason": "RemediationStopped"} 2025-12-30T19:09:35.336305851Z INFO controllers.SelfNodeRemediation Cleaning up a timed-out remediation which is deleted by NHC {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}, "remediation name": "worker-0-1-nb6bc"} 2025-12-30T19:09:35.347838875Z INFO controllers.SelfNodeRemediation finalizer removed {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:09:35.350189431Z ERROR controllers.SelfNodeRemediation failed to update snr status {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}, "error": "selfnoderemediations.self-node-remediation.medik8s.io \"worker-0-1-nb6bc\" not found"} github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).updateSnrStatus /app/self-node-remediation/controllers/selfnoderemediation_controller.go:699 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager.func1 /app/self-node-remediation/controllers/selfnoderemediation_controller.go:217 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).ReconcileManager /app/self-node-remediation/controllers/selfnoderemediation_controller.go:261 github.com/medik8s/self-node-remediation/controllers.(*SelfNodeRemediationReconciler).Reconcile /app/self-node-remediation/controllers/selfnoderemediation_controller.go:159 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:116 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:303 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:224 2025-12-30T19:09:35.35026409Z ERROR Reconciler error {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "SelfNodeRemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}, "namespace": "openshift-workload-availability", "name": "worker-0-1-nb6bc", "reconcileID": "6759c39d-2497-404a-9f59-d4f23bb86523", "error": "selfnoderemediations.self-node-remediation.medik8s.io \"worker-0-1-nb6bc\" not found", "errorCauses": [{"error": "selfnoderemediations.self-node-remediation.medik8s.io \"worker-0-1-nb6bc\" not found"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/self-node-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:224 2025-12-30T19:09:35.35031098Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} 2025-12-30T19:09:35.355542693Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"worker-0-1-nb6bc","namespace":"openshift-workload-availability"}} Far Logs: ========================================================================================================================================================== [root@cert-rhosp-02 ~]# oc logs pod/fence-agents-remediation-controller-manager-74bfb5d6bd-b4fkz 2025-12-30T18:57:39.977885035Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-12-30T18:57:39.977964567Z INFO setup Go OS/Arch: linux/amd64 2025-12-30T18:57:39.977968085Z INFO setup Operator Version: bd73055e 2025-12-30T18:57:39.977970284Z INFO setup Git Commit: bd73055ef2c68bfdc865d2c54179f4448bd454da 2025-12-30T18:57:39.977975393Z INFO setup Build Date: 2025-09-08T09:09:10+00:00 2025-12-30T18:57:39.977990363Z INFO setup HTTP/2 for webhooks disabled 2025-12-30T18:57:39.984447339Z INFO validation out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 34} 2025-12-30T18:57:39.984469846Z INFO setup out-of-service taint is supported on this cluster 2025-12-30T18:57:39.984506419Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation"} 2025-12-30T18:57:39.984534811Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediation", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2025-12-30T18:57:39.984594092Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediation"} 2025-12-30T18:57:39.98463252Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-12-30T18:57:39.984662473Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-12-30T18:57:39.98468653Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "fence-agents-remediation.medik8s.io/v1alpha1, Kind=FenceAgentsRemediationTemplate", "path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-12-30T18:57:39.984716474Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-fence-agents-remediation-medik8s-io-v1alpha1-fenceagentsremediationtemplate"} 2025-12-30T18:57:39.984734015Z INFO setup starting manager 2025-12-30T18:57:39.984833458Z INFO controller-runtime.metrics Starting metrics server 2025-12-30T18:57:39.984883047Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-12-30T18:57:39.984913184Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2025-12-30T18:57:39.984929085Z INFO controller-runtime.webhook Starting webhook server I1230 18:57:39.985018 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/cb305759.medik8s.io... 2025-12-30T18:57:39.985100535Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-12-30T18:57:39.985154678Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-12-30T18:57:39.985205406Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} I1230 18:57:57.604137 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/cb305759.medik8s.io 2025-12-30T18:57:57.604400186Z INFO Starting EventSource {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation", "source": "kind source: *v1alpha1.FenceAgentsRemediation"} 2025-12-30T18:57:57.604422368Z INFO Starting Controller {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation"} 2025-12-30T18:57:57.60435834Z DEBUG events fence-agents-remediation-controller-manager-74bfb5d6bd-b4fkz_b28f0477-db61-43ed-833f-1a4baf0bd9e0 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"cb305759.medik8s.io","uid":"e609d66a-d089-4051-8bed-58a90e97fb4a","apiVersion":"coordination.k8s.io/v1","resourceVersion":"3618742"}, "reason": "LeaderElection"} 2025-12-30T18:57:57.706547487Z INFO Starting workers {"controller": "fenceagentsremediation", "controllerGroup": "fence-agents-remediation.medik8s.io", "controllerKind": "FenceAgentsRemediation", "worker count": 1} 2025-12-30T19:03:37.364935943Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-12-30T19:03:37.364978333Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-12-30T19:03:37.475055864Z INFO controllers.FenceAgentsRemediation Finalizer was added {"CR Name": "worker-0-1-jqcbg"} 2025-12-30T19:03:37.475091729Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "True", "fenceAgentActionSucceededConditionStatus": "Unknown", "succeededConditionStatus": "Unknown", "reason": "RemediationStarted", "LastUpdateTime": "2025-12-30T19:03:37.475090208Z"} 2025-12-30T19:03:37.475145255Z DEBUG events [remediation] Remediation started {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-jqcbg","uid":"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3622996"}, "reason": "RemediationStarted"} 2025-12-30T19:03:37.475196947Z DEBUG events [remediation] Finalizer was added {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-jqcbg","uid":"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3622996"}, "reason": "AddFinalizer"} 2025-12-30T19:03:37.680982841Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-12-30T19:03:37.681101433Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-12-30T19:03:37.681114567Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-12-30T19:03:37.688538322Z INFO taints Taint was added {"taint effect": "NoExecute", "taint list": [{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-12-30T19:02:06Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-12-30T19:02:06Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-12-30T19:02:37Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-12-30T19:02:37Z"},{"key":"medik8s.io/fence-agents-remediation","effect":"NoExecute","timeAdded":"2025-12-30T19:03:37Z"}]} 2025-12-30T19:03:37.688684404Z INFO controllers.FenceAgentsRemediation FAR remediation taint was added {"Node Name": "worker-0-1"} 2025-12-30T19:03:37.688728037Z INFO controllers.FenceAgentsRemediation Build fence agent command line {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-1"} 2025-12-30T19:03:37.68883011Z DEBUG events [remediation] Remediation taint was added {"type": "Normal", "object": {"kind":"Node","name":"worker-0-1","uid":"02b444ed-b331-485c-b376-7b1f58aa17dc","apiVersion":"v1","resourceVersion":"3622252"}, "reason": "AddRemediationTaint"} 2025-12-30T19:03:37.889337525Z INFO controllers.FenceAgentsRemediation Execute the fence agent {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-1", "FAR uid": "a7f5adfd-8c3c-4a0a-9af7-b56a972647d1", "Parameters": ["--ip","--ipport","--lanplus","--password","--username","--action"]} 2025-12-30T19:03:37.889504055Z INFO executer fence agent start {"uid": "a7f5adfd-8c3c-4a0a-9af7-b56a972647d1", "fence_agent": "fence_ipmilan", "retryCount": 1, "retryInterval": "1s", "timeout": "1s"} 2025-12-30T19:03:37.889604458Z DEBUG events [remediation] Fence agent was executed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-jqcbg","uid":"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3622998"}, "reason": "FenceAgentExecuted"} 2025-12-30T19:03:37.894694321Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-12-30T19:03:37.894747426Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-12-30T19:03:37.894756065Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-12-30T19:03:37.89479537Z INFO controllers.FenceAgentsRemediation A Fence Agent is already running {"Fence Agent": "fence_ipmilan", "Node Name": "worker-0-1", "FAR uid": "a7f5adfd-8c3c-4a0a-9af7-b56a972647d1"} 2025-12-30T19:03:37.899495813Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-12-30T19:03:38.891602332Z INFO executer command failed {"uid": "a7f5adfd-8c3c-4a0a-9af7-b56a972647d1", "response": "", "errMessage": "", "err": "signal: killed"} 2025-12-30T19:03:38.891656128Z INFO executer fence agent done {"uid": "a7f5adfd-8c3c-4a0a-9af7-b56a972647d1", "fence_agent": "fence_ipmilan", "stdout": "", "stderr": "", "err": "signal: killed"} 2025-12-30T19:03:38.891665105Z INFO executer fence agent context timed out 2025-12-30T19:03:38.891668825Z INFO executer updating status {"FAR uid": "a7f5adfd-8c3c-4a0a-9af7-b56a972647d1"} 2025-12-30T19:03:38.891717509Z INFO executer Updating Status Condition {"processingConditionStatus": "False", "fenceAgentActionSucceededConditionStatus": "False", "succeededConditionStatus": "False", "reason": "FenceAgentFailed", "LastUpdateTime": "2025-12-30T19:03:38.891716353Z"} 2025-12-30T19:03:38.897821796Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-12-30T19:03:38.89790025Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-12-30T19:03:38.897848905Z INFO executer status updated {"FAR uid": "a7f5adfd-8c3c-4a0a-9af7-b56a972647d1"} 2025-12-30T19:03:38.903242912Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-12-30T19:03:38.936107755Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-12-30T19:03:38.936204547Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-12-30T19:03:38.936249803Z INFO executer cancelling fence agent routine {"uid": "a7f5adfd-8c3c-4a0a-9af7-b56a972647d1"} 2025-12-30T19:03:38.936265989Z INFO controllers.FenceAgentsRemediation Remediation was stopped by the Node Healthcheck Operator 2025-12-30T19:03:38.936279212Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "False", "fenceAgentActionSucceededConditionStatus": "False", "succeededConditionStatus": "False", "reason": "RemediationInterruptedByNHC", "LastUpdateTime": "2025-12-30T19:03:38Z"} 2025-12-30T19:03:38.936413217Z DEBUG events [remediation] NHC added the timed-out annotation, remediation will be stopped {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-jqcbg","uid":"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3623028"}, "reason": "RemediationStopped"} 2025-12-30T19:03:38.941572009Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-12-30T19:09:35.345644811Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-12-30T19:09:35.345690749Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-12-30T19:09:35.345701267Z ERROR controllers.FenceAgentsRemediation Could not find CR's target node {"CR's Name": "worker-0-1-jqcbg", "Expected node name": "worker-0-1"} github.com/medik8s/fence-agents-remediation/controllers.(*FenceAgentsRemediationReconciler).Reconcile /app/fence-agents-remediation/controllers/fenceagentsremediation_controller.go:138 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/fence-agents-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:116 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/fence-agents-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:303 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/fence-agents-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/fence-agents-remediation/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:224 2025-12-30T19:09:35.345722445Z INFO controllers.FenceAgentsRemediation Updating Status Condition {"processingConditionStatus": "False", "fenceAgentActionSucceededConditionStatus": "False", "succeededConditionStatus": "False", "reason": "RemediationFinishedNodeNotFound", "LastUpdateTime": "2025-12-30T19:03:38Z"} 2025-12-30T19:09:35.345866582Z DEBUG events [remediation] CR name doesn't match a node name {"type": "Warning", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-jqcbg","uid":"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3625304"}, "reason": "NodeNotFound"} 2025-12-30T19:09:35.352066779Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-12-30T19:23:32.03311457Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-12-30T19:23:32.033189266Z INFO controllers.FenceAgentsRemediation Check FAR CR's name 2025-12-30T19:23:32.033221166Z INFO controllers.FenceAgentsRemediation Cleaning up a timed-out remediation which is deleted by NHC {"remediation name": "worker-0-1-jqcbg"} 2025-12-30T19:23:32.033248519Z INFO controllers.FenceAgentsRemediation FAR remediation taint was removed {"Node Name": "worker-0-1"} 2025-12-30T19:23:32.033385039Z DEBUG events [remediation] Remediation taint was removed {"type": "Normal", "object": {"kind":"Node","name":"worker-0-1","uid":"b2c72e8b-3b1a-4d3e-a6aa-8c4c8b26c2c1","apiVersion":"v1","resourceVersion":"3631607"}, "reason": "RemoveRemediationTaint"} 2025-12-30T19:23:32.044548331Z INFO controllers.FenceAgentsRemediation Finalizer was removed {"CR Name": "worker-0-1-jqcbg"} 2025-12-30T19:23:32.044585701Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile 2025-12-30T19:23:32.044608156Z DEBUG events [remediation] Finalizer was removed {"type": "Normal", "object": {"kind":"FenceAgentsRemediation","namespace":"openshift-workload-availability","name":"worker-0-1-jqcbg","uid":"a7f5adfd-8c3c-4a0a-9af7-b56a972647d1","apiVersion":"fence-agents-remediation.medik8s.io/v1alpha1","resourceVersion":"3631612"}, "reason": "RemoveFinalizer"} 2025-12-30T19:23:32.045005102Z INFO controllers.FenceAgentsRemediation Begin FenceAgentsRemediation Reconcile 2025-12-30T19:23:32.045032657Z INFO controllers.FenceAgentsRemediation FenceAgentsRemediation CR was not found {"CR Name": "worker-0-1-jqcbg", "CR Namespace": "openshift-workload-availability"} 2025-12-30T19:23:32.045036612Z INFO controllers.FenceAgentsRemediation Finish FenceAgentsRemediation Reconcile