[root@cert-rhosp-02 ~]# export KUBECONFIG=/home/kni/clusterconfigs/auth/kubeconfig [root@cert-rhosp-02 ~]# oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.19.0-0.nightly-2025-11-16-151024 True False 32m Cluster version is 4.19.0-0.nightly-2025-11-16-151024 [root@cert-rhosp-02 ~]# oc get csv NAME DISPLAY VERSION REPLACES PHASE node-healthcheck-operator.v0.10.1 Node Health Check Operator 0.10.1 node-healthcheck-operator.v0.10.0 Installing self-node-remediation.v0.11.0 Self Node Remediation Operator 0.11.0 self-node-remediation.v0.10.2 Succeeded [root@cert-rhosp-02 ~]# oc get nodes -l 'node-role.kubernetes.io/worker' NAME STATUS ROLES AGE VERSION worker-0-0 Ready worker 70m v1.32.9 [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 114m v1.32.9 master-0-1 Ready control-plane,master 114m v1.32.9 master-0-2 Ready control-plane,master 114m v1.32.9 worker-0-0 Ready worker 70m v1.32.9 [root@cert-rhosp-02 ~]# oc get pods NAME READY STATUS RESTARTS AGE node-healthcheck-controller-manager-5cc8797bb5-5cb2w 2/2 Running 0 32m node-healthcheck-controller-manager-5cc8797bb5-f5v46 2/2 Running 0 32m node-healthcheck-node-remediation-console-plugin-5cc968499gs5w2 1/1 Running 0 33m self-node-remediation-controller-manager-7dc64b6879-ss67r 2/2 Running 1 (33m ago) 33m self-node-remediation-ds-2zwbf 1/1 Running 0 33m self-node-remediation-ds-9pbwg 1/1 Running 0 33m self-node-remediation-ds-fnl4p 1/1 Running 0 33m self-node-remediation-ds-qsnpj 1/1 Running 0 33m [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep node-healthcheck-controller-manager) [root@cert-rhosp-02 ~]# echo $PODS pod/node-healthcheck-controller-manager-5cc8797bb5-5cb2w pod/node-healthcheck-controller-manager-5cc8797bb5-f5v46 [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/node-healthcheck-controller-manager-5cc8797bb5-5cb2w "master-0-0" == pod/node-healthcheck-controller-manager-5cc8797bb5-f5v46 "master-0-2" [root@cert-rhosp-02 ~]# PODS=$(oc get pods -o name -n openshift-workload-availability | grep self-node-remediation-controller-manager) [root@cert-rhosp-02 ~]# echo $PODS pod/self-node-remediation-controller-manager-7dc64b6879-ss67r [root@cert-rhosp-02 ~]# for p in $PODS; do > echo "== $p" > oc get "$p" -n openshift-workload-availability -o json | jq .spec.nodeName > done == pod/self-node-remediation-controller-manager-7dc64b6879-ss67r "worker-0-0" [root@cert-rhosp-02 ~]# oc get snr No resources found in openshift-workload-availability namespace. [root@cert-rhosp-02 ~]# oc debug node/master-0-1 -- chroot /host bash -c "uptime -s && date" Temporary namespace openshift-debug-4kmqz is created for debugging node... Starting pod/master-0-1-debug-7cfzc ... To use host binaries, run `chroot /host` 2025-11-17 06:48:51 Mon Nov 17 09:28:49 UTC 2025 Removing debug pod ... Temporary namespace openshift-debug-4kmqz was removed. [root@cert-rhosp-02 ~]# oc debug node/master-0-1 -- chroot /host bash -c "systemctl stop kubelet" Temporary namespace openshift-debug-xmjwq is created for debugging node... Starting pod/master-0-1-debug-8rhdt ... To use host binaries, run `chroot /host` [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 160m v1.32.9 master-0-1 NotReady control-plane,master 160m v1.32.9 master-0-2 Ready control-plane,master 160m v1.32.9 worker-0-0 Ready worker 116m v1.32.9 [root@cert-rhosp-02 ~]# vi test.yaml [root@cert-rhosp-02 ~]# cat test.yaml --- apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate metadata: namespace: openshift-workload-availability name: selfnoderemediationtemplate-sample spec: template: spec: remediationStrategy: Automatic --- apiVersion: remediation.medik8s.io/v1alpha1 kind: NodeHealthCheck metadata: name: nhc-with-mdr spec: maxUnhealthy: 2 healthyDelay: 5s remediationTemplate: apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediationTemplate name: selfnoderemediationtemplate-sample namespace: openshift-workload-availability selector: matchExpressions: - key: node-role.kubernetes.io/master operator: Exists unhealthyConditions: - duration: 90s status: 'False' type: Ready - duration: 90s status: 'Unknown' type: Ready [root@cert-rhosp-02 ~]# oc apply -f test.yaml selfnoderemediationtemplate.self-node-remediation.medik8s.io/selfnoderemediationtemplate-sample created nodehealthcheck.remediation.medik8s.io/nhc-with-mdr created [root@cert-rhosp-02 ~]# oc get nhc NAME AGE nhc-with-mdr 31s [root@cert-rhosp-02 ~]# oc get snrt NAME AGE self-node-remediation-automatic-strategy-template 58m selfnoderemediationtemplate-sample 34s [root@cert-rhosp-02 ~]# oc get snr -o yaml apiVersion: v1 items: - apiVersion: self-node-remediation.medik8s.io/v1alpha1 kind: SelfNodeRemediation metadata: annotations: remediation.medik8s.io/node-name: master-0-1 remediation.medik8s.io/template-name: selfnoderemediationtemplate-sample creationTimestamp: "2025-11-17T09:42:56Z" finalizers: - self-node-remediation.medik8s.io/snr-finalizer generateName: master-0-1- generation: 1 labels: app.kubernetes.io/part-of: node-healthcheck-controller remediation.medik8s.io/isControlPlaneNode: "" name: master-0-1-pq8fp namespace: openshift-workload-availability ownerReferences: - apiVersion: remediation.medik8s.io/v1alpha1 controller: false kind: NodeHealthCheck name: nhc-with-mdr uid: 0f4c524f-1cda-49cd-8f73-75243a3f6005 resourceVersion: "72296" uid: e6cae342-aa0b-47bb-a449-4b5218fb5600 spec: remediationStrategy: Automatic status: conditions: - lastTransitionTime: "2025-11-17T09:42:57Z" message: "" reason: RemediationStarted status: "True" type: Processing - lastTransitionTime: "2025-11-17T09:42:57Z" message: "" reason: RemediationStarted status: Unknown type: Succeeded phase: Pre-Reboot-Completed timeAssumedRebooted: "2025-11-17T09:44:58Z" kind: List metadata: resourceVersion: "" [root@cert-rhosp-02 ~]# oc get snr NAME AGE master-0-1-pq8fp 42s [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 173m v1.32.9 master-0-1 NotReady,SchedulingDisabled control-plane,master 173m v1.32.9 master-0-2 Ready control-plane,master 173m v1.32.9 worker-0-0 Ready worker 129m v1.32.9 [root@cert-rhosp-02 ~]# oc get nodes NAME STATUS ROLES AGE VERSION master-0-0 Ready control-plane,master 175m v1.32.9 master-0-1 Ready control-plane,master 175m v1.32.9 master-0-2 Ready control-plane,master 175m v1.32.9 worker-0-0 Ready worker 131m v1.32.9 [root@cert-rhosp-02 ~]# oc get snr No resources found in openshift-workload-availability namespace. NHC Logs: 2025-11-17T08:42:35.883365734Z INFO setup Go Version: go1.24.4 (Red Hat 1.24.4-2.el9) X:strictfipsruntime 2025-11-17T08:42:35.883459398Z INFO setup Go OS/Arch: linux/amd64 2025-11-17T08:42:35.883462577Z INFO setup Operator Version: 2ef588d 2025-11-17T08:42:35.883464312Z INFO setup Git Commit: 2ef588d65de4087c46447401cd6e757459d79210 2025-11-17T08:42:35.883466348Z INFO setup Build Date: 2025-11-11T13:40:45+00:00 2025-11-17T08:42:35.883467935Z INFO setup HTTP/2 for metrics and webhook server disabled 2025-11-17T08:42:35.90253163Z INFO setup supported control plane topology {"topology": "HighlyAvailable"} 2025-11-17T08:42:35.902564016Z INFO setup Cluster capabilities {"IsOnOpenshift": true, "HasMachineAPI": true} 2025-11-17T08:42:35.90275451Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "remediation.medik8s.io/v1alpha1, Kind=NodeHealthCheck", "path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2025-11-17T08:42:35.902822588Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-remediation-medik8s-io-v1alpha1-nodehealthcheck"} 2025-11-17T08:42:35.902857873Z INFO setup starting manager 2025-11-17T08:42:35.902901733Z INFO controller-runtime.metrics Starting metrics server 2025-11-17T08:42:35.902975105Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-11-17T08:42:35.902986917Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": "127.0.0.1:8080", "secure": false} 2025-11-17T08:42:35.903027609Z INFO controller-runtime.webhook Starting webhook server 2025-11-17T08:42:35.903194647Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-11-17T08:42:35.903258542Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-11-17T08:42:35.903333557Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} I1117 08:42:36.003271 1 leaderelection.go:257] attempting to acquire leader lease openshift-workload-availability/e1f13584.medik8s.io... 2025-11-17T09:33:21.333794081Z INFO nodehealthcheck-resource validate update {"name": "nhc-with-mdr"} E1117 09:43:35.544009 1 leaderelection.go:436] error retrieving resource lock openshift-workload-availability/e1f13584.medik8s.io: Get "https://172.30.0.1:443/apis/coordination.k8s.io/v1/namespaces/openshift-workload-availability/leases/e1f13584.medik8s.io?timeout=5s": net/http: request canceled (Client.Timeout exceeded while awaiting headers) E1117 09:43:43.919272 1 leaderelection.go:436] error retrieving resource lock openshift-workload-availability/e1f13584.medik8s.io: Get "https://172.30.0.1:443/apis/coordination.k8s.io/v1/namespaces/openshift-workload-availability/leases/e1f13584.medik8s.io?timeout=5s": net/http: request canceled (Client.Timeout exceeded while awaiting headers) I1117 09:44:03.376786 1 leaderelection.go:271] successfully acquired lease openshift-workload-availability/e1f13584.medik8s.io 2025-11-17T09:44:03.376884348Z DEBUG events node-healthcheck-controller-manager-5cc8797bb5-5cb2w_9bf2f72e-4bb1-45da-9740-6787e48c9642 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"e1f13584.medik8s.io","uid":"2e04d23f-6f5f-4c70-8d43-050209903627","apiVersion":"coordination.k8s.io/v1","resourceVersion":"72707"}, "reason": "LeaderElection"} 2025-11-17T09:44:03.377085912Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "channel source: 0xc00072a770"} 2025-11-17T09:44:03.37709401Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1alpha1.NodeHealthCheck"} 2025-11-17T09:44:03.377108254Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *v1.Node"} 2025-11-17T09:44:03.377139648Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.MachineHealthCheck"} 2025-11-17T09:44:03.377147258Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1beta1.Machine"} 2025-11-17T09:44:03.377157682Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "kind source: *v1.Node"} 2025-11-17T09:44:03.37718143Z INFO Starting EventSource {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "source": "channel source: 0xc00072a9a0"} I1117 09:44:03.379390 1 shared_informer.go:313] Waiting for caches to sync for feature gate accessor 2025-11-17T09:44:03.47814423Z INFO MHCChecker found termination handler MHC, will ignore Nodes with Terminating condition 2025-11-17T09:44:03.478224721Z INFO MHCChecker MHC Checker status changed, notifying NHC controller 2025-11-17T09:44:03.478227648Z INFO Starting Controller {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck"} 2025-11-17T09:44:03.478251389Z INFO Starting workers {"controller": "machinehealthcheck", "controllerGroup": "machine.openshift.io", "controllerKind": "MachineHealthCheck", "worker count": 1} 2025-11-17T09:44:03.478207057Z INFO Starting Controller {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck"} 2025-11-17T09:44:03.47828384Z INFO Starting workers {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "worker count": 1} 2025-11-17T09:44:03.478343723Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:03.478429354Z INFO adding all NHCs to reconcile queue for handling MHC event I1117 09:44:03.480268 1 shared_informer.go:320] Caches are synced for feature gate accessor I1117 09:44:03.480288 1 simple_featuregate_reader.go:171] Starting feature-gate-detector 2025-11-17T09:44:03.48039807Z INFO FeatureGateAccessor FeatureGates initialized I1117 09:44:03.480447 1 recorder_logging.go:49] &Event{ObjectMeta:{dummy.1878c213954a1a26.b88366b0 dummy 0 0001-01-01 00:00:00 +0000 UTC map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Pod,Namespace:dummy,Name:dummy,UID:,APIVersion:v1,ResourceVersion:,FieldPath:,},Reason:FeatureGatesInitialized,Message:FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AWSEFSDriverVolumeMetrics", "AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BareMetalLoadBalancer", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ChunkSizeMiB", "CloudDualStackNodeIPs", "ConsolePluginContentSecurityPolicy", "DisableKubeletCloudCredentialProviders", "GCPLabelsTags", "GatewayAPI", "GatewayAPIController", "HardwareSpeed", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "MultiArchInstallAWS", "MultiArchInstallGCP", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "NewOLM", "NodeDisruptionPolicy", "OnClusterBuild", "PersistentIPsForVirtualization", "PinnedImages", "PrivateHostedZoneAWS", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "VSphereDriverConfiguration", "VSphereMultiVCenters", "ValidatingAdmissionPolicy"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AutomatedEtcdBackup", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "HighlyAvailableArbiter", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "InsightsRuntimeExtractor", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "MachineAPIProviderOpenStack", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PlatformOperators", "ProcMountType", "SELinuxChangePolicy", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StreamingCollectionEncodingToJSON", "StreamingCollectionEncodingToProtobuf", "TranslateStreamCloseWebsocketRequests", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass", "VolumeGroupSnapshot"}},Source:EventSource{Component:,Host:,},FirstTimestamp:2025-11-17 09:44:03.480377894 +0000 UTC m=+3687.616574326,LastTimestamp:2025-11-17 09:44:03.480377894 +0000 UTC m=+3687.616574326,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:,ReportingInstance:,} 2025-11-17T09:44:03.482566232Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: rpc error: code = Unavailable desc = error reading from server: read tcp 192.168.123.143:33250->192.168.123.141:2379: read: connection reset by peer", "errorVerbose": "rpc error: code = Unavailable desc = error reading from server: read tcp 192.168.123.143:33250->192.168.123.141:2379: read: connection reset by peer\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:03.482926872Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-with-mdr", "new status": {"observedNodes":0,"healthyNodes":0,"unhealthyNodes":[{"name":"master-0-1","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"master-0-1-pq8fp","uid":"e6cae342-aa0b-47bb-a449-4b5218fb5600","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-11-17T09:42:56Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-17T09:40:30Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-17T09:42:56Z"}, "patch": "{\"status\":{\"healthyNodes\":0,\"observedNodes\":0}}"} 2025-11-17T09:44:03.689134879Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: rpc error: code = Unavailable desc = error reading from server: read tcp 192.168.123.143:33250->192.168.123.141:2379: read: connection reset by peer", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: rpc error: code = Unavailable desc = error reading from server: read tcp 192.168.123.143:33250->192.168.123.141:2379: read: connection reset by peer", "errorVerbose": "rpc error: code = Unavailable desc = error reading from server: read tcp 192.168.123.143:33250->192.168.123.141:2379: read: connection reset by peer\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-11-17T09:44:03.689281377Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-with-mdr"}, "namespace": "", "name": "nhc-with-mdr", "reconcileID": "69cb2403-72ae-445f-ad89-94e9fa8b4351", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: rpc error: code = Unavailable desc = error reading from server: read tcp 192.168.123.143:33250->192.168.123.141:2379: read: connection reset by peer", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: rpc error: code = Unavailable desc = error reading from server: read tcp 192.168.123.143:33250->192.168.123.141:2379: read: connection reset by peer", "errorVerbose": "rpc error: code = Unavailable desc = error reading from server: read tcp 192.168.123.143:33250->192.168.123.141:2379: read: connection reset by peer\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:03.689373458Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:03.691463798Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:03.691769171Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-11-17T09:44:03.69183107Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-with-mdr"}, "namespace": "", "name": "nhc-with-mdr", "reconcileID": "a4027d2e-775f-4cc2-a614-35008a8d4c3f", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:03.694546764Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:03.696813354Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:03.697066045Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-11-17T09:44:03.697119537Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-with-mdr"}, "namespace": "", "name": "nhc-with-mdr", "reconcileID": "00be0da6-1a8b-4602-9f7c-9e1b79328738", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:03.717374273Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:03.719927504Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:03.720172157Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-11-17T09:44:03.720229197Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-with-mdr"}, "namespace": "", "name": "nhc-with-mdr", "reconcileID": "5783f7ca-31a6-40e8-befd-404e64fb0ab3", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:03.760489901Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:03.763047428Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:03.763268196Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-11-17T09:44:03.763319881Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-with-mdr"}, "namespace": "", "name": "nhc-with-mdr", "reconcileID": "3c5c8d08-0f0a-4ce9-bce0-05684af86b4a", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:03.791085042Z INFO console-plugin successfully created / updated console plugin resources 2025-11-17T09:44:03.844255435Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:03.846736517Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:03.846944721Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-11-17T09:44:03.847002583Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-with-mdr"}, "namespace": "", "name": "nhc-with-mdr", "reconcileID": "0dcfb2e2-393c-4ad5-8f44-d6356dc0aa5e", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:04.007150014Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:04.009431022Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:04.009672304Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-11-17T09:44:04.009722492Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-with-mdr"}, "namespace": "", "name": "nhc-with-mdr", "reconcileID": "41743186-c482-4f39-ad3a-b9175b3e7a1a", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:04.330272786Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:04.33313104Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:04.333354043Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-11-17T09:44:04.333399058Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-with-mdr"}, "namespace": "", "name": "nhc-with-mdr", "reconcileID": "650d9292-6d61-4d26-a3e1-ee6d0ac547f6", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:04.974177114Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:04.977645583Z ERROR controllers.NodeHealthCheck failed to validate template {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"} github.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile /app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:04.977856206Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}], "requeue": false, "requeuAfter": "0s"} 2025-11-17T09:44:04.977902343Z ERROR Reconciler error {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "NodeHealthCheck": {"name":"nhc-with-mdr"}, "namespace": "", "name": "nhc-with-mdr", "reconcileID": "e4553a8f-4a3b-41fa-8215-1955ed480f63", "error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorCauses": [{"error": "failed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample: selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"", "errorVerbose": "selfnoderemediationtemplates.self-node-remediation.medik8s.io \"selfnoderemediationtemplate-sample\" is forbidden: User \"system:serviceaccount:openshift-workload-availability:node-healthcheck-controller-manager\" cannot get resource \"selfnoderemediationtemplates\" in API group \"self-node-remediation.medik8s.io\" in the namespace \"openshift-workload-availability\"\nfailed to get external remediation template openshift-workload-availability/selfnoderemediationtemplate-sample\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplateWithFallbackNamespace\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:98\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).getTemplate\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:80\ngithub.com/medik8s/node-healthcheck-operator/controllers/resources.(*manager).ValidateTemplates\n\t/app/node-healthcheck-operator/controllers/resources/templates.go:120\ngithub.com/medik8s/node-healthcheck-operator/controllers.(*NodeHealthCheckReconciler).Reconcile\n\t/app/node-healthcheck-operator/controllers/nodehealthcheck_controller.go:200\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"}]} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /app/node-healthcheck-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:255 2025-11-17T09:44:06.25814732Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:06.263913779Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-11-17T09:44:06.263947751Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation template CRs {"kind": "SelfNodeRemediationTemplate"} 2025-11-17T09:44:06.263963883Z INFO Starting EventSource {"controller": "nodehealthcheck", "controllerGroup": "remediation.medik8s.io", "controllerKind": "NodeHealthCheck", "source": "kind source: *unstructured.Unstructured"} 2025-11-17T09:44:06.263969973Z INFO controllers.NodeHealthCheck.WatchManager added watch for remediation CRs {"kind": "SelfNodeRemediation"} 2025-11-17T09:44:06.264073469Z INFO controllers.NodeHealthCheck Node matches unhealthy condition {"node": "master-0-1", "condition type": "Ready", "condition status": "Unknown"} 2025-11-17T09:44:06.268390313Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-with-mdr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-17T09:42:57Z"} 2025-11-17T09:44:06.268421022Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-0"} time="2025-11-17T09:44:06Z" level=info msg="invalidating lease" time="2025-11-17T09:44:06Z" level=info msg="getting lease" 2025-11-17T09:44:06.37101285Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-2"} time="2025-11-17T09:44:06Z" level=info msg="invalidating lease" time="2025-11-17T09:44:06Z" level=info msg="getting lease" 2025-11-17T09:44:06.373519629Z INFO controllers.NodeHealthCheck handling unhealthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-1"} 2025-11-17T09:44:06.473506446Z INFO controllers.NodeHealthCheck.resource manager external remediation CR already exists {"CR name": "master-0-1-pq8fp", "kind": "SelfNodeRemediation", "namespace": "openshift-workload-availability"} time="2025-11-17T09:44:06Z" level=info msg="getting lease" 2025-11-17T09:44:06.473584005Z INFO controllers.NodeHealthCheck.nhc lease manager managing lease - about to try to acquire/extended the lease {"NodeHealthCheck name": "nhc-with-mdr", "lease name": "node-master-0-1", "NHC is lease owner": true, "lease expiration time": "10m0s"} time="2025-11-17T09:44:06Z" level=info msg="request lease" time="2025-11-17T09:44:06Z" level=info msg="getting lease" time="2025-11-17T09:44:06Z" level=info msg="renew lease owned by NodeHealthCheck-nhc-with-mdr setAcquireTime=false" 2025-11-17T09:44:06.573830144Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-with-mdr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"master-0-1","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"master-0-1-pq8fp","uid":"e6cae342-aa0b-47bb-a449-4b5218fb5600","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-11-17T09:42:56Z","templateName":"selfnoderemediationtemplate-sample"}]}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-17T09:40:30Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-17T09:44:03Z"}, "patch": "{\"status\":{\"healthyNodes\":2,\"observedNodes\":3}}"} 2025-11-17T09:44:06.778227258Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": null, "requeue": false, "requeuAfter": "10m0s"} 2025-11-17T09:44:44.261300558Z INFO adding NHC to reconcile queue for handling node {"node": "master-0-1", "NHC": "nhc-with-mdr"} 2025-11-17T09:44:44.261349221Z INFO adding NHC to reconcile queue for handling node {"node": "master-0-1", "NHC": "nhc-with-mdr"} 2025-11-17T09:44:44.261381991Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:44.272159253Z INFO controllers.NodeHealthCheck Node is going to match unhealthy condition {"node": "master-0-1", "condition type": "Ready", "condition status": "False", "duration left": "1m29.727843893s"} 2025-11-17T09:44:44.283039278Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-with-mdr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-17T09:42:57Z"} 2025-11-17T09:44:44.28306882Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-0"} time="2025-11-17T09:44:44Z" level=info msg="invalidating lease" time="2025-11-17T09:44:44Z" level=info msg="getting lease" 2025-11-17T09:44:44.383573811Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-2"} time="2025-11-17T09:44:44Z" level=info msg="invalidating lease" time="2025-11-17T09:44:44Z" level=info msg="getting lease" 2025-11-17T09:44:44.385521017Z INFO controllers.NodeHealthCheck Ignoring node, because it was unhealthy, and is likely to be unhealthy again. {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-1"} 2025-11-17T09:44:44.385784474Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": null, "requeue": false, "requeuAfter": "1m30.727843893s"} 2025-11-17T09:44:46.98802531Z INFO adding NHC to reconcile queue for handling node {"node": "master-0-1", "NHC": "nhc-with-mdr"} 2025-11-17T09:44:46.988084887Z INFO adding NHC to reconcile queue for handling node {"node": "master-0-1", "NHC": "nhc-with-mdr"} 2025-11-17T09:44:46.988132406Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:46.9995011Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-with-mdr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-17T09:42:57Z"} 2025-11-17T09:44:46.999530087Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-2"} time="2025-11-17T09:44:47Z" level=info msg="invalidating lease" time="2025-11-17T09:44:47Z" level=info msg="getting lease" 2025-11-17T09:44:47.001460884Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-0"} time="2025-11-17T09:44:47Z" level=info msg="invalidating lease" time="2025-11-17T09:44:47Z" level=info msg="getting lease" 2025-11-17T09:44:47.099999308Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-1"} 2025-11-17T09:44:47.101985603Z INFO controllers.NodeHealthCheck.resource manager setting a delay for node getting healthy {"node name": "master-0-1", "delay in seconds": 5} 2025-11-17T09:44:47.110136504Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-with-mdr","uid":"0f4c524f-1cda-49cd-8f73-75243a3f6005","controller":false}} 2025-11-17T09:44:47.110205369Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-with-mdr", "Remediation CR Name": "master-0-1-pq8fp", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-17T09:44:47.11021678Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-with-mdr","uid":"0f4c524f-1cda-49cd-8f73-75243a3f6005","controller":false}} 2025-11-17T09:44:47.110226023Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-with-mdr", "Remediation CR Name": "master-0-1-pq8fp", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-17T09:44:47.110293772Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-with-mdr", "new status": {"observedNodes":3,"healthyNodes":2,"unhealthyNodes":[{"name":"master-0-1","remediations":[{"resource":{"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"master-0-1-pq8fp","uid":"e6cae342-aa0b-47bb-a449-4b5218fb5600","apiVersion":"self-node-remediation.medik8s.io/v1alpha1"},"started":"2025-11-17T09:42:56Z","templateName":"selfnoderemediationtemplate-sample"}],"conditionsHealthyTimestamp":"2025-11-17T09:44:47Z","healthyDelayed":true}],"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-17T09:40:30Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Remediating","reason":"NHC is remediating 1 nodes","lastUpdateTime":"2025-11-17T09:44:06Z"}, "patch": "{\"status\":{\"unhealthyNodes\":[{\"conditionsHealthyTimestamp\":\"2025-11-17T09:44:47Z\",\"healthyDelayed\":true,\"name\":\"master-0-1\",\"remediations\":[{\"resource\":{\"apiVersion\":\"self-node-remediation.medik8s.io/v1alpha1\",\"kind\":\"SelfNodeRemediation\",\"name\":\"master-0-1-pq8fp\",\"namespace\":\"openshift-workload-availability\",\"uid\":\"e6cae342-aa0b-47bb-a449-4b5218fb5600\"},\"started\":\"2025-11-17T09:42:56Z\",\"templateName\":\"selfnoderemediationtemplate-sample\"}]}]}}"} 2025-11-17T09:44:47.315360652Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": null, "requeue": false, "requeuAfter": "6s"} 2025-11-17T09:44:47.315451011Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:47.32456962Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-with-mdr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-17T09:42:57Z"} 2025-11-17T09:44:47.324595373Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-0"} time="2025-11-17T09:44:47Z" level=info msg="invalidating lease" time="2025-11-17T09:44:47Z" level=info msg="getting lease" 2025-11-17T09:44:47.326054528Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-1"} 2025-11-17T09:44:47.327466662Z INFO controllers.NodeHealthCheck.resource manager delaying node getting healthy {"node name": "master-0-1", "remaining time in seconds": 4.672535049} 2025-11-17T09:44:47.327484366Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-2"} time="2025-11-17T09:44:47Z" level=info msg="invalidating lease" time="2025-11-17T09:44:47Z" level=info msg="getting lease" 2025-11-17T09:44:47.329475534Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": null, "requeue": false, "requeuAfter": "5.672535049s"} 2025-11-17T09:44:53.002572103Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:53.009674772Z INFO controllers.NodeHealthCheck found condition {"NodeHealthCheck name": "nhc-with-mdr", "type": "Succeeded", "status": "Unknown", "reason": "", "message": "", "lastTransition": "2025-11-17T09:42:57Z"} 2025-11-17T09:44:53.009701488Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-2"} time="2025-11-17T09:44:53Z" level=info msg="invalidating lease" time="2025-11-17T09:44:53Z" level=info msg="getting lease" 2025-11-17T09:44:53.011420065Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-0"} time="2025-11-17T09:44:53Z" level=info msg="invalidating lease" time="2025-11-17T09:44:53Z" level=info msg="getting lease" 2025-11-17T09:44:53.013193576Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-1"} 2025-11-17T09:44:53.014615823Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "master-0-1"} 2025-11-17T09:44:53.021170183Z INFO controllers.NodeHealthCheck.resource manager deleted remediation CR {"name": "master-0-1-pq8fp"} 2025-11-17T09:44:53.02121862Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-with-mdr","uid":"0f4c524f-1cda-49cd-8f73-75243a3f6005","controller":false}} 2025-11-17T09:44:53.021247166Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-with-mdr", "Remediation CR Name": "master-0-1-pq8fp", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-17T09:44:53.021256083Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-with-mdr","uid":"0f4c524f-1cda-49cd-8f73-75243a3f6005","controller":false}} 2025-11-17T09:44:53.021262344Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-with-mdr", "Remediation CR Name": "master-0-1-pq8fp", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-17T09:44:53.021266998Z DEBUG events [remediation] Deleted remediation CR of kind SelfNodeRemediation with name master-0-1-pq8fp {"type": "Normal", "object": {"kind":"NodeHealthCheck","name":"nhc-with-mdr","uid":"0f4c524f-1cda-49cd-8f73-75243a3f6005","apiVersion":"remediation.medik8s.io/v1alpha1","resourceVersion":"73205"}, "reason": "RemediationRemoved"} 2025-11-17T09:44:53.021357363Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": null, "requeue": false, "requeuAfter": "11s"} 2025-11-17T09:44:53.021401818Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:53.111473663Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-0"} time="2025-11-17T09:44:53Z" level=info msg="invalidating lease" time="2025-11-17T09:44:53Z" level=info msg="getting lease" 2025-11-17T09:44:53.113182894Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-1"} 2025-11-17T09:44:53.211990289Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "master-0-1"} 2025-11-17T09:44:53.214276147Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-2"} time="2025-11-17T09:44:53Z" level=info msg="invalidating lease" time="2025-11-17T09:44:53Z" level=info msg="getting lease" 2025-11-17T09:44:53.216228889Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": null, "requeue": false, "requeuAfter": "17s"} 2025-11-17T09:44:59.015552986Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-with-mdr","uid":"0f4c524f-1cda-49cd-8f73-75243a3f6005","controller":false}} 2025-11-17T09:44:59.0156021Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-with-mdr", "Remediation CR Name": "master-0-1-pq8fp", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-17T09:44:59.015617966Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-with-mdr","uid":"0f4c524f-1cda-49cd-8f73-75243a3f6005","controller":false}} 2025-11-17T09:44:59.015622155Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-with-mdr", "Remediation CR Name": "master-0-1-pq8fp", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-17T09:44:59.015649639Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:59.034846386Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-0"} time="2025-11-17T09:44:59Z" level=info msg="invalidating lease" time="2025-11-17T09:44:59Z" level=info msg="getting lease" 2025-11-17T09:44:59.135544188Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-1"} 2025-11-17T09:44:59.145160192Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-with-mdr","uid":"0f4c524f-1cda-49cd-8f73-75243a3f6005","controller":false}} 2025-11-17T09:44:59.145193688Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-with-mdr", "Remediation CR Name": "master-0-1-pq8fp", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-17T09:44:59.145203903Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-with-mdr","uid":"0f4c524f-1cda-49cd-8f73-75243a3f6005","controller":false}} 2025-11-17T09:44:59.145207834Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-with-mdr", "Remediation CR Name": "master-0-1-pq8fp", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-17T09:44:59.145516247Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "master-0-1"} 2025-11-17T09:44:59.147675948Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-2"} time="2025-11-17T09:44:59Z" level=info msg="invalidating lease" time="2025-11-17T09:44:59Z" level=info msg="getting lease" 2025-11-17T09:44:59.23568199Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": null, "requeue": false, "requeuAfter": "17s"} 2025-11-17T09:44:59.235761772Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:44:59.336079499Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-0"} time="2025-11-17T09:44:59Z" level=info msg="invalidating lease" time="2025-11-17T09:44:59Z" level=info msg="getting lease" 2025-11-17T09:44:59.337971676Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-1"} 2025-11-17T09:44:59.437580598Z INFO controllers.NodeHealthCheck.resource manager delaying for node getting healthy is done, about to remove the remediation CR {"node name": "master-0-1"} 2025-11-17T09:44:59.442661447Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-2"} time="2025-11-17T09:44:59Z" level=info msg="invalidating lease" time="2025-11-17T09:44:59Z" level=info msg="getting lease" 2025-11-17T09:44:59.537038365Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": null, "requeue": false, "requeuAfter": "17s"} 2025-11-17T09:45:00.182319532Z INFO controllers.NodeHealthCheck.WatchManager Request info {"owner ref": {"apiVersion":"remediation.medik8s.io/v1alpha1","kind":"NodeHealthCheck","name":"nhc-with-mdr","uid":"0f4c524f-1cda-49cd-8f73-75243a3f6005","controller":false}} 2025-11-17T09:45:00.182350894Z INFO controllers.NodeHealthCheck.WatchManager mapper: found NHC for remediation CR {"NHC Name": "nhc-with-mdr", "Remediation CR Name": "master-0-1-pq8fp", "Remediation CR Kind": "SelfNodeRemediation"} 2025-11-17T09:45:00.182387681Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:45:00.194342003Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-0"} time="2025-11-17T09:45:00Z" level=info msg="invalidating lease" time="2025-11-17T09:45:00Z" level=info msg="getting lease" 2025-11-17T09:45:00.294697784Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-1"} time="2025-11-17T09:45:00Z" level=info msg="invalidating lease" time="2025-11-17T09:45:00Z" level=info msg="getting lease" 2025-11-17T09:45:00.398895063Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-2"} time="2025-11-17T09:45:00Z" level=info msg="invalidating lease" time="2025-11-17T09:45:00Z" level=info msg="getting lease" 2025-11-17T09:45:00.495967337Z INFO controllers.NodeHealthCheck Patching NHC status {"NodeHealthCheck name": "nhc-with-mdr", "new status": {"observedNodes":3,"healthyNodes":3,"conditions":[{"type":"Disabled","status":"False","lastTransitionTime":"2025-11-17T09:40:30Z","reason":"NodeHealthCheckEnabled","message":"No issues found, NodeHealthCheck is enabled."}],"phase":"Enabled","reason":"NHC is enabled, no ongoing remediation","lastUpdateTime":"2025-11-17T09:44:47Z"}, "patch": "{\"status\":{\"healthyNodes\":3,\"phase\":\"Enabled\",\"reason\":\"NHC is enabled, no ongoing remediation\",\"unhealthyNodes\":null}}"} 2025-11-17T09:45:00.701325461Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": null, "requeue": false, "requeuAfter": "0s"} 2025-11-17T09:45:04.021838358Z INFO controllers.NodeHealthCheck reconciling {"NodeHealthCheck name": "nhc-with-mdr"} 2025-11-17T09:45:04.030105595Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-1"} time="2025-11-17T09:45:04Z" level=info msg="invalidating lease" time="2025-11-17T09:45:04Z" level=info msg="getting lease" 2025-11-17T09:45:04.033349757Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-2"} time="2025-11-17T09:45:04Z" level=info msg="invalidating lease" time="2025-11-17T09:45:04Z" level=info msg="getting lease" 2025-11-17T09:45:04.034983472Z INFO controllers.NodeHealthCheck handling healthy node {"NodeHealthCheck name": "nhc-with-mdr", "node": "master-0-0"} time="2025-11-17T09:45:04Z" level=info msg="invalidating lease" time="2025-11-17T09:45:04Z" level=info msg="getting lease" 2025-11-17T09:45:04.036380485Z INFO controllers.NodeHealthCheck reconcile end {"NodeHealthCheck name": "nhc-with-mdr", "error": null, "requeue": false, "requeuAfter": "0s"} SNR Logs: 2025-11-17T08:42:04.40196435Z INFO setup Go Version: go1.23.4 (Red Hat 1.23.4-1.el9) X:strictfipsruntime 2025-11-17T08:42:04.402027191Z INFO setup Go OS/Arch: linux/amd64 2025-11-17T08:42:04.402030278Z INFO setup Operator Version: 0bfa3d8 2025-11-17T08:42:04.402032307Z INFO setup Git Commit: 0bfa3d8cc4f3191c831d36b702aed4b32b7ef625 2025-11-17T08:42:04.402034745Z INFO setup Build Date: 2025-09-29T06:25:34+00:00 2025-11-17T08:42:04.40203635Z INFO setup HTTP/2 for metrics and webhook server disabled 2025-11-17T08:42:04.402432222Z INFO controller-runtime.metrics Metrics server is starting to listen {"addr": "127.0.0.1:8080"} 2025-11-17T08:42:04.4090429Z INFO utils-taints out of service taint strategy {"isSupported": true, "k8sMajorVersion": 1, "k8sMinorVersion": 32} 2025-11-17T08:42:04.409068839Z INFO utils-taints out of service taint strategy {"isGA": true, "k8sMajorVersion": 1, "k8sMinorVersion": 32} 2025-11-17T08:42:04.409076253Z INFO setup Starting as a manager that installs the daemonset 2025-11-17T08:42:04.409094895Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig"} 2025-11-17T08:42:04.409145357Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationConfig", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2025-11-17T08:42:04.409269833Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationconfig"} 2025-11-17T08:42:04.409342023Z INFO controller-runtime.builder Registering a mutating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-11-17T08:42:04.409399317Z INFO controller-runtime.webhook Registering webhook {"path": "/mutate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-11-17T08:42:04.409441453Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediationTemplate", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-11-17T08:42:04.409499942Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediationtemplate"} 2025-11-17T08:42:04.409536947Z INFO controller-runtime.builder skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation"} 2025-11-17T08:42:04.409574049Z INFO controller-runtime.builder Registering a validating webhook {"GVK": "self-node-remediation.medik8s.io/v1alpha1, Kind=SelfNodeRemediation", "path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2025-11-17T08:42:04.409647067Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-self-node-remediation-medik8s-io-v1alpha1-selfnoderemediation"} 2025-11-17T08:42:04.409722456Z INFO setup starting manager 2025-11-17T08:42:04.409896594Z INFO controller-runtime.webhook.webhooks Starting webhook server 2025-11-17T08:42:04.409945387Z INFO Starting server {"kind": "health probe", "addr": "[::]:8081"} 2025-11-17T08:42:04.410117189Z INFO starting server {"path": "/metrics", "kind": "metrics", "addr": "127.0.0.1:8080"} 2025-11-17T08:42:04.410485336Z INFO controller-runtime.certwatcher Updated current TLS certificate I1117 08:42:04.410578 1 leaderelection.go:245] attempting to acquire leader lease openshift-workload-availability/547f6cb6.medik8s.io... 2025-11-17T08:42:04.410606096Z INFO controller-runtime.certwatcher Starting certificate watcher 2025-11-17T08:42:04.411255777Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} I1117 08:42:21.729090 1 leaderelection.go:255] successfully acquired lease openshift-workload-availability/547f6cb6.medik8s.io 2025-11-17T08:42:21.72916557Z DEBUG events self-node-remediation-controller-manager-7dc64b6879-ss67r_2080ebd4-d08c-4f8d-8b0e-7af064c949f2 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"openshift-workload-availability","name":"547f6cb6.medik8s.io","uid":"5b0289d9-6196-428d-961d-a550f5f970ed","apiVersion":"coordination.k8s.io/v1","resourceVersion":"52259"}, "reason": "LeaderElection"} 2025-11-17T08:42:21.729340674Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1alpha1.SelfNodeRemediationConfig"} 2025-11-17T08:42:21.729366503Z INFO Starting EventSource {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "source": "kind source: *v1.DaemonSet"} 2025-11-17T08:42:21.729370383Z INFO Starting Controller {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig"} 2025-11-17T08:42:21.729470803Z INFO Starting EventSource {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "source": "kind source: *v1alpha1.SelfNodeRemediation"} 2025-11-17T08:42:21.729487991Z INFO Starting Controller {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation"} 2025-11-17T08:42:21.745113081Z INFO selfnoderemediationconfig-resource validate create {"name": "self-node-remediation-config"} 2025-11-17T08:42:21.832777312Z INFO Starting workers {"controller": "selfnoderemediationconfig", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediationConfig", "worker count": 1} 2025-11-17T08:42:21.832787494Z INFO Starting workers {"controller": "selfnoderemediation", "controllerGroup": "self-node-remediation.medik8s.io", "controllerKind": "SelfNodeRemediation", "worker count": 1} 2025-11-17T08:42:21.832920988Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-11-17T08:42:21.935004159Z INFO controllers.SelfNodeRemediationConfig Creating new certs 2025-11-17T08:42:25.764177846Z INFO controllers.SelfNodeRemediationConfig Storing certs in new secret 2025-11-17T08:42:25.768088403Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-11-17T08:42:25.769753702Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025-11-17T08:42:25.769796011Z INFO controllers.SelfNodeRemediationConfig snr didn't find old daemonset to be deleted 2025/11/17 08:42:25 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/11/17 08:42:25 does not exist, creating (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/11/17 08:42:25 successfully created (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025-11-17T08:42:25.78300103Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-11-17T08:42:25.783037488Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-11-17T08:42:25.78304311Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-11-17T08:42:25.783638748Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/11/17 08:42:25 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/11/17 08:42:25 update was successful 2025-11-17T08:42:31.743066285Z INFO selfnoderemediationtemplate-resource default {"name": "self-node-remediation-automatic-strategy-template"} 2025-11-17T08:42:31.750946651Z INFO selfnoderemediationtemplate-resource validate create {"name": "self-node-remediation-automatic-strategy-template"} 2025-11-17T09:26:33.706499617Z INFO selfnoderemediationtemplate-resource default {"name": "selfnoderemediationtemplate-sample"} 2025-11-17T09:26:33.709790233Z INFO selfnoderemediationtemplate-resource validate create {"name": "selfnoderemediationtemplate-sample"} 2025-11-17T09:40:30.216232644Z INFO selfnoderemediationtemplate-resource default {"name": "selfnoderemediationtemplate-sample"} 2025-11-17T09:40:30.219293288Z INFO selfnoderemediationtemplate-resource validate create {"name": "selfnoderemediationtemplate-sample"} 2025-11-17T09:42:56.990700278Z INFO selfnoderemediation-resource validate create {"name": "master-0-1-pq8fp"} 2025-11-17T09:42:57.094328352Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:42:57.094354759Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:42:57.09439349Z DEBUG events [remediation] Remediation started by SNR manager {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"master-0-1-pq8fp","uid":"e6cae342-aa0b-47bb-a449-4b5218fb5600","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"72211"}, "reason": "RemediationStarted"} 2025-11-17T09:42:57.202119527Z INFO selfnoderemediation-resource validate update {"name": "master-0-1-pq8fp"} 2025-11-17T09:42:57.205091353Z INFO controllers.SelfNodeRemediation finalizer added {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:42:57.205224755Z DEBUG events [remediation] Remediation process - successful adding finalizer {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"master-0-1-pq8fp","uid":"e6cae342-aa0b-47bb-a449-4b5218fb5600","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"72216"}, "reason": "AddFinalizer"} 2025-11-17T09:42:57.209799709Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:42:57.20981397Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:42:57.219722139Z INFO controllers.SelfNodeRemediation NoExecute taint added {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node-role.kubernetes.io/master","effect":"NoSchedule"},{"key":"node.kubernetes.io/unreachable","effect":"NoSchedule","timeAdded":"2025-11-17T09:30:09Z"},{"key":"node.kubernetes.io/unreachable","effect":"NoExecute","timeAdded":"2025-11-17T09:30:15Z"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-17T09:42:57Z"}]} 2025-11-17T09:42:57.219784799Z INFO controllers.SelfNodeRemediation Marking node as unschedulable {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}, "node name": "master-0-1"} 2025-11-17T09:42:57.219969982Z DEBUG events [remediation] Remediation process - NoExecute taint added to the unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"master-0-1","uid":"023e8fd6-8b7a-405c-ab1a-b2ddff878b88","apiVersion":"v1","resourceVersion":"72219"}, "reason": "AddNoExecute"} 2025-11-17T09:42:57.229382418Z DEBUG events [remediation] Remediation process - unhealthy node marked as unschedulable {"type": "Normal", "object": {"kind":"Node","name":"master-0-1","uid":"023e8fd6-8b7a-405c-ab1a-b2ddff878b88","apiVersion":"v1","resourceVersion":"72221"}, "reason": "MarkUnschedulable"} 2025-11-17T09:42:57.23573101Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:42:57.235749339Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:42:57.235939474Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}, "node name": "master-0-1"} 2025-11-17T09:42:57.243311339Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:42:57.243330787Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:42:57.243562363Z INFO controllers.SelfNodeRemediation waiting for unschedulable taint to appear {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}, "node name": "master-0-1"} 2025-11-17T09:42:58.236186054Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:42:58.236205024Z INFO controllers.SelfNodeRemediation pre-reboot not completed yet, prepare for rebooting {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:42:58.236440511Z INFO rebootDurationCalculator No SafeTimeToAssumeNodeRebootedSeconds specified, using calculated minimum safe reboot time {"calculated minimum time in seconds": 120} 2025-11-17T09:42:58.236460373Z INFO controllers.SelfNodeRemediation setting SNR's time to assume node has been rebooted {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}, "node name": "master-0-1", "time": "2025-11-17 09:44:58.236459748 +0000 UTC m=+3773.846811871"} 2025-11-17T09:42:58.236579007Z DEBUG events [remediation] Remediation process - about to update required fencing time on snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"master-0-1-pq8fp","uid":"e6cae342-aa0b-47bb-a449-4b5218fb5600","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"72224"}, "reason": "UpdateTimeAssumedRebooted"} 2025-11-17T09:42:58.241497855Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:42:58.241516944Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}, "node name": "master-0-1", "time left": "2m0.758483772s"} E1117 09:43:39.576444 1 leaderelection.go:364] Failed to update lock: rpc error: code = Unavailable desc = error reading from server: read tcp 192.168.123.143:55104->192.168.123.141:2379: read: connection timed out 2025-11-17T09:44:47.107450954Z INFO selfnoderemediation-resource validate update {"name": "master-0-1-pq8fp"} 2025-11-17T09:44:47.110276558Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:44:47.110290367Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}, "node name": "master-0-1", "time left": "11.88971046s"} 2025-11-17T09:44:53.021572389Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:44:53.021590264Z INFO controllers.SelfNodeRemediation Node didn't reboot yet, waiting for it to reboot {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}, "node name": "master-0-1", "time left": "5.978410556s"} 2025-11-17T09:44:59.005629262Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:44:59.005764094Z INFO controllers.SelfNodeRemediation TimeAssumedRebooted is old. The unhealthy node assumed to been rebooted {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}, "node name": "master-0-1"} 2025-11-17T09:44:59.01635322Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:44:59.02444191Z INFO controllers.SelfNodeRemediation out-of-service taint added {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node-role.kubernetes.io/master","effect":"NoSchedule"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-17T09:42:57Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-11-17T09:42:57Z"},{"key":"node.kubernetes.io/out-of-service","value":"nodeshutdown","effect":"NoExecute","timeAdded":"2025-11-17T09:44:59Z"}]} 2025-11-17T09:44:59.024512889Z DEBUG events [remediation] Remediation process - add out-of-service taint to unhealthy node {"type": "Normal", "object": {"kind":"Node","name":"master-0-1","uid":"023e8fd6-8b7a-405c-ab1a-b2ddff878b88","apiVersion":"v1","resourceVersion":"74094"}, "reason": "AddOutOfService"} 2025-11-17T09:44:59.139368929Z INFO controllers.SelfNodeRemediation out-of-service taint removed {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node-role.kubernetes.io/master","effect":"NoSchedule"},{"key":"medik8s.io/remediation","value":"self-node-remediation","effect":"NoExecute","timeAdded":"2025-11-17T09:42:57Z"},{"key":"node.kubernetes.io/unschedulable","effect":"NoSchedule","timeAdded":"2025-11-17T09:42:57Z"}]} 2025-11-17T09:44:59.13946604Z DEBUG events [remediation] Remediation process - remove out-of-service taint from node {"type": "Normal", "object": {"kind":"Node","name":"master-0-1","uid":"023e8fd6-8b7a-405c-ab1a-b2ddff878b88","apiVersion":"v1","resourceVersion":"74108"}, "reason": "RemoveOutOfService"} 2025-11-17T09:44:59.139532478Z DEBUG events [remediation] Remediation process - finished deleting unhealthy node resources {"type": "Normal", "object": {"kind":"Node","name":"master-0-1","uid":"023e8fd6-8b7a-405c-ab1a-b2ddff878b88","apiVersion":"v1","resourceVersion":"74108"}, "reason": "DeleteResources"} 2025-11-17T09:44:59.145749377Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:44:59.145767102Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:44:59.153099381Z DEBUG events [remediation] Remediation process - mark healthy remediated node as schedulable {"type": "Normal", "object": {"kind":"Node","name":"master-0-1","uid":"023e8fd6-8b7a-405c-ab1a-b2ddff878b88","apiVersion":"v1","resourceVersion":"74113"}, "reason": "MarkNodeSchedulable"} 2025-11-17T09:45:00.16206255Z INFO controllers.SelfNodeRemediation Remediating with OutOfServiceTaint Remediation strategy (auto-selected) {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:45:00.162079243Z INFO controllers.SelfNodeRemediation fencing completed, cleaning up {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:45:00.169206921Z INFO controllers.SelfNodeRemediation NoExecute taint removed {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}, "new taints": [{"key":"node-role.kubernetes.io/master","effect":"NoSchedule"}]} 2025-11-17T09:45:00.169512837Z DEBUG events [remediation] Remediation process - remove NoExecute taint from healthy remediated node {"type": "Normal", "object": {"kind":"Node","name":"master-0-1","uid":"023e8fd6-8b7a-405c-ab1a-b2ddff878b88","apiVersion":"v1","resourceVersion":"74188"}, "reason": "RemoveNoExecuteTaint"} 2025-11-17T09:45:00.17471513Z INFO selfnoderemediation-resource validate update {"name": "master-0-1-pq8fp"} 2025-11-17T09:45:00.182299189Z INFO controllers.SelfNodeRemediation finalizer removed {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:45:00.182535738Z DEBUG events [remediation] Remediation process - remove finalizer from snr {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"master-0-1-pq8fp","uid":"e6cae342-aa0b-47bb-a449-4b5218fb5600","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"74109"}, "reason": "RemoveFinalizer"} 2025-11-17T09:45:00.182562443Z DEBUG events [remediation] Remediation finished {"type": "Normal", "object": {"kind":"SelfNodeRemediation","namespace":"openshift-workload-availability","name":"master-0-1-pq8fp","uid":"e6cae342-aa0b-47bb-a449-4b5218fb5600","apiVersion":"self-node-remediation.medik8s.io/v1alpha1","resourceVersion":"74109"}, "reason": "RemediationFinished"} 2025-11-17T09:45:00.186662627Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:45:01.187021486Z INFO controllers.SelfNodeRemediation SNR already deleted {"pod": "manager", "selfnoderemediation": {"name":"master-0-1-pq8fp","namespace":"openshift-workload-availability"}} 2025-11-17T09:55:47.197192055Z INFO controllers.SelfNodeRemediationConfig Syncing certs 2025-11-17T09:55:47.197258736Z INFO controllers.SelfNodeRemediationConfig Cert secret already exists 2025-11-17T09:55:47.197262526Z INFO controllers.SelfNodeRemediationConfig.syncConfigDaemonset Start to sync config daemonset 2025-11-17T09:55:47.197834858Z INFO controllers.SelfNodeRemediationConfig Updating DS tolerations 2025/11/17 09:55:47 reconciling (apps/v1, Kind=DaemonSet) openshift-workload-availability/self-node-remediation-ds 2025/11/17 09:55:47 update was successful